Fix netifaces fallback

If psutil is not available, continue with netifaces as before

confluent_client/addattribs.py

@@ -14,11 +14,16 @@ import shutil
 
 shutil.copyfile('doc/man/nodeattrib.ronn.tmpl', 'doc/man/nodeattrib.ronn')
 shutil.copyfile('doc/man/nodegroupattrib.ronn.tmpl', 'doc/man/nodegroupattrib.ronn')
-with open('doc/man/nodeattrib.ronn', 'a') as outf:
-    for field in sorted(attr.node):
-        outf.write('\n* `{0}`:\n  {1}\n'.format(field, attr.node[field]['description']))
-with open('doc/man/nodegroupattrib.ronn', 'a') as outf:
-    for field in sorted(attr.node):
-        outf.write('\n* `{0}`:\n  {1}\n'.format(field, attr.node[field]['description']))
 
+def append_attributes(filename):
+    with open(filename, 'a') as outf:
+        for field in sorted(attr.node):
+            outf.write('\n* `{0}`:\n  {1}\n'.format(field, attr.node[field]['description']))
+            # Optionally write valid values if they exist
+            for key, values in attr.node[field].items():
+                if key.startswith('valid'):
+                    values_formatted = ', '.join("'{0}'".format(v) for v in values)
+                    outf.write(f'\n    Valid values: {values_formatted}\n')
 
+append_attributes('doc/man/nodeattrib.ronn')
+append_attributes('doc/man/nodegroupattrib.ronn')

confluent_client/bin/confetty

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
@@ -654,13 +654,17 @@ def quitconfetty(code=0, fullexit=False, fixterm=True):
 def get_session_node(shellargs):
     # straight to node console
     if len(shellargs) == 1 and ' ' not in shellargs[0]:
-        return shellargs[0]
+        targ = "/nodes/%s/console/session" % shellargs[0]
+        return targ, shellargs[0]
     if len(shellargs) == 2 and shellargs[0] == 'start':
         args = [s for s in shellargs[1].split('/') if s]
         if len(args) == 4 and args[0] == 'nodes' and args[2] == 'console' and \
                 args[3] == 'session':
-            return args[1]
-    return None
+            return shellargs[1], args[1]
+        if len(args) == 5 and args[0] == 'nodes' and args[2] == 'shell' and \
+                args[3] == 'sessions':
+            return shellargs[1], args[1]
+    return None, None
 
 
 def run_inline_command(path, arg, completion, **kwargs):
@@ -917,10 +921,10 @@ def main():
     doexit = False
     inconsole = False
     pendingcommand = ""
-    session_node = get_session_node(shellargs)
+    targ, session_node = get_session_node(shellargs)
     if session_node is not None:
         consoleonly = True
-        do_command("start /nodes/%s/console/session" % session_node, netserver)
+        do_command("start %s" % targ, netserver)
         doexit = True
     elif shellargs:
         do_command(shellargs, netserver)

confluent_client/bin/confluent2hosts

@@ -118,6 +118,7 @@ def main():
     ap.add_argument('-a', '--attrib', help='Pull ip addresses and hostnames from attribute database', action='store_true')
     ap.add_argument('-i', '--ip', help='Expression to generate addresses (e.g. 172.16.1.{n1} or fd2b:246f:8a50::{n1:x})')
     ap.add_argument('-n', '--name', help='Expression for name to add ({node}-compute, etc). If unspecified, "{node} {node}.{dns.domain}" will be used', action='append')
+    ap.add_argument('-f', '--fqdn-first', help='Put the FQDN first in the hosts entries', action='store_true')
     args = ap.parse_args()
     c = client.Command()
     if args.name:
@@ -173,7 +174,13 @@ def main():
                                     break
                             else:
                                 for name in list(names):
-                                    names.append('{0}.{1}'.format(name, mydomain))
+                                    fqdn = '{0}.{1}'.format(name, mydomain)
+                                    if args.fqdn_first:
+                                        # Insert FQDN at the beginning if --fqdn-first flag is set
+                                        names.insert(0, fqdn)
+                                    else:
+                                        # Otherwise, append FQDN at the end (original behavior)
+                                        names.append(fqdn)
                     names = ' '.join(names)
                     merger.add_entry(ipdb[node][currnet], names)
         merger.write_out('/etc/whatnowhosts')

confluent_client/bin/dir2img

@@ -8,30 +8,35 @@ import os
 import subprocess
 import sys
 
-def create_image(directory, image, label=None):
-    ents = 0
-    datasz = 512
-    for dir in os.walk(sys.argv[1]):
-        ents += 1
-        for filen in dir[2]:
+def create_image(directory, image, label=None, esize=0, totalsize=None):
+
+    if totalsize:
+        datasz = totalsize * 1048576
+    else:
+        ents = 0
+        datasz = 512 + (esize * 1048576)
+        for dir in os.walk(sys.argv[1]):
             ents += 1
-            filename = os.path.join(dir[0], filen)
-            currsz = os.path.getsize(filename)
-            # assuming up to 65k cluster
-            currsz = (currsz // 512 +1) * 512
-            datasz += currsz
-    datasz += ents * 32768
-    datasz = datasz // 16384 + 1
+            for filen in dir[2]:
+                ents += 1
+                filename = os.path.join(dir[0], filen)
+                currsz = os.path.getsize(filename)
+                # assuming up to 65k cluster
+                currsz = (currsz // 512 + 1) * 512
+                datasz += currsz
+        datasz += ents * 32768
+    datasz = datasz // 65536 + 1
     with open(image, 'wb') as imgfile:
-        imgfile.seek(datasz * 16384 - 1)
+        imgfile.seek(datasz * 65536 - 1)
         imgfile.write(b'\x00')
     if label:
+        # 4 heads, 32 sectors, means 65k per track
         subprocess.check_call(['mformat', '-i', image, '-v', label,
                                '-r', '16', '-d', '1', '-t', str(datasz),
-                               '-s', '16','-h', '2', '::'])
+                               '-s', '32','-h', '4', '::'])
     else:
         subprocess.check_call(['mformat', '-i', image, '-r', '16', '-d', '1', '-t',
-                                str(datasz), '-s', '16','-h', '2', '::'])
+                                str(datasz), '-s', '32','-h', '4', '::'])
     # Some clustered filesystems will have the lock from mformat
     # linger after close (mformat doesn't unlock)
     # do a blocking wait for shared lock and then explicitly
@@ -56,6 +61,21 @@ if __name__ == '__main__':
             sys.argv[0]))
         sys.exit(1)
     label = None
-    if len(sys.argv) > 3:
-        label = sys.argv[3]
-    create_image(sys.argv[1], sys.argv[2], label)
+    args = sys.argv
+    esize = 0
+    try:
+        earg = args.index('-e')
+        esize = int(args[earg + 1])
+        args = args[:earg] + args[earg +2:]
+    except ValueError:
+        pass
+    totsize = None
+    try:
+        earg = args.index('-s')
+        totsize = int(args[earg + 1])
+        args = args[:earg] + args[earg +2:]
+    except ValueError:
+        pass
+    if len(args) > 3:
+        label = args[3]
+    create_image(args[1], args[2], label, esize, totsize)

confluent_client/bin/nodeapply

@@ -68,6 +68,7 @@ def run():
 
     currprocs = 0
     all = set([])
+    poller = select.epoll()
     pipedesc = {}
     pendingexecs = deque()
     exitcode = 0
@@ -102,19 +103,23 @@ def run():
         cmdv = ['ssh', sshnode] + cmdvbase + cmdstorun[0]
         if currprocs < concurrentprocs:
             currprocs += 1
-            run_cmdv(sshnode, cmdv, all, pipedesc)
+            run_cmdv(sshnode, cmdv, all, poller, pipedesc)
         else:
             pendingexecs.append((sshnode, cmdv))
     if not all or exitcode:
         sys.exit(exitcode)
-    rdy, _, _ = select.select(all, [], [], 10)
+    rdy = poller.poll(10)
     while all:
         pernodeout = {}
         for r in rdy:
+            r = r[0]
             desc = pipedesc[r]
+            r = desc['file']
             node = desc['node']
             data = True
-            while data and select.select([r], [], [], 0)[0]:
+            singlepoller = select.epoll()
+            singlepoller.register(r, select.EPOLLIN)
+            while data and singlepoller.poll(0):
                 data = r.readline()
                 if data:
                     if desc['type'] == 'stdout':
@@ -131,15 +136,17 @@ def run():
                     if ret is not None:
                         exitcode = exitcode | ret
                         all.discard(r)
+                        poller.unregister(r)
                         r.close()
                         if desc['type'] == 'stdout':
                             if idxbynode[node] < len(cmdstorun):
                                 cmdv =  ['ssh', sshnode] + cmdvbase + cmdstorun[idxbynode[node]]
                                 idxbynode[node] += 1
-                                run_cmdv(node, cmdv, all, pipedesc)
+                                run_cmdv(node, cmdv, all, poller, pipedesc)
                             elif pendingexecs:
                                 node, cmdv = pendingexecs.popleft()
-                                run_cmdv(node, cmdv, all, pipedesc)
+                                run_cmdv(node, cmdv, all, poller. pipedesc)
+            singlepoller.close()
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
                 line = client.stringify(line)
@@ -147,19 +154,21 @@ def run():
                 sys.stdout.write('{0}: {1}'.format(node, line))
         sys.stdout.flush()
         if all:
-            rdy, _, _ = select.select(all, [], [], 10)
+            rdy = poller.poll(10)
     sys.exit(exitcode)
 
 
-def run_cmdv(node, cmdv, all, pipedesc):
+def run_cmdv(node, cmdv, all, poller, pipedesc):
     nopen = subprocess.Popen(
         cmdv, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
-                              'type': 'stdout'}
-    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
-                              'type': 'stderr'}
+    pipedesc[nopen.stdout.fileno()] = {'node': node, 'popen': nopen,
+                              'type': 'stdout', 'file': nopen.stdout}
+    pipedesc[nopen.stderr.fileno()] = {'node': node, 'popen': nopen,
+                              'type': 'stderr', 'file': nopen.stderr}
     all.add(nopen.stdout)
+    poller.register(nopen.stdout, select.EPOLLIN)
     all.add(nopen.stderr)
+    poller.register(nopen.stderr, select.EPOLLIN)
 
 
 if __name__ == '__main__':

confluent_client/bin/nodeconsole

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
 import optparse
 import os
 import subprocess
@@ -28,8 +29,31 @@ import confluent.client as client
 import confluent.sortutil as sortutil
 import confluent.logreader as logreader
 import time
+import select
+import signal
 import socket
 import re
+import tty
+import termios
+import fcntl
+import confluent.screensqueeze as sq
+try:
+    from PIL import Image, ImageDraw
+except ImportError:
+    Image = None
+
+try:
+    # sixel is optional, attempt to import but stub out if unavailable
+    import io
+    import sixel
+
+    class DumbWriter(sixel.SixelWriter):
+        def restore_position(self, output):
+            return
+except ImportError:
+    class DumbWriter():
+        def draw(self, imgfile):
+            sys.stderr.write("PySixel not detected, Sixel format display not supported\n")
 
 confettypath = os.path.join(os.path.dirname(sys.argv[0]), 'confetty')
 argparser = optparse.OptionParser(
@@ -46,6 +70,11 @@ argparser.add_option('-l', '--log', action='store_true', default=False,
 argparser.add_option('-T', '--Timestamp', action='store_true', default=False,
                      help= 'Dump log in stdout with timestamps')
 
+argparser.add_option('-s', '--screenshot', action='store_true', default=False,
+                     help='Attempt to grab screenshot and render using kitty image protocol')
+argparser.add_option('-i', '--interval', type='float',
+                     help='Interval in seconds to redraw the screenshot. Currently only '
+                          'works for one node')
 argparser.add_option('-w','--windowed', action='store_true', default=False,
                      help='Open terminal windows for each node.  The '
                           'environment variable NODECONSOLE_WINDOWED_COMMAND '
@@ -69,6 +98,232 @@ argparser.add_option('-w','--windowed', action='store_true', default=False,
 
 (options, args) = argparser.parse_args()
 
+oldtcattr = None
+oldfl = None
+
+def get_coords():
+    sys.stdout.write('\x1b[6n') #
+    sys.stdout.flush()
+    gotreply = select.select([sys.stdin,], [], [], 0.250)[0]
+    if gotreply:
+        response = ''
+        while select.select([sys.stdin,], [], [], 0.1)[0] and 'R' not in response:
+            response += sys.stdin.read()
+        coords = response.replace('R', '').split('[')[1].split(';')
+    #sys.stdout.write('\x1b[{}:{}H'.format(*coords))
+
+def direct_console():
+    global oldtcattr
+    global oldfl
+    oldtcattr = termios.tcgetattr(sys.stdin.fileno())
+    oldfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
+    tty.setraw(sys.stdin.fileno())
+    fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, oldfl | os.O_NONBLOCK)
+
+def indirect_console():
+    fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, oldfl & ~os.O_NONBLOCK)
+    termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, oldtcattr)
+
+def determine_tile_size(numnodes):
+    # for now, smash everything to a common aspect ratio.  16:11
+    # is pretty much wrong for everything, making 4:3 a bit too wide
+    # and 16:9 significantly too narrow, but it is serviceable
+    # An improvement could come with us owning the scaling
+    # instead of delegating to Kitty, which says if we specify both,
+    # we get stretching.  In theory we should be able to get aspect correct
+    # from kitty by omitting, but:
+    # then we don't know how much to move the cursor left after draw_image
+    # Konsole won't scale at all with only partial scaling specified
+    cheight, cwidth, pixwidth, pixheight = sq.get_screengeom()
+    # 16:12 is to roughly account for the 'titles' of the tiles
+    ratio = (pixwidth / 16) / (pixheight / 12)
+    bestdeviation = None
+    bestdims = []
+    for i in range(1, numnodes + 1):
+        number = numnodes
+        while number % i != 0:
+            number += 1
+        columns = i
+        rows = number // i
+        deviation = abs(ratio -  (columns / rows))
+        if bestdeviation is None:
+            bestdeviation = deviation
+            bestdims = [columns, rows]
+        elif deviation < bestdeviation:
+            bestdeviation = deviation
+            bestdims = [columns, rows]
+    # ok, the above algorithm can still pick things like
+    # 1 2 3
+    # 4
+    # So we will let it pick the number of rows, and
+    # then see if we can chop columns and still fit
+    while (bestdims[0] - 1) * bestdims[1] >= numnodes:
+        bestdims[0] = bestdims[0] - 1
+    cellswide = cwidth // bestdims[0]
+    cellshigh = cheight // bestdims[1]
+    tilewidth = cellswide * pixwidth / cwidth
+    tileheight = cellshigh * pixheight / cheight
+    if tilewidth > (tileheight * 16 / 11):
+        tilewidth = tileheight * 16 / 11
+        cellswide = int(tilewidth // (pixwidth / cwidth))
+    if tileheight > (tilewidth * 11 /16):
+        tileheight = tilewidth * 11 / 16
+        cellshigh = int(tileheight // (pixheight / cheight))
+    bestdims = bestdims + [cellswide, cellshigh, cellshigh * bestdims[1]]
+    # incur any scrolling we might get. This allows us to accurately
+    # save/restore cursor or even get coordinates without scrolling fouling
+    # the desired target
+    sys.stdout.write('\n' * bestdims[4])
+    sys.stdout.flush()
+    cursor_up(bestdims[4])
+    return bestdims
+
+cursor_saved = False
+def sticky_cursor():
+    global cursor_saved
+# get cursor restore_position
+    if sys.stdin.isatty() and not cursor_saved:
+        try:
+            direct_console()
+            sys.stdout.write('\x1b7')
+            cursor_saved = True
+        finally:
+            indirect_console()
+    elif cursor_saved:
+        try:
+            direct_console()
+            sys.stdout.write('\x1b8')
+        finally:
+            indirect_console()
+
+def cursor_up(count=1):
+    sys.stdout.write(f'\x1b[{count}A')
+def cursor_down(count=1):
+    sys.stdout.write(f'\x1b[{count}B')
+def cursor_right(count=1):
+    sys.stdout.write(f'\x1b[{count}C')
+def cursor_left(count=1):
+    sys.stdout.write(f'\x1b[{count}D')
+def cursor_save():
+    sys.stdout.write('\x1b7')
+def cursor_restore():
+    sys.stdout.write('\x1b8')
+def cursor_hide():
+    sys.stdout.write('\x1b[?25l')
+def cursor_show():
+    sys.stdout.write('\x1b[?25h')
+
+def get_pix_dimensions(width, height):
+    cheight, cwidth, pixwidth, pixheight = sq.get_screengeom()
+    imgwidth = int(pixwidth / cwidth * width)
+    imgheight = int(pixheight / cheight * height)
+    return imgwidth, imgheight
+
+def draw_text(text, width, height):
+    if Image:
+        maxfntsize = 256
+        imgwidth, imgheight = get_pix_dimensions(width, height)
+        nerr = Image.new(mode='RGB', size=(imgwidth, imgheight), color='green')
+        nd = ImageDraw.Draw(nerr)
+        for txtpiece in text.split('\n'):
+            fntsize = 8
+            while nd.textlength(txtpiece, font_size=fntsize) < int(imgwidth * 0.90):
+                fntsize += 1
+            fntsize -= 1
+            if fntsize < maxfntsize:
+                maxfntsize = fntsize
+        hmargin = int(imgwidth * 0.05)
+        vmargin = int(imgheight * 0.10)
+        nd.text((hmargin, vmargin), text, font_size=maxfntsize)
+        nd.rectangle((0, 0, nerr.width - 1, nerr.height -1), outline='white', width=1)
+        outfile = io.BytesIO()
+        nerr.save(outfile, format='PNG')
+        data = base64.b64encode(outfile.getbuffer())
+        draw_image(data, width, height, doscale=False)
+    else:
+        sys.stdout.write(text)
+        cursor_left(len(txt))
+
+def draw_image(data, width, height, doscale=True):
+    imageformat = os.environ.get('CONFLUENT_IMAGE_PROTOCOL', 'kitty')
+    if doscale and Image and width:
+        bindata = base64.b64decode(data)
+        binfile = io.BytesIO()
+        binfile.write(bindata)
+        binfile.seek(0)
+        try:
+            img = Image.open(binfile)
+        except Exception as e:
+            errstr = 'Error rendering image:\n' + str(e)
+            return draw_text(errstr, width, height)
+        imgwidth, imgheight = get_pix_dimensions(width, height)
+        nimg = Image.new(mode='RGBA', size=(imgwidth, imgheight))
+        imgwidth -= 4
+        imgheight -= 4
+        hscalefact = imgwidth / img.width
+        vscalefact = imgheight / img.height
+        if hscalefact < vscalefact:
+            rzwidth = imgwidth
+            rzheight = int(img.height * hscalefact)
+        else:
+            rzwidth = int(img.width * vscalefact)
+            rzheight = imgheight
+        img = img.resize((rzwidth, rzheight))
+        nd = ImageDraw.Draw(nimg)
+        nd.rectangle((1, 1, rzwidth + 2, rzheight + 2), outline='black', width=1)
+        nd.rectangle((0, 0, rzwidth + 3, rzheight + 3), outline='white', width=1)
+        nimg.paste(img, box=(2, 2))
+        outfile = io.BytesIO()
+        nimg.save(outfile, format='PNG')
+        data = base64.b64encode(outfile.getbuffer())
+    if imageformat == 'sixel':
+        sixel_draw(data)
+    elif imageformat == 'iterm':
+        iterm_draw(data, width, height)
+    else:
+        kitty_draw(data, width, height)
+
+
+def sixel_draw(data):
+    bindata = base64.b64decode(data)
+    binfile = io.BytesIO()
+    binfile.write(bindata)
+    binfile.seek(0)
+    DumbWriter().draw(binfile)
+
+def iterm_draw(data, width, height):
+    if not height:
+        height = 'auto'
+    if not width:
+        width = 'auto'
+    bindata = base64.b64decode(data)
+    datalen = len(bindata)
+    sys.stdout.write(
+        '\x1b]1337;File=inline=1;width={};height={};size={}:'.format(width,height,datalen))
+    sys.stdout.write(data.decode('utf8'))
+    sys.stdout.write('\a')
+    sys.stdout.flush()
+
+
+def kitty_draw(data, width, height):
+    preamble = '\x1b_Ga=T,f=100'
+    if height:
+        preamble += f',r={height},c={width}'
+    #sys.stdout.write(repr(preamble))
+    #sys.stdout.write('\xb[{}D'.format(len(repr(preamble))))
+    #return
+    first = True
+    while data:
+        chunk, data = data[:4096], data[4096:]
+        m = 1 if data else 0
+        if first:
+            sys.stdout.write('{},m={};'.format(preamble, m))
+        else:
+            sys.stdout.write('\x1b_Gm={};'.format(m))
+        sys.stdout.write(chunk.decode('utf8'))
+        sys.stdout.write('\x1b\\')
+        sys.stdout.flush()
+
 pass_through_args = []
 killcon = False
 try:
@@ -106,6 +361,201 @@ if options.Timestamp:
     logreader.dump_to_console(logname)
     sys.exit(0)
 
+def prep_node_tile(node):
+    currcolcell, currrowcell = nodepositions[node]
+    if currcolcell:
+        cursor_right(currcolcell)
+    if currrowcell:
+        cursor_down(currrowcell)
+    sys.stdout.write('▏' + node)
+    cursor_left(len(node) + 1)
+    cursor_down()
+
+def reset_cursor(node):
+    currcolcell, currrowcell = nodepositions[node]
+    if currcolcell:
+        cursor_left(currcolcell)
+    cursor_up(currrowcell + 1)
+
+nodepositions = {}
+numrows = 0
+cwidth = 0
+cheight = 0
+imagedatabynode = {}
+
+def redraw():
+    for node in imagedatabynode:
+        imgdata = imagedatabynode[node]
+        if node in nodepositions:
+            prep_node_tile(node)
+            cursor_save()
+        else:
+            if options.interval is not None:
+                if node != firstnodename:
+                    sys.stderr.write('Multiple nodes not supported for interval')
+                    sys.exit(1)
+                sticky_cursor()
+            sys.stdout.write('{}: '.format(node))
+        # one row is used by our own name, so cheight - 1 for that allowance
+        draw_image(imgdata.encode(), cwidth, cheight - 1 if cheight else cheight)
+        if node in nodepositions:
+            cursor_restore()
+            reset_cursor(node)
+        else:
+            sys.stdout.write('\n')
+        sys.stdout.flush()
+resized = False
+def do_screenshot():
+    global resized
+    global numrows
+    sess = client.Command()
+    if options.tile:
+        imageformat = os.environ.get('CONFLUENT_IMAGE_PROTOCOL', 'kitty')
+        if imageformat not in ('kitty', 'iterm'):
+            sys.stderr.write('Tiled screenshots only supported with kitty or iterm protocol')
+            sys.exit(1)
+        allnodes = []
+        numnodes = 0
+        for res in sess.read('/noderange/{}/nodes/'.format(args[0])):
+            allnodes.append(res['item']['href'].replace('/', ''))
+            numnodes += 1
+        resized = False
+        def do_resize(a=None, b=None):
+            global resized
+            if a:
+                resized = True
+                # on a window resize, clear the old stuff
+                # ideally we'd retain the images and redraw them
+                sys.stdout.write('\x1bc')
+            global numrows
+            global cwidth
+            global cheight
+            cols, rows, cwidth, cheight, numrows = determine_tile_size(numnodes)
+            currcol = 1
+            currcolcell = 0
+            currrowcell = 0
+            for node in allnodes:
+                nodepositions[node] = currcolcell, currrowcell
+                if currcol < cols:
+                    currcol += 1
+                    currcolcell += cwidth
+                else:
+                    currcol = 1
+                    currcolcell = 0
+                    currrowcell += cheight
+            if a:
+                redraw()
+        do_resize()
+        signal.signal(signal.SIGWINCH, do_resize)
+    elif options.interval is not None:
+        sys.stdout.write('\x1bc')
+    firstnodename = None
+    dorefresh = True
+    vnconly = set([])
+    while dorefresh:
+        for res in sess.read('/noderange/{}/console/ikvm_screenshot'.format(args[0])):
+            for node in res.get('databynode', {}):
+                errorstr = ''
+                if not firstnodename:
+                    firstnodename = node
+                error = res['databynode'][node].get('error')
+                if error and 'vnc available' in error:
+                    vnconly.add(node)
+                    continue
+                elif error:
+                    errorstr = error
+                imgdata = res['databynode'][node].get('image', {}).get('imgdata', None)
+                if imgdata:
+                    if len(imgdata) < 32: # We were subjected to error
+                        errorstr = f'Unable to get screenshot'
+                if errorstr or imgdata:
+                    draw_node(node, imgdata, errorstr, firstnodename, cwidth, cheight)
+        if asyncvnc:
+            urlbynode = {}
+            for node in vnconly:
+                for res in sess.update(f'/nodes/{node}/console/ikvm', {'method': 'unix'}):
+                    url = res.get('item', {}).get('href')
+                    if url:
+                        urlbynode[node] = url
+            draw_vnc_grabs(urlbynode, cwidth, cheight)
+            if resized:
+                do_resize(True)
+                resized = False
+        elif vnconly:
+            sys.stderr.write("Require asyncvnc installed to do VNC screenshotting\n")
+        if options.interval is None:
+            dorefresh = False
+        else:
+            dorefresh = True
+            time.sleep(options.interval)
+    sys.exit(0)
+
+try:
+    import asyncio, asyncvnc
+except ImportError:
+    asyncvnc = None
+
+def draw_vnc_grabs(urlbynode, cwidth, cheight):
+    asyncio.run(grab_vncs(urlbynode, cwidth, cheight))
+async def grab_vncs(urlbynode, cwidth, cheight):
+    tasks = []
+    for node in urlbynode:
+        url = urlbynode[node]
+        tasks.append(asyncio.create_task(do_vnc_screenshot(node, url, cwidth, cheight)))
+    await asyncio.gather(*tasks)
+
+async def my_opener(host, port):
+    # really, host is the unix
+    return await asyncio.open_unix_connection(host)
+
+async def do_vnc_screenshot(node, url, cwidth, cheight):
+    async with asyncvnc.connect(url, opener=my_opener) as client:
+        # Retrieve pixels as a 3D numpy array
+        pixels = await client.screenshot()
+        # Save as PNG using PIL/pillow
+        image = Image.fromarray(pixels)
+        outfile = io.BytesIO()
+        image.save(outfile, format='PNG')
+        imgdata = base64.b64encode(outfile.getbuffer()).decode()
+        if imgdata:
+            draw_node(node, imgdata, '', '', cwidth, cheight)
+
+def draw_node(node, imgdata, errorstr, firstnodename, cwidth, cheight):
+    imagedatabynode[node] = imgdata
+    if node in nodepositions:
+        prep_node_tile(node)
+        cursor_save()
+    else:
+        if options.interval is not None:
+            if node != firstnodename:
+                sys.stderr.write('Multiple nodes not supported for interval')
+                sys.exit(1)
+            sticky_cursor()
+        sys.stdout.write('{}: '.format(node))
+    # one row is used by our own name, so cheight - 1 for that allowance
+    if errorstr:
+        draw_text(errorstr, cwidth, cheight -1 if cheight else cheight)
+    else:
+        draw_image(imgdata.encode(), cwidth, cheight - 1 if cheight else cheight)
+    if node in nodepositions:
+        cursor_restore()
+        reset_cursor(node)
+    else:
+        sys.stdout.write('\n')
+    sys.stdout.flush()
+
+if options.screenshot:
+    try:
+        cursor_hide()
+        do_screenshot()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        cursor_show()
+        cursor_down(numrows)
+        sys.stdout.write('\n')
+    sys.exit(0)
+
 def kill(noderange):
     sess = client.Command()
     envstring=os.environ.get('NODECONSOLE_WINDOWED_COMMAND')
@@ -113,29 +563,23 @@ def kill(noderange):
         envstring = 'xterm'
 
     nodes = []
-    for res in sess.read('/noderange/{0}/nodes/'.format(args[0])):
+    for res in sess.read('/noderange/{0}/nodes/'.format(noderange)):
         node = res.get('item', {}).get('href', '/').replace('/', '')
         if not node:
             sys.stderr.write(res.get('error', repr(res)) + '\n')
             sys.exit(1)
         nodes.append(node) 
 
-    for node in nodes:
-        s=socket.socket(socket.AF_UNIX)
-        winid=None
+    for node in nodes:    
+        command = "ps auxww | grep {0} | grep console | egrep '\\b{1}\\b' | grep -v grep | awk '{{print $2}}'".format(envstring, node)
+        process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = process.communicate()
         try:
-            win=subprocess.Popen(['xwininfo', '-tree', '-root'], stdout=subprocess.PIPE)
-            wintr=win.communicate()[0]
-            for line in wintr.decode('utf-8').split('\n'):
-                if 'console: {0}'.format(node) in line or 'confetty' in line:
-                    win_obj = [ele for ele in line.split(' ') if ele.strip()]
-                    winid = win_obj[0]
-        except:
-            print("Error: cannot retrieve window id of node {}".format(node))
-
-        if winid:
-            ps_data=subprocess.Popen(['xkill', '-id', winid ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
+            process_id = stdout.decode('utf-8').split()[0]
+        except IndexError:
+            sys.stderr.write(node + ": console window not found \n")
+            continue
+        subprocess.Popen(["kill", process_id], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     sys.exit(0)
 
 def handle_geometry(envlist, sizegeometry, side_pad=0, top_pad=0, first=False):    
@@ -257,7 +701,6 @@ if options.windowed:
         screenheight -= wmyo
         currx = window_width
         curry = 0
-        maxcol = int(screenwidth/window_width)
 
     for node in sortutil.natural_sort(nodes):
         if options.tile and envlist[0] == 'xterm':
@@ -265,7 +708,7 @@ if options.windowed:
             corrected_y = curry
             xgeometry = '{0}+{1}+{2}'.format(sizegeometry, corrected_x, corrected_y)
             currx += window_width
-            if currx >= screenwidth:
+            if currx + window_width >= screenwidth:
                 currx=0
                 curry += window_height
                 if curry > screenheight:

confluent_client/bin/nodedeploy

@@ -117,6 +117,16 @@ def main(args):
             else:
                 sys.stderr.write('No deployment profiles available, try osdeploy import or imgutil capture\n')
             sys.exit(1)
+        lockednodes = []
+        for lockinfo in c.read('/noderange/{0}/deployment/lock'.format(args.noderange)):
+            for node in lockinfo.get('databynode', {}):
+                lockstate = lockinfo['databynode'][node]['lock']['value']
+                if lockstate == 'locked':
+                    lockednodes.append(node)
+        if lockednodes:
+            sys.stderr.write('Requested noderange has nodes with locked deployment: ' + ','.join(lockednodes))
+            sys.stderr.write('\n')
+            sys.exit(1)
         armonce(args.noderange, c)
         setpending(args.noderange, args.profile, c)
     else:

confluent_client/bin/nodediscover

@@ -123,7 +123,7 @@ def process_header(header):
             fields.append('serial')
         elif datum == 'uuid':
             fields.append('uuid')
-        elif datum in ('bmc', 'imm', 'xcc'):
+        elif datum in ('bmc', 'imm', 'xcc', 'ip'):
             fields.append('hardwaremanagement.manager')
         elif datum in ('bmc gateway', 'xcc gateway', 'imm gateway'):
             fields.append('net.bmc.ipv4_gateway')
@@ -191,6 +191,7 @@ def import_csv(options, session):
             if field in unique_fields:
                 unique_data[field] = set([])
         broken = False
+        alldata=[]
         for record in records:
             currfields = list(fields)
             nodedatum = {}
@@ -207,9 +208,15 @@ def import_csv(options, session):
                 nodedatum[currfield] = datum
             if not datum_complete(nodedatum):
                 sys.exit(1)
+            alldata.append(nodedatum)
+        allthere = True
+        for nodedatum in alldata:
             if not search_record(nodedatum, options, session) and not broken:
+                allthere = False
                 blocking_scan(session)
-            if not search_record(nodedatum, options, session):
+                break
+        for nodedatum in alldata:
+            if not allthere and not search_record(nodedatum, options, session):
                 sys.stderr.write(
                     "Could not match the following data: " +
                     repr(nodedatum) + '\n')
@@ -230,8 +237,12 @@ def import_csv(options, session):
                 print('Defined ' + res['created'])
             else:
                 print(repr(res))
+        child = os.fork()
+        if child:
+            continue
         for mac in maclist:
-            for res in session.update('/discovery/by-mac/{0}'.format(mac),
+            mysess = client.Command()
+            for res in mysess.update('/discovery/by-mac/{0}'.format(mac),
                                       {'node': nodename}):
                 if 'error' in res:
                     sys.stderr.write(res['error'] + '\n')
@@ -241,6 +252,12 @@ def import_csv(options, session):
                     print('Discovered ' + res['assigned'])
                 else:
                     print(repr(res))
+        sys.exit(0)
+    while True:
+        try:
+            os.wait()
+        except ChildProcessError:
+            break
     if exitcode:
         sys.exit(exitcode)
 

confluent_client/bin/nodeeventlog

@@ -78,8 +78,11 @@ exitcode = 0
 def format_event(evt):
     retparts = []
     if 'timestamp' in evt and evt['timestamp'] is not None:
-        display = dt.strptime(evt['timestamp'], '%Y-%m-%dT%H:%M:%S')
-        retparts.append(display.strftime('%m/%d/%Y %H:%M:%S'))
+        try:
+            display = dt.strptime(evt['timestamp'], '%Y-%m-%dT%H:%M:%S')
+            retparts.append(display.strftime('%m/%d/%Y %H:%M:%S'))
+        except ValueError:
+            display = ''    
     dscparts = []
     if evt.get('log_id', None):
         retparts.append(evt['log_id'] + ':')

confluent_client/bin/nodefirmware

@@ -56,7 +56,7 @@ components = ['all']
 
 argparser = optparse.OptionParser(
     usage="Usage: "
-          "%prog <noderange> [list][update [--backup <file>]]|[<components>]")
+          "%prog <noderange> [list][updatestatus][update [--backup <file>]]|[<components>]")
 argparser.add_option('-b', '--backup', action='store_true',
                      help='Target a backup bank rather than primary')
 argparser.add_option('-m', '--maxnodes', type='int',
@@ -65,14 +65,18 @@ argparser.add_option('-m', '--maxnodes', type='int',
              
 (options, args) = argparser.parse_args()
 upfile = None
+querystatus = False
 try:
     noderange = args[0]
     if len(args) > 1:
         if args[1] == 'update':
             upfile = args[2]
         else:
+            comps = []
             if args[1] == 'list':
                 comps = args[2:]
+            elif args[1] == 'updatestatus':
+                querystatus = True
             else:
                 comps = args[1:]
             components = []
@@ -171,7 +175,16 @@ def show_firmware(session):
 
 try:
     session = client.Command()
-    if upfile is None:
+    if querystatus:
+        for res in session.read(
+                '/noderange/{0}/inventory/firmware/updatestatus'.format(noderange)):
+            for node in res.get('databynode', {}):
+                currstat = res['databynode'][node].get('status', None)
+                if currstat:
+                    print('{}: {}'.format(node, currstat))
+                else:
+                    print(repr(res))
+    elif upfile is None:
         show_firmware(session)
     else:
         update_firmware(session, upfile)

confluent_client/bin/nodeping

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -42,6 +42,8 @@ def run():
         usage="Usage: %prog [options] noderange")
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of commands to run at a time')
+    argparser.add_option('-o', '--origname', action='store_true',
+                         help='Use original nodename in print out even if substituted')
     argparser.add_option('-s', '--substitutename',
                          help='Use a different name other than the nodename for ping, with {}, it is the entire name evaluated as an expression, otherwise it is used as a suffix')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
@@ -56,6 +58,7 @@ def run():
 
     currprocs = 0
     all = set([])
+    poller = select.epoll()
     pipedesc = {}
     pendingexecs = deque()
     exitcode = 0
@@ -83,19 +86,29 @@ def run():
             cmdv = ['ping', '-c', '1', '-W', '1', pingnode]
             if currprocs < concurrentprocs:
                 currprocs += 1
-                run_cmdv(pingnode, cmdv, all, pipedesc)
+                if options.origname:
+                    run_cmdv(node, cmdv, all, poller, pipedesc)
+                else:
+                    run_cmdv(pingnode, cmdv, all, poller, pipedesc)
             else:
-                pendingexecs.append((pingnode, cmdv))
+                if options.origname:
+                    pendingexecs.append((node, cmdv))
+                else:
+                    pendingexecs.append((pingnode, cmdv))
     if not all or exitcode:
         sys.exit(exitcode)
-    rdy, _, _ = select.select(all, [], [], 10)
+    rdy = poller.poll(10)
     while all:
         pernodeout = {}
         for r in rdy:
+            r = r[0]
             desc = pipedesc[r]
+            r = desc['file']
             node = desc['node']
             data = True
-            while data and select.select([r], [], [], 0)[0]:
+            singlepoller = select.epoll()
+            singlepoller.register(r, select.EPOLLIN)
+            while data and singlepoller.poll(0):
                 data = r.readline()
                 if not data:
                     pop = desc['popen']
@@ -103,6 +116,7 @@ def run():
                     if ret is not None:
                         exitcode = exitcode | ret
                         all.discard(r)
+                        poller.unregister(r)
                         r.close()
                         if desc['type'] == 'stdout':
                             if ret:
@@ -111,7 +125,8 @@ def run():
                                 print('{0}: ping'.format(node))
                             if pendingexecs:
                                 node, cmdv = pendingexecs.popleft()
-                                run_cmdv(node, cmdv, all, pipedesc)
+                                run_cmdv(node, cmdv, all, poller, pipedesc)
+            singlepoller.close()
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
                 line = client.stringify(line)
@@ -121,19 +136,21 @@ def run():
                     sys.stdout.write('{0}: {1}'.format(node, line))
         sys.stdout.flush()
         if all:
-            rdy, _, _ = select.select(all, [], [], 10)
+            rdy = poller.poll(10)
     sys.exit(exitcode)
 
 
-def run_cmdv(node, cmdv, all, pipedesc):
+def run_cmdv(node, cmdv, all, poller, pipedesc):
     nopen = subprocess.Popen(
         cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
-                              'type': 'stdout'}
-    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
-                              'type': 'stderr'}
+    pipedesc[nopen.stdout.fileno()] = {'node': node, 'popen': nopen,
+                              'type': 'stdout', 'file': nopen.stdout}
+    pipedesc[nopen.stderr.fileno()] = {'node': node, 'popen': nopen,
+                              'type': 'stderr', 'file': nopen.stderr}
     all.add(nopen.stdout)
+    poller.register(nopen.stdout, select.EPOLLIN)
     all.add(nopen.stderr)
+    poller.register(nopen.stderr, select.EPOLLIN)
 
 
 if __name__ == '__main__':

confluent_client/bin/noderun

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -67,6 +67,7 @@ def run():
 
     currprocs = 0
     all = set([])
+    poller = select.epoll()
     pipedesc = {}
     pendingexecs = deque()
     exitcode = 0
@@ -84,19 +85,23 @@ def run():
             cmdv = shlex.split(cmd)
             if currprocs < concurrentprocs:
                 currprocs += 1
-                run_cmdv(node, cmdv, all, pipedesc)
+                run_cmdv(node, cmdv, all, poller, pipedesc)
             else:
                 pendingexecs.append((node, cmdv))
     if not all or exitcode:
         sys.exit(exitcode)
-    rdy, _, _ = select.select(all, [], [], 10)
+    rdy = poller.poll(10)
     while all:
         pernodeout = {}
         for r in rdy:
+            r = r[0]
             desc = pipedesc[r]
+            r = desc['file']
             node = desc['node']
             data = True
-            while data and select.select([r], [], [], 0)[0]:
+            singlepoller = select.epoll()
+            singlepoller.register(r, select.EPOLLIN)
+            while data and singlepoller.poll(0):
                 data = r.readline()
                 if data:
                     if desc['type'] == 'stdout':
@@ -116,10 +121,12 @@ def run():
                     if ret is not None:
                         exitcode = exitcode | ret
                         all.discard(r)
+                        poller.unregister(r)
                         r.close()
                         if desc['type'] == 'stdout' and pendingexecs:
                             node, cmdv = pendingexecs.popleft()
-                            run_cmdv(node, cmdv, all, pipedesc)
+                            run_cmdv(node, cmdv, all, poller, pipedesc)
+            singlepoller.close()
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
                 line = client.stringify(line)
@@ -129,11 +136,11 @@ def run():
                     sys.stdout.write('{0}: {1}'.format(node, line))
         sys.stdout.flush()
         if all:
-            rdy, _, _ = select.select(all, [], [], 10)
+            rdy = poller.poll(10)
     sys.exit(exitcode)
 
 
-def run_cmdv(node, cmdv, all, pipedesc):
+def run_cmdv(node, cmdv, all, poller, pipedesc):
     try:
         nopen = subprocess.Popen(
             cmdv, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -142,13 +149,14 @@ def run_cmdv(node, cmdv, all, pipedesc):
             sys.stderr.write('{0}: Unable to find local executable file "{1}"'.format(node, cmdv[0]))
             return
         raise
-    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
-                              'type': 'stdout'}
-    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
-                              'type': 'stderr'}
+    pipedesc[nopen.stdout.fileno()] = {'node': node, 'popen': nopen,
+                              'type': 'stdout', 'file': nopen.stdout}
+    pipedesc[nopen.stderr.fileno()] = {'node': node, 'popen': nopen,
+                              'type': 'stderr', 'file': nopen.stderr}
     all.add(nopen.stdout)
+    poller.register(nopen.stdout, select.EPOLLIN)
     all.add(nopen.stderr)
-
+    poller.register(nopen.stderr, select.EPOLLIN)
 
 if __name__ == '__main__':
     run()

confluent_client/bin/nodeshell

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -109,6 +109,7 @@ def run():
             ex = exp.get('databynode', ())
             for node in ex:
                 cmdparms.append((node, ex[node]['value']))
+    poller = select.epoll()
     for node, cmd in cmdparms:
         sshnode = nodemap.get(node, node)
         if not isinstance(cmd, str) and not isinstance(cmd, bytes):
@@ -121,19 +122,23 @@ def run():
         cmdv += [sshnode, cmd]
         if currprocs < concurrentprocs:
             currprocs += 1
-            run_cmdv(node, cmdv, all, pipedesc)
+            run_cmdv(node, cmdv, all, poller, pipedesc)
         else:
             pendingexecs.append((node, cmdv))
     if not all or exitcode:
         sys.exit(exitcode)
-    rdy, _, _ = select.select(all, [], [], 10)
+    rdy = poller.poll(10)
     while all:
         pernodeout = {}
         for r in rdy:
+            r = r[0]
             desc = pipedesc[r]
+            r = desc['file']
             node = desc['node']
             data = True
-            while data and select.select([r], [], [], 0)[0]:
+            singlepoller = select.epoll()
+            singlepoller.register(r, select.EPOLLIN)
+            while data and singlepoller.poll(0):
                 data = r.readline()
                 if data:
                     if desc['type'] == 'stdout':
@@ -153,10 +158,12 @@ def run():
                     if ret is not None:
                         exitcode = exitcode | ret
                         all.discard(r)
+                        poller.unregister(r)
                         r.close()
                         if desc['type'] == 'stdout' and pendingexecs:
                             node, cmdv = pendingexecs.popleft()
-                            run_cmdv(node, cmdv, all, pipedesc)
+                            run_cmdv(node, cmdv, all, poller, pipedesc)
+            singlepoller.close()
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
                 line = client.stringify(line)
@@ -167,19 +174,21 @@ def run():
                     sys.stdout.write('{0}: {1}'.format(node, line))
         sys.stdout.flush()
         if all:
-            rdy, _, _ = select.select(all, [], [], 10)
+            rdy = poller.poll(10)
     sys.exit(exitcode)
 
 
-def run_cmdv(node, cmdv, all, pipedesc):
+def run_cmdv(node, cmdv, all, poller, pipedesc):
     nopen = subprocess.Popen(
         cmdv, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
-                              'type': 'stdout'}
-    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
-                              'type': 'stderr'}
+    pipedesc[nopen.stdout.fileno()] = {'node': node, 'popen': nopen,
+                              'type': 'stdout', 'file': nopen.stdout}
+    pipedesc[nopen.stderr.fileno()] = {'node': node, 'popen': nopen,
+                              'type': 'stderr', 'file': nopen.stderr}
     all.add(nopen.stdout)
     all.add(nopen.stderr)
+    poller.register(nopen.stdout, select.EPOLLIN)
+    poller.register(nopen.stderr, select.EPOLLIN)
 
 
 if __name__ == '__main__':

confluent_client/bin/stats

@@ -19,6 +19,7 @@ import argparse
 import base64
 import csv
 import io
+import os
 import numpy as np
 import sys
 
@@ -72,6 +73,8 @@ def plot(gui, output, plotdata, bins, fmt):
             tdata = io.BytesIO()
         plt.savefig(tdata)
     if not gui and not output:
+        if fmt == 'environment':
+            fmt = os.environ.get('CONFLUENT_IMAGE_PROTOCOL', 'kitty')
         if fmt == 'sixel':
             writer = DumbWriter()
             writer.draw(tdata)
@@ -108,7 +111,7 @@ aparser = argparse.ArgumentParser(description='Quick access to common statistics
 aparser.add_argument('-c', type=int, default=0, help='Column number to analyze (default is last column)')
 aparser.add_argument('-d', default=None, help='Value used to separate columns')
 aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in graphical format')
-aparser.add_argument('-f', default='sixel', help='Format for histogram output (sixel/iterm/kitty)')
+aparser.add_argument('-f', default='environment', help='Format for histogram output (sixel/iterm/kitty)')
 aparser.add_argument('-s', default=0, help='Number of header lines to skip before processing')
 aparser.add_argument('-g', default=False, action='store_true', help='Open histogram in separate graphical window')
 aparser.add_argument('-o', default=None, help='Output histogram to the specified filename in PNG format')

confluent_client/confluent/screensqueeze.py

@@ -18,8 +18,9 @@ import struct
 import termios
 
 def get_screengeom():
-    return struct.unpack('hh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ,
-                                           b'....'))
+    # returns height in cells, width in cells, width in pixels, height in pixels
+    return struct.unpack('hhhh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ,
+                                           b'........'))
 class ScreenPrinter(object):
 
     def __init__(self, noderange, client, textlen=4):
@@ -58,7 +59,7 @@ class ScreenPrinter(object):
 
     def drawscreen(self, node=None):
         if self.squeeze:
-            currheight, currwidth = get_screengeom()
+            currheight, currwidth, _, _ = get_screengeom()
             currheight -= 2
             if currheight < 1:
                 currheight = 1
@@ -120,6 +121,7 @@ if __name__ == '__main__':
     c = client.Command()
     p = ScreenPrinter('d1-d12', c)
     p.set_output('d3', 'Upload: 67%')
+    p.set_output('d7', 'Upload: 67%')
 
 
 

confluent_client/confluent_client.spec.tmpl

@@ -1,7 +1,7 @@
 %define name confluent_client
 %define version #VERSION#
 %define fversion %{lua:
-sv, _ = string.gsub("#VERSION#", "[~+]", "-")
+sv, _ = string.gsub("#VERSION#", "[~]", "-")
 print(sv)
 }
 %define release 1

confluent_client/doc/man/confluent2hosts.ronn

@@ -13,7 +13,7 @@ noderange.  There are two general approaches.
 
 It can be used ad-hoc, using -i and -n to specify the address and name portions respectively.  This accepts the standard confluent expression syntax, allowing for things like 172.30.1.{n1} or {node}.{dns.domain} or {bmc}.
 
-It can also read from the confluent db, using `-a`.  In this mode, each net.<value>.<attribute> group is pulled together into hosts lines.  ipv4_address and ipv6_address fields are associated with the corresponding hostname attributes.
+It can also read from the confluent db, using `-a`.  In this mode, each net.<value>.<attribute> group is pulled together into hosts lines.  ipv4_address and ipv6_address fields are associated with the corresponding hostname attributes. You can use `-f` to put the FQDN first.
 
 ## EXAMPLES
 

confluent_client/doc/man/confluentdbutil.ronn

@@ -38,5 +38,8 @@ the json files (password protected, removed from the files, or unprotected).
   keys do not change and as such they do not require
   incremental backup.
   
+* `-y`, `--yaml
+  Use YAML instead of JSON as file format
+  
   * `-h`, `--help`:
   Show help message and exit

confluent_client/doc/man/nodeconsole.ronn

@@ -21,9 +21,15 @@ console process which will result in the console window closing.
 
 ## OPTIONS
 
+* `-i N`, `--interval`:
+  For screenshot mode, fetch new screenshots and overwrite old screenshots every N seconds.
+  For example, `nodeconsole r3u[21:24] -tsi 3` will tile screenshots of r3u21 through r3u24 and
+  refresh them every 3 seconds.
+
 * `-t`, `--tile`:
-  Use tmux to arrange consoles of the given noderange into a tiled layout on
-  the terminal screen
+  For text consoles, use tmux to arrange consoles of the given noderange into a tiled layout on
+  the terminal screen.  If using 'screenshot' mode, divide the terminal and display the images
+  in a grid. 
 
 * `-l`, `--log`:
   Perform a log reply on the current, local log in /var/log/confluent/consoles.
@@ -34,6 +40,14 @@ console process which will result in the console window closing.
  Dump the log with Timpstamps on the current, local log in /var/log/confluent/consoles.
   If in collective mode, this only makes sense to use on the current collective
   manager at this time.
+
+* `-s`, `--screenshot`:
+  Attempt to grab screenshot(s) and render using a terminal
+  image protocol. The image protocol defaults to kitty, and
+  can be selected by CONFLUENT_IMAGE_PROTOCOL environment variable.
+  Supported protocols are kitty, iterm, and, sixel (sixel only
+  if PySixel is installed).  This only presents screenshots, there
+  is no input supported to graphical consoles from a terminal.
   
 * `-w`, `--windowed`:
   Open terminal windows for each node.  The

confluent_client/doc/man/nodefirmware.ronn

@@ -3,7 +3,7 @@ nodefirmware(8) -- Report firmware information on confluent nodes
 
 ## SYNOPSIS
 
-`nodefirmware <noderange> [list][update [--backup <file>]]|[<components>]`
+`nodefirmware <noderange> [list][updatestatus][update [--backup <file>]]|[<components>]`
 
 ## DESCRIPTION
 
@@ -17,6 +17,9 @@ not be relevant to redfish.  Additionally, the Lenovo XCC makes certain
 information available over IPMI that is not otherwise available (for example
 the FPGA version where applicable).
 
+The updatestatus argument will describe the state of firmware updates on the
+nodes.
+
 In the update form, it accepts a single file and attempts to update it using
 the out of band facilities.  Firmware updates can end in one of three states:
 

confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient

@@ -14,6 +14,8 @@ import ssl
 import sys
 import struct
 import time
+import re
+import json
 
 class InvalidApiKey(Exception):
     pass
@@ -89,8 +91,11 @@ def scan_confluents():
                 confluentuuid = line.split(': ')[1]
                 msg += '/confluentuuid=' + confluentuuid
                 break
-    with open('/sys/devices/virtual/dmi/id/product_uuid') as uuidin:
-        msg += '/uuid=' + uuidin.read().strip()
+    try:
+        with open('/sys/devices/virtual/dmi/id/product_uuid') as uuidin:
+            msg += '/uuid=' + uuidin.read().strip()
+    except Exception:
+        pass
     for addrf in glob.glob('/sys/class/net/*/address'):
         with open(addrf) as addrin:
             hwaddr = addrin.read().strip()
@@ -228,6 +233,7 @@ def get_apikey(nodename, hosts, errout=None):
 
 class HTTPSClient(client.HTTPConnection, object):
     def __init__(self, usejson=False, port=443, host=None, errout=None, phmac=None, checkonly=False):
+        self.ignorehosts = set([])
         self.phmac = phmac
         self.errout = None
         if errout:
@@ -315,6 +321,8 @@ class HTTPSClient(client.HTTPConnection, object):
         ctx.check_hostname = True
         for timeo in (0.1, 5):
             for host in hosts:
+                if host in self.ignorehosts:
+                    continue
                 try:
                     addrinf = socket.getaddrinfo(host, self.port)[0]
                     psock = socket.socket(addrinf[0])
@@ -410,10 +418,100 @@ class HTTPSClient(client.HTTPConnection, object):
                     akfile.write('')
                 self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(
                     self.node, [self.host], errout=self.errout)
+            if rsp.status == 503:  # confluent is down, but the server running confluent is otherwise up
+                authed = False
+                self.ignorehosts.add(self.host)
+                host = self.check_connections()
+                client.HTTPConnection.__init__(self, host, self.port)
+
         raise Exception(rsp.read())
 
+def get_current_vmnic_vswitch():
+    uplinkmatch = re.compile(r'^\s*Uplinks:\s*(.*)')
+    switchmatch = re.compile(r'^\s*Name:\s*(.*)')
+
+    vswinfo = subprocess.check_output(['localcli', 'network', 'vswitch', 'standard', 'list']).decode()
+
+    vmnic = None
+    vswitch_name = None
+
+    for info in vswinfo.split('\n'):
+        name_match = switchmatch.match(info)
+        if name_match:
+            vswitch_name = name_match.group(1).strip()
+
+        upinfo = uplinkmatch.match(info)
+        if upinfo:
+            vmnic = upinfo.group(1).strip()
+            if vmnic and 'vusb0' not in vmnic:
+                return vswitch_name, vmnic
+
+    return vswitch_name, vmnic
+
+def get_available_nics():
+    nicinfo = subprocess.check_output(['localcli', 'network', 'nic', 'list']).decode('utf8').split('\n')
+    available_nics = {}
+
+    # Skip headers and separators
+    parsing_started = False
+    for line in nicinfo:
+        if re.match(r'^-+', line):
+            parsing_started = True
+            continue
+        if not parsing_started or not line.strip():
+            continue
+
+        parts = re.split(r'\s{2,}', line.strip())
+        if len(parts) >= 5:
+            nic_name = parts[0]
+            nic_status = parts[4]  # "Link Status" este al 5-lea câmp
+            available_nics[nic_name] = nic_status
+
+    return available_nics
+
+def is_esxi():
+    return os.path.isdir("/etc/vmware")
+
+def fix_vswitch():
+    if is_esxi():
+        start_time = time.time()
+        while True:
+
+            current_vswitch, current_vmnic = get_current_vmnic_vswitch()
+            if current_vswitch is None:
+                raise RuntimeError("Panic: current vswitch is None")
+            if current_vmnic is None:
+                raise RuntimeError("Panic: current vmnic is None")
+
+            available_nics = get_available_nics()
+
+            if current_vmnic and available_nics.get(current_vmnic) == 'Up':
+                break
+
+            new_vmnic = next((nic for nic, status in available_nics.items() if status == 'Up'), None)
+
+            if new_vmnic and new_vmnic != current_vmnic:
+                subprocess.check_call(['localcli', 'network', 'vswitch', 'standard', 'uplink', 'remove',
+                                       '--uplink-name', current_vmnic, '--vswitch-name', current_vswitch])
+                subprocess.check_call(['localcli', 'network', 'vswitch', 'standard', 'uplink', 'add',
+                                       '--uplink-name', new_vmnic, '--vswitch-name', current_vswitch])
+            elif not new_vmnic:
+                if time.time() - start_time > 300:
+                    break
+            time.sleep(5)
+
+        time.sleep(5)
+
 if __name__ == '__main__':
     data = None
+
+    if '-f' in sys.argv:
+        try:
+            fix_vswitch()
+        except Exception as e:
+            print(f"fix_vswitch() error: {e}")
+        sys.argv.remove('-f')
+        sys.exit(0)
     usejson = False
     if '-j' in sys.argv:
         usejson = True
@@ -468,13 +566,14 @@ if __name__ == '__main__':
                 outf.write(chunk)
                 chunk = reader.read(16384)
         sys.exit(0)
-    client = HTTPSClient(usejson, errout=errout, phmac=phmac, checkonly=checkonly)
+
+    mclient = HTTPSClient(usejson, errout=errout, phmac=phmac, checkonly=checkonly)
     if waitfor:
         status = 201
         while status != waitfor:
-            status, rsp = client.grab_url_with_status(sys.argv[1], data)
+            status, rsp = mclient.grab_url_with_status(sys.argv[1], data)
             sys.stdout.write(rsp.decode())
     elif checkonly:
-        sys.stdout.write(client.check_connections())
+        sys.stdout.write(mclient.check_connections())
     else:
-        sys.stdout.write(client.grab_url(sys.argv[1], data).decode())
+        sys.stdout.write(mclient.grab_url(sys.argv[1], data).decode())

confluent_osdeploy/common/profile/scripts/autoconsole

@@ -0,0 +1,119 @@
+#!/usr/bin/python3
+
+# This script evaluates whether firmware redirection is likely. It uses three cues:
+# - Does the system offer up SPCR? This would indicate that the firmware is doing serial output.
+#   Otherwise, there's no indication that the firmware cares about serial console.
+# - Is the system EFI?  BIOS implementations may not intercept text draw calls after POST exit,
+#   thus even when BIOS tells us serial port is in use, it may not be doing anything when
+#   grub would be running
+# - Is the serial port connected? In the event that firmware indicates serial port, but
+#   serial port is not reporting DCD, then it doesn't look like a comfortable enough scenario
+
+import fcntl
+import os
+import os.path
+import struct
+import subprocess
+import termios
+
+
+addrtoname = {
+    0x3f8: '/dev/ttyS0',
+    0x2f8: '/dev/ttyS1',
+    0x3e8: '/dev/ttyS2',
+    0x2e8: '/dev/ttyS3',
+}
+speedmap = {
+    0: None,
+    3: 9600,
+    4: 19200,
+    6: 57600,
+    7: 115200,
+}
+
+termiobaud = {
+    9600: termios.B9600,
+    19200: termios.B19200,
+    57600: termios.B57600,
+    115200: termios.B115200,
+}
+
+
+def deserialize_grub_rh():
+    if 'console=ttyS' in open('/proc/cmdline').read():
+        return None # User manually indicated serial config
+        # they own the grub behavior too for now
+    grublines = []
+    with open('/etc/default/grub') as grubin:
+        grublines = grubin.read().split('\n')
+    with open('/etc/default/grub', 'w') as grubout:
+        for grubline in grublines:
+            if grubline.startswith('GRUB_TERMINAL'):
+                grubline = grubline.replace('serial ', '')
+            grubout.write(grubline + '\n')
+    subprocess.check_call(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])
+
+def fixup_ubuntu_grub_serial():
+    # Ubuntu aggressively tries to graphics up
+    # grub. We will counter that for serial
+    # They also aggressively hide UI and
+    # block ability to interject. We will
+    # compromise and lean on nodeboot <node> setup
+    # as a means to give someone reasonable shot at
+    # the short timeout
+    with open('/etc/default/grub') as grubin:
+        grublines = grubin.read().split('\n')
+    with open('/etc/default/grub', 'w') as grubout:
+        for grubline in grublines:
+            if grubline.startswith('GRUB_TIMEOUT_STYLE=hidden'):
+                grubline = 'GRUB_TIMEOUT_STYLE=menu'
+            elif grubline.startswith('GRUB_TIMEOUT=0'):
+                grubline = 'GRUB_TIMEOUT=2'
+            elif grubline.startswith('#GRUB_TERMINAL=console'):
+                grubline = grubline.replace('#', '')
+            grubout.write(grubline + '\n')
+    subprocess.check_call(['update-grub'])
+
+def get_serial_config():
+    if not os.path.exists('/sys/firmware/efi'):
+        return None
+    if not os.path.exists('/sys/firmware/acpi/tables/SPCR'):
+        return None
+    spcr = open("/sys/firmware/acpi/tables/SPCR", "rb")
+    spcr = bytearray(spcr.read())
+    if spcr[8] != 2 or spcr[36] != 0 or spcr[40] != 1:
+        return None
+    address = struct.unpack('<Q', spcr[44:52])[0]
+    tty = None
+    try:
+        tty = addrtoname[address]
+    except KeyError:
+        return None
+    retval = { 'tty': tty }
+    try:
+        retval['speed'] = speedmap[spcr[58]]
+    except KeyError:
+        return None
+    if retval['speed']:
+        ttyf = os.open(tty, os.O_RDWR | os.O_NOCTTY)
+        currattr = termios.tcgetattr(ttyf)
+        currattr[4:6] = [0, termiobaud[retval['speed']]]
+        termios.tcsetattr(ttyf, termios.TCSANOW, currattr)
+    retval['connected'] = bool(struct.unpack('<I', fcntl.ioctl(
+        ttyf, termios.TIOCMGET, '\x00\x00\x00\x00'))[0] & termios.TIOCM_CAR)
+    os.close(ttyf)
+    return retval
+
+def main():
+    autoconscfg = get_serial_config()
+    if not autoconscfg or not autoconscfg['connected']:
+        return
+    if os.path.exists('/etc/redhat-release'):  # redhat family
+        deserialize_grub_rh()
+    elif os.path.exists('/etc/os-release'):
+        with open('/etc/os-release') as osr:
+            if 'Ubuntu' in osr.read():
+                fixup_ubuntu_grub_serial()
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/common/profile/scripts/confignet

@@ -367,7 +367,7 @@ class NetworkManager(object):
                 args.append(bondcfg[parm])
             subprocess.check_call(['nmcli', 'c', 'm', team] + args)
 
-    def apply_configuration(self, cfg):
+    def apply_configuration(self, cfg, lastchance=False):
         cmdargs = {}
         cmdargs['connection.autoconnect'] = 'yes'
         stgs = cfg['settings']
@@ -409,7 +409,7 @@ class NetworkManager(object):
                 cargs.append(cmdargs[arg])
             if stgs['team_mode'] in self.bondtypes:
                 stgs['team_mode'] = self.bondtypes[stgs['team_mode']]
-            subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond', 'con-name', cname, 'connection.interface-name', cname, 'bond.options', 'mode={}'.format(stgs['team_mode'])] + cargs)
+            subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond', 'con-name', cname, 'connection.interface-name', cname, 'bond.options', 'miimon=100,mode={}'.format(stgs['team_mode'])] + cargs)
             for iface in cfg['interfaces']:
                 self.add_team_member(cname, iface)
             subprocess.check_call(['nmcli', 'c', 'u', cname])
@@ -418,8 +418,9 @@ class NetworkManager(object):
             iname = list(cfg['interfaces'])[0]
             ctype = self.devtypes.get(iname, None)
             if not ctype:
-                sys.stderr.write("Warning, no device found for interface_name ({0}), skipping setup\n".format(iname))
-                return
+                if lastchance:
+                    sys.stderr.write("Warning, no device found for interface_name ({0}), skipping setup\n".format(iname))
+                return 1
             if stgs.get('vlan_id', None):
                 vlan = stgs['vlan_id']
                 if ctype == 'infiniband':
@@ -543,8 +544,21 @@ if __name__ == '__main__':
         nm = NetworkManager(devtypes, dc)
     elif os.path.exists('/usr/sbin/wicked'):
         nm = WickedManager()
+    retrynics = []
     for netn in netname_to_interfaces:
-        nm.apply_configuration(netname_to_interfaces[netn])
+        redo = nm.apply_configuration(netname_to_interfaces[netn])
+        if redo == 1:
+            retrynics.append(netn)
+    if retrynics:
+        idxmap, devtypes = map_idx_to_name()
+        if os.path.exists('/usr/sbin/netplan'):
+            nm = NetplanManager(dc)
+        if os.path.exists('/usr/bin/nmcli'):
+            nm = NetworkManager(devtypes, dc)
+        elif os.path.exists('/usr/sbin/wicked'):
+            nm = WickedManager()
+        for netn in retrynics:
+            nm.apply_configuration(netname_to_interfaces[netn], lastchance=True)
     if havefirewall:
         subprocess.check_call(['systemctl', 'start', 'firewalld'])
     await_tentative()

confluent_osdeploy/common/profile/scripts/sample/consoleredirect

@@ -1,49 +0,0 @@
-is_suse=false
-is_rhel=false
-
-if test -f /boot/efi/EFI/redhat/grub.cfg; then
-    grubcfg="/boot/efi/EFI/redhat/grub.cfg"
-    grub2-mkconfig -o $grubcfg
-    is_rhel=true
-elif test -f /boot/efi/EFI/sle_hpc/grub.cfg; then
-    grubcfg="/boot/efi/EFI/sle_hpc/grub.cfg"
-    grub2-mkconfig -o $grubcfg
-    is_suse=true
-else
-    echo "Expected File missing: Check if os sle_hpc or redhat"
-    exit
-fi
-
-# working on SUSE
-if $is_suse; then
-    start=false
-    num_line=0
-    lines_to_edit=()
-    while read line; do
-        ((num_line++))
-        if [[ $line == *"grub_platform"* ]]; then
-            start=true
-        fi
-        if $start; then
-            if [[ $line != "#"* ]];then
-                lines_to_edit+=($num_line)
-            fi
-        fi
-        if [[ ${#line} -eq 2 && $line == *"fi" ]]; then
-            if $start; then
-                start=false
-            fi
-        fi
-    done < grub_cnf.cfg
-
-    for line_num in "${lines_to_edit[@]}"; do
-        line_num+="s"
-        sed -i "${line_num},^,#," $grubcfg
-    done
-    sed -i 's,^terminal,#terminal,' $grubcfg
-fi
-
-# Working on Redhat
-if $is_rhel; then
-    sed -i 's,^serial,#serial, ; s,^terminal,#terminal,' $grubcfg
-fi

confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl

@@ -26,11 +26,14 @@ mkdir -p opt/confluent/bin
 mkdir -p stateless-bin
 cp -a el8bin/* .
 ln -s el8 el9
-for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9; do
+ln -s el8 el10
+for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9 el10; do
     mkdir ${os}out
     cd ${os}out
     if [ -d ../${os}bin ]; then 
         cp -a ../${os}bin/opt .
+    elif [ $os = el10 ]; then
+        cp -a ../el9bin/opt .
     else
         cp -a ../opt .
     fi
@@ -40,11 +43,13 @@ for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreo
     mv ../addons.cpio .
     cd ..
 done
-for os in el7 el8 suse15 el9 ubuntu20.04; do
+for os in el7 el8 suse15 el9 el10 ubuntu20.04; do
     mkdir ${os}disklessout
     cd ${os}disklessout
     if [ -d ../${os}bin ]; then
         cp -a ../${os}bin/opt .
+    elif [ $os = el10 ]; then
+        cp -a ../el9bin/opt .
     else
         cp -a ../opt .
     fi
@@ -76,7 +81,7 @@ cp -a esxi7 esxi8
 %install
 mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
 #cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
-for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 esxi6 esxi7 esxi8 coreos; do
+for os in rhvh4 el7 el8 el9 el10 genesis suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 esxi6 esxi7 esxi8 coreos; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs/aarch64/
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs/aarch64/
     if [ -d ${os}disklessout ]; then

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -30,7 +30,7 @@ This contains support utilities for enabling deployment of x86_64 architecture s
 ln -s el8 el9
 cp -a el8 el10
 mv el10/initramfs/usr el10/initramfs/var
-for os in rhvh4 el7 genesis el8 suse15 ubuntu18.04 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9 el10; do
+for os in rhvh4 el7 genesis el8 suse15 debian ubuntu18.04 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9 el10; do
     mkdir ${os}out
     cd ${os}out
     if [ -d ../${os}bin ]; then 
@@ -46,11 +46,13 @@ for os in rhvh4 el7 genesis el8 suse15 ubuntu18.04 ubuntu20.04 ubuntu22.04 ubunt
     mv ../addons.cpio .
     cd ..
 done
-for os in el7 el8 suse15 el9 ubuntu20.04 ubuntu22.04 ubuntu24.04; do
+for os in el7 el8 suse15 el9 el10 ubuntu20.04 ubuntu22.04 ubuntu24.04; do
     mkdir ${os}disklessout
     cd ${os}disklessout
     if [ -d ../${os}bin ]; then
         cp -a ../${os}bin/opt .
+    elif [ $os = el10 ]; then
+        cp -a ../el9bin/opt .
     else
         cp -a ../el8bin/opt .
     fi
@@ -77,12 +79,14 @@ cd ..
 cp -a esxi7out esxi6out
 cp -a esxi7 esxi6
 cp -a esxi7out esxi8out
+cp -a esxi7out esxi9out
 cp -a esxi7 esxi8
+cp -a esxi7 esxi9
 
 %install
 mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
 cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
-for os in rhvh4 el7 el8 el9 el10 genesis suse15 ubuntu20.04 ubuntu18.04 ubuntu22.04 ubuntu24.04 esxi6 esxi7 esxi8 coreos; do
+for os in rhvh4 el7 el8 el9 el10 genesis suse15 ubuntu20.04 debian ubuntu18.04 ubuntu22.04 ubuntu24.04 esxi6 esxi7 esxi8 esxi9 coreos; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs

confluent_osdeploy/debian/initramfs/lib/debian-installer-startup.d/S25confluentinit

@@ -14,65 +14,127 @@ setdebopt() {
     echo d-i $1 $3 $2 >> /preseed.cfg
 }
 
-dhuuid=$(reverse_uuid $(cat /sys/devices/virtual/dmi/id/product_uuid))
-dhcpid=$(mktemp)
 mkdir -p /etc/confluent
-cp /tls/* /etc/ssl/certs/
-for nic in $(ip link | grep mtu|grep -v LOOPBACK|cut -d: -f 2|sed -e 's/ //'); do
-     ip link set $nic up
+for i in /sys/class/net/*; do
+    ip link set $(basename $i) up
 done
-for nic in $(ip link | grep mtu|grep -v LOOPBACK|grep LOWER_UP|cut -d: -f 2|sed -e 's/ //'); do
-    if udhcpc -i $nic -p $dhcpid -t 2 -T 2 -n -x 93:0007 -x 97:00$dhuuid -q; then
-         /opt/confluent/bin/copernicus > /etc/confluent/confluent.info
-         if grep ^MANAGER:.*\\. /etc/confluent/confluent.info ; then
-             break
-         fi
+TRIES=5
+while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
+done
+for i in /sys/class/net/*; do
+    ip link set $(basename $i) down
+    udevadm info $i | grep ID_NET_DRIVER=cdc_ether > /dev/null &&  continue
+    ip link set $(basename $i) up
+done
+cp -a /tls/* /etc/ssl/certs/
+mkdir -p /etc/confluent
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=$(mktemp -d)
+    tcfg=$(mktemp)
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    nodename=$(grep ^nodename: cnflnt.yml|cut -f 2 -d ' ')
+    echo NODENAME: $nodename > /etc/confluent/confluent.info
+    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    if [ "$autoconfigmethod" = "static" ]; then
+        setdebopt netcfg/disable_dhcp true boolean
+        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        v4gw=$(grep ^ipv4_gateway: $tcfg|cut -d: -f 2| sed -e 's/ //')
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ^ipv4_netmask: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        setdebopt netcfg/get_netmask $v4nm string
+        setdebopt netcfg/get_ipaddress ${v4addr%/*} string
+        setdebopt netcfg/confirm_static true boolean
+        if [ ! -z "$v4gw" ]; then
+            setdebopt netcfg/get_gateway $v4gw string
+        fi
+        NIC=""
+        while [ -z "$NIC" ]; do
+            for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+                ip addr add dev $NICGUESS $v4addr
+                if [ ! -z "$v4gw" ]; then
+                    ip route add default via $v4gw
+                fi
+                for dsrv in $deploysrvs; do
+                    if wget https://$dsrv/confluent-public/ --tries=1 --timeout=1 -O /dev/null > /dev/null 2>&1; then
+                        deploysrvs=$dsrv
+                        NIC=$NICGUESS
+                        setdebopt netcfg/choose_interface $NIC select
+                        break
+                    fi
+                done
+                if [ -z "$NIC" ]; then
+                    ip -4 a flush dev $NICGUESS
+                else
+                    break
+                fi
+            done
+        done
+        #TODO: nameservers
+    elif [ "$v4cfgmeth" = "dhcp" ]; then
+        setdebopt netcfg/disable_dhcp false boolean
+        setdebopt netcfg/confirm_static false boolean
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            udhcpc $NICGUESS
+        done
+        for dsrv in $deploysrvs; do
+            if wget https://$dsrv/confluent-public/ --tries=1 --timeout=1 -O /dev/null > /dev/null 2>&1; then
+                deploysrvs=$dsrv
+            fi
+        done
     fi
-    ip -4 flush dev $nic
-done    
-mgr=$(grep ^MANAGER:.*\\. /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
-nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
-/opt/confluent/bin/clortho $nodename $mgr > /etc/confluent/confluent.apikey
+    mgr=$deploysrvs
+    ln -s /opt/confluent/bin/clortho /opt/confluent/bin/genpasshmac
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    passfile=/tmp/cnflnttmppassfile
+    passcrypt=/tmp/cnflntcryptfile
+    hmacfile=/tmp/cnflnthmacfile
+    echo -n $(grep ^apitoken: cnflnt.yml|cut -d ' ' -f 2) > $hmackeyfile
+    /opt/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" --post-file=$passcrypt https://$mgr/confluent-api/self/registerapikey -O - --quiet
+    cp $passfile /etc/confluent/confluent.apikey
+    nic=$NIC
+else
+    dhuuid=$(reverse_uuid $(cat /sys/devices/virtual/dmi/id/product_uuid))
+    dhcpid=$(mktemp)
+    mkdir -p /etc/confluent
+    cp /tls/* /etc/ssl/certs/
+    cat /tls/*.pem >> /etc/confluent/ca.pem
+    for nic in $(ip link | grep mtu|grep -v LOOPBACK|cut -d: -f 2|sed -e 's/ //'); do
+        ip link set $nic up
+    done
+    for nic in $(ip link | grep mtu|grep -v LOOPBACK|grep LOWER_UP|cut -d: -f 2|sed -e 's/ //'); do
+        if udhcpc -i $nic -p $dhcpid -t 2 -T 2 -n -x 93:0007 -x 97:00$dhuuid -q; then
+            /opt/confluent/bin/copernicus > /etc/confluent/confluent.info
+            if grep ^MANAGER:.*\\. /etc/confluent/confluent.info ; then
+                break
+            fi
+        fi
+        ip -4 flush dev $nic
+    done
+    mgr=$(grep ^MANAGER:.*\\. /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
+    nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
+    /opt/confluent/bin/clortho $nodename $mgr > /etc/confluent/confluent.apikey
+fi
 apikey=$(cat /etc/confluent/confluent.apikey)
 cd /etc/confluent
 wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/deploycfg
 cd -
 predir=$(mktemp -d)
 cd $predir
+cp /etc/confluent/deploycfg /etc/confluent/confluent.deploycfg
 profile=$(grep ^profile: /etc/confluent/deploycfg|cut -d ' ' -f 2)
-wget https://$mgr/confluent-public/os/$profile/scripts/pre.sh
-chmod u+x pre.sh
-wget https://$mgr/confluent-public/os/$profile/preseed.cfg
-mv preseed.cfg /
-setdebopt auto-install/enable true boolean
-setdebopt partman/early_command  $predir/pre.sh string
-cd -
-ip -4 a flush dev $nic
-setdebopt netcfg/choose_interface $nic select
-setdebopt netcfg/get_hostname $nodename string
-v4cfgmeth=$(grep ipv4_method: /etc/confluent/deploycfg |cut -d: -f 2|sed -e 's/ //')
-if [ "$v4cfgmeth" = "static" ]; then
-    setdebopt netcfg/disable_dhcp true boolean
-    v4addr=$(grep ^ipv4_address: /etc/confluent/deploycfg|cut -d: -f 2|sed -e 's/ //')
-    v4gw=$(grep ^ipv4_gateway: /etc/confluent/deploycfg|cut -d: -f 2| sed -e 's/ //')
-    if [ "$v4gw" = "null" ]; then
-        v4gw=""
-    fi
-    v4nm=$(grep ^ipv4_netmask: /etc/confluent/deploycfg|cut -d: -f 2|sed -e 's/ //')
-    setdebopt netcfg/get_netmask $v4nm string
-    setdebopt netcfg/get_ipaddress $v4addr string
-    setdebopt netcfg/confirm_static true boolean
-    if [ ! -z "$v4gw" ]; then
-        setdebopt netcfg/get_gateway $v4gw string
-    fi
-    namesrvs=$(sed -n '/^nameservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
-    for namesrv in "$namesrvs"; do
-        setdebopt netcfg/get_nameservers $namesrv string
-    done
-elif [ "$v4cfgmeth" = "dhcp" ]; then
-    setdebopt netcfg/disable_dhcp false boolean
-    setdebopt netcfg/confirm_static false boolean
-fi
+namesrvs=$(sed -n '/^nameservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
+for namesrv in "$namesrvs"; do
+    setdebopt netcfg/get_nameservers $namesrv string
+done
 rootpass=$(grep ^rootpassword: /etc/confluent/deploycfg|cut -d ' ' -f 2|sed -e 's/ //')
 if [ "$rootpass" = null ] || [ -z "$rootpass" ]; then
     setdebopt passwd/root-login false boolean
@@ -84,9 +146,8 @@ setdebopt time/zone $(grep ^timezone: /etc/confluent/deploycfg|cut -d ' ' -f 2|s
 ntpsrvs=$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
 for ntpsrv in "$ntpsrvs"; do
     setdebopt clock-setup/ntp true boolean
-    setdebopt clock-setup/ntep-server $ntpsrv string
+    setdebopt clock-setup/ntp-server $ntpsrv string
 done
-#setdebopt console-setup/layoutcode $(grep ^keymap: /etc/confluent/deploycfg|cut -d ' ' -f 2) string
 setdebopt debian-installer/locale $(grep ^locale: /etc/confluent/deploycfg|cut -d ' ' -f 2) select
 domainname=$(grep ^dnsdomain: /etc/confluent/deploycfg|cut -d ' ' -f 2)
 if [ ! -z "$domainname" ] && [ "$domainname" != "null" ]; then
@@ -95,3 +156,54 @@ fi
 
 
 
+
+wget https://$mgr/confluent-public/os/$profile/scripts/pre.sh
+chmod u+x pre.sh
+wget https://$mgr/confluent-public/os/$profile/scripts/prechroot.sh
+chmod u+x prechroot.sh
+wget https://$mgr/confluent-public/os/$profile/scripts/post.sh
+chmod u+x post.sh
+wget https://$mgr/confluent-public/os/$profile/preseed.cfg
+cat preseed.cfg >> /preseed.cfg
+echo $mgr > /etc/confluent/deployer
+setdebopt auto-install/enable true boolean
+setdebopt partman/early_command  $predir/pre.sh string
+setdebopt preseed/late_command $predir/prechroot.sh string
+mv $predir/post.sh /tmp/
+cd -
+ip -4 a flush dev $nic
+setdebopt netcfg/choose_interface $nic select
+setdebopt netcfg/get_hostname $nodename string
+setdebopt netcfg/hostname $nodename string
+setdebopt mirror/protocol https string
+setdebopt mirror/country manual string
+setdebopt mirror/https/hostname deb.debian.org string
+setdebopt mirror/https/directory /debian/ string
+setdebopt mirror/protocol https string
+setdebopt mirror/https/proxy "" string
+#setdebopt apt-setup/security_host $mgr string
+if [ ! -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    v4cfgmeth=$(grep ipv4_method: /etc/confluent/deploycfg |cut -d: -f 2|sed -e 's/ //')
+    if [ "$v4cfgmeth" = "static" ]; then
+        setdebopt netcfg/disable_dhcp true boolean
+        v4addr=$(grep ^ipv4_address: /etc/confluent/deploycfg|cut -d: -f 2|sed -e 's/ //')
+        v4gw=$(grep ^ipv4_gateway: /etc/confluent/deploycfg|cut -d: -f 2| sed -e 's/ //')
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ^ipv4_netmask: /etc/confluent/deploycfg|cut -d: -f 2|sed -e 's/ //')
+        setdebopt netcfg/get_netmask $v4nm string
+        setdebopt netcfg/get_ipaddress $v4addr string
+        setdebopt netcfg/confirm_static true boolean
+        if [ ! -z "$v4gw" ]; then
+            setdebopt netcfg/get_gateway $v4gw string
+        fi
+        namesrvs=$(sed -n '/^nameservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
+        for namesrv in "$namesrvs"; do
+            setdebopt netcfg/get_nameservers $namesrv string
+        done
+    elif [ "$vpcfgmeth" = "dhcp" ]; then
+        setdebopt netcfg/disable_dhcp false boolean
+        setdebopt netcfg/confirm_static false boolean
+    fi
+fi

confluent_osdeploy/debian/profiles/default/initprofile.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+sed -i 's/label: debian/label: Debian/' $2/profile.yaml && \
+ln -s $1/linux $2/boot/kernel && \
+ln -s $1/initrd.gz $2/boot/initramfs/distribution && \
+mkdir -p $2/boot/efi/boot && \
+rm $2/distribution && \
+mcopy -i $1/boot/grub/efi.img ::/efi/boot/* $2/boot/efi/boot
+

confluent_osdeploy/debian/profiles/default/preseed.cfg

@@ -0,0 +1,27 @@
+d-i anna/choose_modules string openssh-server-udeb
+d-i partman-auto/method string regular
+d-i partman-lvm/device_remove_lvm boolean true
+d-i partman-md/device_remove_md boolean true
+d-i partman-auto/expert_recipe_file string /tmp/partitionfile
+d-i partman/confirm_write_new_label boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm boolean true
+d-i partman/confirm_nooverwrite boolean true
+d-i passwd/make-user boolean false
+d-i clock-setup/utc boolean true
+d-i apt-setup/multiverse boolean false
+d-i apt-setup/universe boolean false
+d-i apt-setup/backports boolean false
+d-i apt-setup/updates boolean false
+d-i grub-installer/only_debian boolean true
+tasksel tasksel/first multiselect standard
+d-i pkgsel/include string openssh-server curl
+d-i pkgsel/update-policy select none
+d-i pkgsel/updatedb boolean false
+d-i finish-install/reboot_in_progress note
+popularity-contest popularity-contest/participate boolean false
+d-i partman-auto/method string lvm
+d-i partman-auto/choose_recipe select atomic
+d-i partman-lvm/confirm boolean true
+d-i partman-lvm/confirm_nooverwrite boolean true
+d-i partman-auto-lvm/guided_size string max

confluent_osdeploy/debian/profiles/default/profile.yaml

@@ -0,0 +1,3 @@
+label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet osprofile=%%PROFILE%%
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/debian/profiles/default/scripts/confignet

@@ -0,0 +1,565 @@
+#!/usr/bin/python
+
+import glob
+import json
+import os
+import socket
+import sys
+import time
+import shlex
+import subprocess
+try:
+    import yaml
+except ImportError:
+    pass
+try:
+    from importlib.machinery import SourceFileLoader
+    def load_source(mod, path):
+        return SourceFileLoader(mod, path).load_module()
+except ImportError:
+    from imp import load_source
+
+try:
+    apiclient = load_source('apiclient', '/opt/confluent/bin/apiclient')
+except IOError:
+    apiclient = load_source('apiclient', '/etc/confluent/apiclient')
+
+def add_lla(iface, mac):
+    pieces = mac.split(':')
+    initbyte = int(pieces[0], 16) ^ 2
+    lla = 'fe80::{0:x}{1}:{2}ff:fe{3}:{4}{5}/64'.format(initbyte, pieces[1], pieces[2], pieces[3], pieces[4], pieces[5])
+    try:
+        with open('/proc/sys/net/ipv6/conf/{0}/disable_ipv6'.format(iface), 'w') as setin:
+            setin.write('0')
+        subprocess.check_call(['ip', 'addr', 'add', 'dev', iface, lla, 'scope', 'link'])
+    except Exception:
+        return None
+    return lla
+
+#cli = apiclient.HTTPSClient(json=True)
+#c = cli.grab_url_with_status('/confluent-api/self/netcfg')
+def add_missing_llas():
+    #NetworkManager goes out of its way to suppress ipv6 lla, so will just add some
+    added = {}
+    linkinfo = subprocess.check_output(['ip', '-br', 'l']).decode('utf8')
+    ifaces = {}
+    for line in linkinfo.split('\n'):
+        line = line.strip().split()
+        if not line or 'LOOPBACK' in line[-1] or 'NO-CARRIER' in line[-1]:
+            continue
+        if 'UP' not in line[-1]:
+            subprocess.call(['ip', 'link', 'set', line[0], 'up'])
+        ifaces[line[0]] = line[2]
+    ips = {}
+    ipinfo = subprocess.check_output(['ip', '-br', '-6', 'a']).decode('utf8')
+    for line in ipinfo.split('\n'):
+        line = line.strip().split(None, 2)
+        if not line:
+            continue
+        ips[line[0]] = line[2]
+    for iface in ifaces:
+        for addr in ips.get(iface, '').split():
+            if addr.startswith('fe80::'):
+                break
+        else:
+            newlla = add_lla(iface, ifaces[iface])
+            if newlla:
+                added[iface] = newlla
+    return added
+
+def rm_tmp_llas(tmpllas):
+    for iface in tmpllas:
+        subprocess.check_call(['ip', 'addr', 'del', 'dev', iface, tmpllas[iface]])
+
+def await_tentative():
+    maxwait = 10
+    while b'tentative' in subprocess.check_output(['ip', 'a']):
+        if maxwait == 0:
+            break
+        maxwait -= 1
+        time.sleep(1)
+
+def map_idx_to_name():
+    map = {}
+    devtype = {}
+    prevdev = None
+    for line in subprocess.check_output(['ip', 'l']).decode('utf8').splitlines():
+        if line.startswith(' ') and 'link/' in line:
+            typ = line.split()[0].split('/')[1]
+            devtype[prevdev] = typ if typ != 'ether' else 'ethernet'
+        if line.startswith(' '):
+            continue
+        idx, iface, rst = line.split(':', 2)
+        prevdev = iface.strip()
+        rst = rst.split()
+        try:
+            midx = rst.index('master')
+            continue
+        except ValueError:
+            pass
+        idx = int(idx)
+        iface = iface.strip()
+        map[idx] = iface
+    return map, devtype
+
+
+def get_interface_name(iname, settings):
+    explicitname = settings.get('interface_names', None)
+    if explicitname:
+        return explicitname
+    if settings.get('current_nic', False):
+        return iname
+    return None
+
+class NetplanManager(object):
+    def __init__(self, deploycfg):
+        self.cfgbydev = {}
+        self.read_connections()
+        self.deploycfg = deploycfg
+
+    def read_connections(self):
+        for plan in glob.glob('/etc/netplan/*.y*ml'):
+            with open(plan) as planfile:
+                planinfo = yaml.safe_load(planfile)
+                if not planinfo:
+                    continue
+            nicinfo = planinfo.get('network', {}).get('ethernets', {})
+            for devname in nicinfo:
+                if devname == 'lo':
+                    continue
+                if 'gateway4' in nicinfo[devname]:
+                    # normalize deprecated syntax on read in
+                    gw4 = nicinfo[devname]['gateway4']
+                    del nicinfo[devname]['gateway4']
+                    routeinfo = nicinfo[devname].get('routes', [])
+                    for ri in routeinfo:
+                        if ri.get('via', None) == gw4 and ri.get('to', None) in ('default', '0.0.0.0/0', '0/0'):
+                            break
+                    else:
+                        routeinfo.append({
+                            'to': 'default',
+                            'via': gw4
+                        })
+                    nicinfo[devname]['routes'] = routeinfo
+                self.cfgbydev[devname] = nicinfo[devname]
+    
+    def apply_configuration(self, cfg):
+        devnames = cfg['interfaces']
+        if len(devnames) != 1:
+            raise Exception('Multi-nic team/bonds not yet supported')
+        stgs = cfg['settings']
+        needcfgapply = False
+        for devname in devnames:
+            needcfgwrite = False
+            # ipv6_method missing at uconn...
+            if stgs.get('ipv6_method', None) == 'static':
+                curraddr = stgs['ipv6_address']
+                currips = self.getcfgarrpath([devname, 'addresses'])
+                if curraddr not in currips:
+                    needcfgwrite = True
+                    currips.append(curraddr)
+            if stgs.get('ipv4_method', None) == 'static':
+                curraddr = stgs['ipv4_address']
+                currips = self.getcfgarrpath([devname, 'addresses'])
+                if curraddr not in currips:
+                    needcfgwrite = True
+                    currips.append(curraddr)
+            gws = []
+            gws.append(stgs.get('ipv4_gateway', None))
+            gws.append(stgs.get('ipv6_gateway', None))
+            for gwaddr in gws:
+                if gwaddr:
+                    cfgroutes = self.getcfgarrpath([devname, 'routes'])
+                    for rinfo in cfgroutes:
+                        if rinfo.get('via', None) == gwaddr:
+                            break
+                    else:
+                        needcfgwrite = True
+                        cfgroutes.append({'via': gwaddr, 'to': 'default'})
+            dnsips = self.deploycfg.get('nameservers', [])
+            dnsdomain = self.deploycfg.get('dnsdomain', '')
+            if dnsips:
+                currdnsips = self.getcfgarrpath([devname, 'nameservers', 'addresses'])
+                for dnsip in dnsips:
+                    if dnsip and dnsip not in currdnsips:
+                        needcfgwrite = True
+                        currdnsips.append(dnsip)
+            if dnsdomain:
+                currdnsdomain = self.getcfgarrpath([devname, 'nameservers', 'search'])
+                if dnsdomain not in currdnsdomain:
+                    needcfgwrite = True
+                    currdnsdomain.append(dnsdomain)
+            if needcfgwrite:
+                needcfgapply = True
+                newcfg = {'network': {'version': 2, 'ethernets': {devname: self.cfgbydev[devname]}}}
+                oumask = os.umask(0o77)
+                with open('/etc/netplan/{0}-confluentcfg.yaml'.format(devname), 'w') as planout:
+                    planout.write(yaml.dump(newcfg))
+                os.umask(oumask)
+        if needcfgapply:
+            subprocess.call(['netplan', 'apply'])
+
+    def getcfgarrpath(self, devpath):
+        currptr = self.cfgbydev
+        for k in devpath[:-1]:
+            if k not in currptr:
+                currptr[k] = {}
+            currptr = currptr[k]
+        if devpath[-1] not in currptr:
+            currptr[devpath[-1]] = []
+        return currptr[devpath[-1]]
+        
+
+
+class WickedManager(object):
+    def __init__(self):
+        self.teamidx = 0
+        self.read_connections()
+
+    def read_connections(self):
+        self.cfgbydev = {}
+        for ifcfg in glob.glob('/etc/sysconfig/network/ifcfg-*'):
+            devname = ifcfg.replace('/etc/sysconfig/network/ifcfg-', '')
+            if devname == 'lo':
+                continue
+            currcfg = {}
+            self.cfgbydev[devname] = currcfg
+            for cfg in open(ifcfg).read().splitlines():
+                cfg = cfg.split('#', 1)[0]
+                try:
+                    kv = ' '.join(shlex.split(cfg)).split('=', 1)
+                except Exception:
+                    # unparseable line, likely having something we can't handle
+                    del self.cfgbydev[devname]
+                if len(kv) != 2:
+                    continue
+                k, v = kv
+                k = k.strip()
+                v = v.strip()
+                currcfg[k] = v
+
+    def apply_configuration(self, cfg):
+        stgs = cfg['settings']
+        ipcfg = 'STARTMODE=auto\n'
+        routecfg = ''
+        bootproto4 = stgs.get('ipv4_method', 'none')
+        bootproto6 = stgs.get('ipv6_method', 'none')
+        if bootproto4 == 'dhcp' and bootproto6 == 'dhcp':
+            ipcfg += 'BOOTPROTO=dhcp\n'
+        elif bootproto4 == 'dhcp':
+            ipcfg += 'BOOTPROTO=dhcp4\n'
+        elif bootproto6 == 'dhcp':
+            ipcfg += 'BOOTPROTO=dhcp6\n'
+        else:
+            ipcfg += 'BOOTPROTO=static\n'
+        if stgs.get('ipv4_address', None):
+            ipcfg += 'IPADDR=' + stgs['ipv4_address'] + '\n'
+        v4gw = stgs.get('ipv4_gateway', None)
+        if stgs.get('ipv6_address', None):
+            ipcfg += 'IPADDR_V6=' + stgs['ipv6_address'] + '\n'
+        v6gw = stgs.get('ipv6_gateway', None)
+        cname = None
+        if len(cfg['interfaces']) > 1:  # creating new team
+            if not stgs.get('team_mode', None):
+                sys.stderr.write("Warning, multiple interfaces ({0}) without a team_mode, skipping setup\n".format(','.join(cfg['interfaces'])))
+                return
+            if not stgs.get('connection_name', None):
+                stgs['connection_name'] = 'bond{0}'.format(self.teamidx)
+                self.teamidx += 1
+            cname = stgs['connection_name']
+            with open('/etc/sysconfig/network/ifcfg-{0}'.format(cname), 'w') as teamout:
+                teamout.write(ipcfg)
+                if stgs['team_mode'] == 'lacp':
+                    stgs['team_mode'] = '802.3ad'
+                teamout.write("BONDING_MODULE_OPTS='mode={0} miimon=100'\nBONDING_MASTER=yes\n".format(stgs['team_mode']))
+                idx = 1
+                for iface in cfg['interfaces']:
+                    subprocess.call(['wicked', 'ifdown', iface])
+                    try:
+                        os.remove('/etc/sysconfig/network/ifcfg-{0}'.format(iface))
+                        os.remove('/etc/sysconfig/network/ifroute-{0}'.format(iface))
+                    except OSError:
+                        pass
+                    teamout.write('BONDING_SLAVE{0}={1}\n'.format(idx, iface))
+                    idx += 1
+        else:
+            cname = list(cfg['interfaces'])[0]
+            priorcfg = self.cfgbydev.get(cname, {})
+            for cf in priorcfg:
+                if cf.startswith('TEAM_'):
+                    ipcfg += '{0}={1}\n'.format(cf, priorcfg[cf])
+            with open('/etc/sysconfig/network/ifcfg-{0}'.format(cname), 'w') as iout:
+                iout.write(ipcfg)
+        if v4gw:
+            routecfg += 'default {0} - {1}\n'.format(v4gw, cname)
+        if v6gw:
+            routecfg += 'default {0} - {1}\n'.format(v6gw, cname)
+        if routecfg:
+            with open('/etc/sysconfig/network/ifroute-{0}'.format(cname), 'w') as routeout:
+                routeout.write(routecfg)
+        subprocess.call(['wicked', 'ifup', cname])
+
+
+class NetworkManager(object):
+    bondtypes = {
+            'lacp': '802.3ad',
+            'loadbalance': 'balance-alb',
+            'roundrobin': 'balance-rr',
+            'activebackup': 'active-backup',
+    }
+    def __init__(self, devtypes, deploycfg):
+        self.deploycfg = deploycfg
+        self.connections = {}
+        self.uuidbyname = {}
+        self.uuidbydev = {}
+        self.connectiondetail = {}
+        self.read_connections()
+        self.teamidx = 0
+        self.devtypes = devtypes
+
+    def read_connections(self):
+        self.connections = {}
+        self.uuidbyname = {}
+        self.uuidbydev = {}
+        self.connectiondetail = {}
+        ci = subprocess.check_output(['nmcli', '-t', 'c']).decode('utf8')
+        for inf in ci.splitlines():
+            n, u, t, dev = inf.split(':')
+            if n == 'NAME':
+                continue
+            if dev == '--':
+                dev = None
+            self.uuidbyname[n] = u
+            if dev:
+                self.uuidbydev[dev] = u
+            self.connections[u] = {'name': n, 'uuid': u, 'type': t, 'dev': dev}
+            deats = {}
+            for deat in subprocess.check_output(['nmcli', 'c', 's', u]).decode('utf8').splitlines():
+                k, v = deat.split(':', 1)
+                v = v.strip()
+                if v == '--':
+                    continue
+                if '(default)' in v:
+                    continue
+                deats[k] = v
+            self.connectiondetail[u] = deats
+
+
+    def add_team_member(self, team, member):
+        bondcfg = {}
+        if member in self.uuidbydev:
+            myuuid = self.uuidbydev[member]
+            deats = self.connectiondetail[myuuid]
+            currteam = deats.get('connection.master', None)
+            if currteam == team:
+                return
+            for stg in ('ipv4.dhcp-hostname', 'ipv4.dns', 'ipv6.dns', 'ipv6.dhcp-hostname'):
+                if deats.get(stg, None):
+                    bondcfg[stg] = deats[stg]
+        if member in self.uuidbyname:
+            subprocess.check_call(['nmcli', 'c', 'del', self.uuidbyname[member]])
+        devtype = self.devtypes.get(member, 'bond-slave')
+        subprocess.check_call(['nmcli', 'c', 'add', 'type', devtype, 'master', team, 'con-name', member, 'connection.interface-name', member])
+        if bondcfg:
+            args = []
+            for parm in bondcfg:
+                args.append(parm)
+                args.append(bondcfg[parm])
+            subprocess.check_call(['nmcli', 'c', 'm', team] + args)
+
+    def apply_configuration(self, cfg, lastchance=False):
+        cmdargs = {}
+        cmdargs['connection.autoconnect'] = 'yes'
+        stgs = cfg['settings']
+        cmdargs['ipv6.method'] = stgs.get('ipv6_method', 'link-local')
+        if stgs.get('ipv6_address', None):
+            cmdargs['ipv6.addresses'] = stgs['ipv6_address']
+        cmdargs['ipv4.method'] = stgs.get('ipv4_method', 'disabled')
+        if stgs.get('ipv4_address', None):
+            cmdargs['ipv4.addresses'] = stgs['ipv4_address']
+        if stgs.get('ipv4_gateway', None):
+            cmdargs['ipv4.gateway'] = stgs['ipv4_gateway']
+        if stgs.get('ipv6_gateway', None):
+            cmdargs['ipv6.gateway'] = stgs['ipv6_gateway']
+        dnsips = self.deploycfg.get('nameservers', [])
+        if not dnsips:
+            dnsips = []
+        dns4 = []
+        dns6 = []
+        for dnsip in dnsips:
+            if '.' in dnsip:
+                dns4.append(dnsip)
+            elif ':' in dnsip:
+                dns6.append(dnsip)
+        if dns4:
+            cmdargs['ipv4.dns'] = ','.join(dns4)
+        if dns6:
+            cmdargs['ipv6.dns'] = ','.join(dns6)
+        if len(cfg['interfaces']) > 1:  # team time.. should be..
+            if not cfg['settings'].get('team_mode', None):
+                sys.stderr.write("Warning, multiple interfaces ({0}) without a team_mode, skipping setup\n".format(','.join(cfg['interfaces'])))
+                return
+            if not cfg['settings'].get('connection_name', None):
+                cfg['settings']['connection_name'] = 'team{0}'.format(self.teamidx)
+                self.teamidx += 1
+            cname = cfg['settings']['connection_name']
+            cargs = []
+            for arg in cmdargs:
+                cargs.append(arg)
+                cargs.append(cmdargs[arg])
+            if stgs['team_mode'] in self.bondtypes:
+                stgs['team_mode'] = self.bondtypes[stgs['team_mode']]
+            subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond', 'con-name', cname, 'connection.interface-name', cname, 'bond.options', 'mode={}'.format(stgs['team_mode'])] + cargs)
+            for iface in cfg['interfaces']:
+                self.add_team_member(cname, iface)
+            subprocess.check_call(['nmcli', 'c', 'u', cname])
+        else:
+            cname = stgs.get('connection_name', None)
+            iname = list(cfg['interfaces'])[0]
+            ctype = self.devtypes.get(iname, None)
+            if not ctype:
+                if lastchance:
+                    sys.stderr.write("Warning, no device found for interface_name ({0}), skipping setup\n".format(iname))
+                return 1
+            if stgs.get('vlan_id', None):
+                vlan = stgs['vlan_id']
+                if ctype == 'infiniband':
+                    vlan = '0x{0}'.format(vlan) if not vlan.startswith('0x') else vlan
+                    cmdargs['infiniband.parent'] = iname
+                    cmdargs['infiniband.p-key'] = vlan
+                    iname = '{0}.{1}'.format(iname, vlan[2:])
+                elif ctype == 'ethernet':
+                    ctype = 'vlan'
+                    cmdargs['vlan.parent'] = iname
+                    cmdargs['vlan.id'] = vlan
+                    iname = '{0}.{1}'.format(iname, vlan)
+                else:
+                    sys.stderr.write("Warning, unknown interface_name ({0}) device type ({1}) for VLAN/PKEY, skipping setup\n".format(iname, ctype))
+                    return
+            cname = iname if not cname else cname
+            u = self.uuidbyname.get(cname, None)
+            cargs = []
+            for arg in cmdargs:
+                cargs.append(arg)
+                cargs.append(cmdargs[arg])
+            if u:
+                subprocess.check_call(['nmcli', 'c', 'm', u, 'connection.interface-name', iname] + cargs)
+                subprocess.check_call(['nmcli', 'c', 'u', u])
+            else:
+                subprocess.check_call(['nmcli', 'c', 'add', 'type', ctype, 'con-name', cname, 'connection.interface-name', iname] + cargs)
+                self.read_connections()
+                u = self.uuidbyname.get(cname, None)
+                if u:
+                    subprocess.check_call(['nmcli', 'c', 'u', u])
+
+
+
+if __name__ == '__main__':
+    havefirewall = subprocess.call(['systemctl', 'status', 'firewalld'])
+    havefirewall = havefirewall == 0
+    if havefirewall:
+        subprocess.check_call(['systemctl', 'stop', 'firewalld'])
+    tmpllas = add_missing_llas()
+    await_tentative()
+    idxmap, devtypes = map_idx_to_name()
+    netname_to_interfaces = {}
+    myaddrs = apiclient.get_my_addresses()
+    srvs, _ = apiclient.scan_confluents()
+    doneidxs = set([])
+    dc = None
+    if not srvs:  # the multicast scan failed, fallback to deploycfg cfg file
+        with open('/etc/confluent/confluent.deploycfg', 'r') as dci:
+            for cfgline in dci.read().split('\n'):
+                if cfgline.startswith('deploy_server:'):
+                    srvs = [cfgline.split()[1]]
+                    break
+    for srv in srvs:
+        try:
+            s = socket.create_connection((srv, 443))
+        except socket.error:
+            continue
+        myname = s.getsockname()
+        s.close()
+        if len(myname) == 4:
+            curridx = myname[-1]
+        else:
+            myname = myname[0]
+            myname = socket.inet_pton(socket.AF_INET, myname)
+            for addr in myaddrs:
+                if myname == addr[1].tobytes():
+                    curridx = addr[-1]
+        if curridx in doneidxs:
+            continue
+        for tries in (1, 2, 3):
+            try:
+                status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg')
+                break
+            except Exception:
+                if tries == 3:
+                    raise
+                time.sleep(1)
+                continue
+        nc = json.loads(nc)
+        if not dc:
+            for tries in (1, 2, 3):
+                try:
+                    status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2')
+                    break
+                except Exception:
+                    if tries == 3:
+                        raise
+                    time.sleep(1)
+                    continue
+            dc = json.loads(dc)
+        iname = get_interface_name(idxmap[curridx], nc.get('default', {}))
+        if iname:
+            for iname in iname.split(','):
+                if 'default' in netname_to_interfaces:
+                    netname_to_interfaces['default']['interfaces'].add(iname)
+                else:
+                    netname_to_interfaces['default'] = {'interfaces': set([iname]), 'settings': nc['default']}
+        for netname in nc.get('extranets', {}):
+            uname = '_' + netname
+            iname = get_interface_name(idxmap[curridx], nc['extranets'][netname])
+            if iname:
+                for iname in iname.split(','):
+                    if uname in netname_to_interfaces:
+                        netname_to_interfaces[uname]['interfaces'].add(iname)
+                    else:
+                        netname_to_interfaces[uname] = {'interfaces': set([iname]), 'settings': nc['extranets'][netname]}
+        doneidxs.add(curridx)
+    if 'default' in netname_to_interfaces:
+        for netn in netname_to_interfaces:
+            if netn == 'default':
+                continue
+            netname_to_interfaces['default']['interfaces'] -= netname_to_interfaces[netn]['interfaces']
+        if not netname_to_interfaces['default']['interfaces']:
+            del netname_to_interfaces['default']
+    # Make sure VLAN/PKEY connections are created last
+    netname_to_interfaces = dict(sorted(netname_to_interfaces.items(), key=lambda item: 'vlan_id' in item[1]['settings']))
+    rm_tmp_llas(tmpllas)
+    if os.path.exists('/usr/sbin/netplan'):
+        nm = NetplanManager(dc)
+    if os.path.exists('/usr/bin/nmcli'):
+        nm = NetworkManager(devtypes, dc)
+    elif os.path.exists('/usr/sbin/wicked'):
+        nm = WickedManager()
+    retrynics = []
+    for netn in netname_to_interfaces:
+        redo = nm.apply_configuration(netname_to_interfaces[netn])
+        if redo == 1:
+            retrynics.append(netn)
+    if retrynics:
+        idxmap, devtypes = map_idx_to_name()
+        if os.path.exists('/usr/sbin/netplan'):
+            nm = NetplanManager(dc)
+        if os.path.exists('/usr/bin/nmcli'):
+            nm = NetworkManager(devtypes, dc)
+        elif os.path.exists('/usr/sbin/wicked'):
+            nm = WickedManager()
+        for netn in retrynics:
+            nm.apply_configuration(netname_to_interfaces[netn], lastchance=True)
+    if havefirewall:
+        subprocess.check_call(['systemctl', 'start', 'firewalld'])
+    await_tentative()
+

confluent_osdeploy/debian/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/debian/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+echo "Confluent first boot is running"
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
+#cp -a /etc/confluent/ssh/* /etc/ssh/
+#systemctl restart sshd
+rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+if [ ! -z "$rootpw" -a "$rootpw" != "null" ]; then
+	echo root:$rootpw | chpasswd -e
+fi
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+while ! ping -c 1 $confluent_mgr >& /dev/null; do
+	sleep 1
+done
+source /etc/confluent/functions
+
+run_remote_parts firstboot.d
+run_remote_config firstboot.d
+systemctl disable firstboot
+curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus

confluent_osdeploy/debian/profiles/default/scripts/functions

@@ -0,0 +1,209 @@
+#!/bin/bash
+function test_mgr() {
+    whost=$1
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
+        whost="[$whost]"
+    fi
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
+        return 0
+    fi
+    return 1
+}
+
+function confluentpython() {
+    if [ -x /usr/libexec/platform-python ]; then
+        /usr/libexec/platform-python $*
+    elif [ -x /usr/bin/python3 ]; then
+        /usr/bin/python3 $*
+    elif [ -x /usr/bin/python ]; then
+        /usr/bin/python $*
+    elif [ -x /usr/bin/python2 ]; then
+        /usr/bin/python2 $*
+    fi
+}
+
+function set_confluent_vars() {
+    if [ -z "$nodename" ]; then
+        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+    fi
+    if [[ "$confluent_mgr" == *"%"* ]]; then
+        confluent_mgr=""
+    fi
+    if [ -z "$confluent_mgr" ]; then
+        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+        if ! test_mgr $confluent_mgr; then
+            confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+            if [[ "$confluent_mgr" = *":"* ]]; then
+                confluent_mgr="[$confluent_mgr]"
+            fi
+        fi
+        if ! test_mgr $confluent_mgr; then
+            BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            for confluent_mgr in $BESTMGRS $OKMGRS; do
+                if [[ $confluent_mgr == *":"* ]]; then
+                    confluent_mgr="[$confluent_mgr]"
+                fi
+                if test_mgr $confluent_mgr; then
+                    break
+                fi
+            done
+        fi
+    fi
+    if [ -z "$confluent_profile" ]; then
+        confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+    export confluent_profile confluent_mgr nodename
+}
+
+fetch_remote() {
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
+        whost="[$whost]"
+    fi
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+source_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        source_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+run_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        run_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+source_remote() {
+    set_confluent_vars
+    unsettmpdir=0
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Sourcing from $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    shift
+    source ./$cmd
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    rm -rf $confluentscripttmpdir
+    return $retcode
+}
+
+run_remote() {
+    requestedcmd="'$*'"
+    unsettmpdir=0
+    set_confluent_vars
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    if [ $retcode -ne 0 ]; then
+        echo "$requestedcmd exited with code $retcode"
+    fi
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    mkdir -p $(dirname $1)
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
+        whost="[$whost]"
+    fi
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    confluentpython $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+    return $retcode
+}
+
+run_remote_config() {
+    echo
+    set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile
+    confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204
+    echo
+    echo 'Completed remote configuration'
+    echo '---------------------------------------------------------------------------'
+    return
+}
+#If invoked as a command, use the arguments to actually run a function
+(return 0 2>/dev/null) || $1 "${@:2}"

confluent_osdeploy/debian/profiles/default/scripts/post.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+mkdir -p /run/sshd
+mkdir -p /root/.ssh
+cat /tmp/ssh/*pubkey >> /root/.ssh/authorized_keys
+cat /tmp/ssh/*.ca | sed -e s/^/'@cert-authority * '/ >> /etc/ssh/ssh_known_hosts
+chmod 700 /etc/confluent
+chmod go-rwx /etc/confluent/*
+sshconf=/etc/ssh/ssh_config
+if [ -d /etc/ssh/ssh_config.d/ ]; then
+    sshconf=/etc/ssh/ssh_config.d/01-confluent.conf
+fi
+echo 'Host *' >> $sshconf
+echo '    HostbasedAuthentication yes' >> $sshconf
+echo '    EnableSSHKeysign yes' >> $sshconf
+echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
+/usr/sbin/sshd
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | awk '{print $2}')
+mkdir -p /opt/confluent/bin
+python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/firstboot.sh > /opt/confluent/bin/firstboot.sh
+chmod +x /opt/confluent/bin/firstboot.sh
+python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/firstboot.service > /etc/systemd/system/firstboot.service
+systemctl enable firstboot
+python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+source /etc/confluent/functions
+python3 /opt/confluent/bin/apiclient /confluent-api/self/nodelist | sed -e s/'^- //' > /tmp/allnodes
+cp /tmp/allnodes /root/.shosts
+cp /tmp/allnodes /etc/ssh/shosts.equiv
+if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
+    ntps=$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //' | paste -sd ' ')
+    sed -i "s/#NTP=/NTP=$ntps/" /etc/systemd/timesyncd.conf
+fi
+textcons=$(grep ^textconsole: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+updategrub=0
+if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
+    cons=""
+    if [ -f /tmp/autocons.info ]; then
+        cons=$(cat /tmp/autocons.info)
+    fi
+    if [ ! -z "$cons" ]; then
+        sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 console='${cons#/dev/}'"/' /etc/default/grub
+        updategrub=1
+    fi
+fi
+kargs=$(python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/profile.yaml | grep ^installedargs: | sed -e 's/#.*//')
+if [ ! -z "$kargs" ]; then
+    sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /etc/default/grub
+fi
+
+if [ 1 = $updategrub ]; then
+    update-grub
+fi
+
+if [ -e /sys/firmware/efi ]; then
+    bootnum=$(efibootmgr | grep ubuntu | sed -e 's/ .*//' -e 's/\*//' -e s/Boot//)
+    if [ ! -z "$bootnum" ]; then
+        currboot=$(efibootmgr | grep ^BootOrder: | awk '{print $2}')
+        nextboot=$(echo $currboot| awk -F, '{print $1}')
+        [ "$nextboot" = "$bootnum" ] || efibootmgr -o $bootnum,$currboot
+        efibootmgr -D
+    fi
+fi
+run_remote_python syncfileclient
+run_remote_parts post.d
+run_remote_config post
+
+python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus  -d 'status: staged'
+

confluent_osdeploy/debian/profiles/default/scripts/pre.sh

@@ -1,18 +1,88 @@
-anna-install openssh-server-udeb
-mkdir -p ~/.ssh/
-cat /ssh/*pubkey > ~/.ssh/authorized_keys
-ssh-keygen -A
-mgr=$(grep ^MANAGER:.*\\. /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
-nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
+#!/bin/sh
+## Use the following option to add additional boot parameters for the
+## installed system (if supported by the bootloader installer).
+## Note: options passed to the installer will be added automatically.
+#d-i debian-installer/add-kernel-opts string [from profile.yaml]
+deploycfg=/etc/confluent/confluent.deploycfg
+mgr=$(cat /etc/confluent/deployer)
+
+cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //')
+if [ "$cryptboot" != "" ]  && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then
+   echo "****Encrypted boot requested, but not implemented for this OS, halting install" > /dev/console
+   [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but not implemented for this OS,halting install" >> $(cat /tmp/autoconsdev))
+   while :; do sleep 86400; done
+fi
+cat > /usr/lib/live-installer.d/confluent-certs << EOF
+#!/bin/sh
+cp /tls/* /target/etc/ssl/certs/
+cat /tls/*.pem >> /target/etc/ssl/certs/ca-certificates.crt
+EOF
+chmod a+x /usr/lib/live-installer.d/confluent-certs
+mkdir -p /.ssh/
+cat /ssh/*pubkey > /.ssh/authorized_keys
+mkdir -p /etc/ssh
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|cut -d ' ' -f 2)
 apikey=$(cat /etc/confluent/confluent.apikey)
+ssh-keygen -A
 for pubkey in /etc/ssh/ssh_host*key.pub; do
-    certfile=${pubkey%.pub}-cert.pub
+    certfile=$(echo $pubkey | sed -e s/.pub/-cert.pub/)
     keyfile=${pubkey%.pub}
-    wget --post-file=$pubkey --header='CONFLUENT_NODENAME: '$nodename --header="CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/sshcert -O $certfile
+    wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" --post-file=$pubkey https://$mgr/confluent-api/self/sshcert -O $certfile --quiet
     echo HostKey $keyfile >> /etc/ssh/sshd_config
     echo HostCertificate $certfile >> /etc/ssh/sshd_config
 done
-
-echo sshd:x:939:939::/: >> /etc/passwd
+if [ -e /tmp/installdisk ]; then
+    instdisk=$(cat /tmp/installdisk)
+else
+    for blockdev in $(ls /sys/class/block/); do
+        shortname=$(basename $blockdev)
+        if [ "$shortname" != "${shortname%loop*}" ]; then
+            continue
+        fi
+        udevadm info --query=property /dev/$shortname |grep DEVTYPE=disk > /dev/null || continue # ignore partitions
+        udevadm info --query=property /dev/$shortname |grep DM_NAME > /dev/null && continue # not a real disk
+        sz=$(cat /sys/block/$shortname/size 2> /dev/null)
+        [ -z "$sz" ] && continue
+        [ $sz -lt 1048576 ] && continue # Too small
+        [ -z "$firstdisk" ] && firstdisk=$shortname
+        if udevadm info --query=property /dev/$shortname|grep ID_MODEL=| sed -e s/' '/_/g | grep -iE '(thinksystem_m.2|m.2_nvme_2-bay_raid_kit)' > /dev/null; then
+            instdisk=$shortname
+            break
+        fi
+        if udevadm info --query=property /dev/$shortname|grep MD_CONTAINER=imsm; then
+            sraid=$sortname
+        else
+            drv=$(udevadm info -a /dev/sdb|grep DRIVERS==|grep -Ev '""|"sd"' | sed -e s/.*=// -e s/'"'//g)
+            if [ "ahci" = "$drv" -a -z "$onbdisk" ]; then
+                onbdisk=$shortname
+            elif [ "megaraid" = "$drv" -a -z "$rdisk" ]; then
+                rdisk=$shortname
+            fi
+        fi
+    done
+fi
+if [ -z "$instdisk" ]; then
+    if [ ! -z "$sraid"]; then
+        instdisk=$sraid
+    elif [ ! -z "$onbdisk" ]; then
+        instdisk=$onbdisk
+    elif [ ! -z "$rdisk" ]; then
+        instdisk=$rdisk
+    else
+        instdisk=$firstdisk
+    fi
+fi
+if [ ! -z "$instdisk" ]; then
+    debconf-set partman-auto/disk /dev/$instdisk
+    debconf-set grub-installer/bootdev /dev/$instdisk
+fi
+echo HostbasedAuthentication yes >> /etc/ssh/sshd_config
+echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config
+echo IgnoreRhosts no >> /etc/ssh/sshd_config
+echo sshd:x:1:1::/run/sshd:/bin/false >> /etc/passwd
 /usr/sbin/sshd
-
+wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/nodelist -O /tmp/allnodes --quiet
+#kill -HUP $(ps | grep -v grep | grep /usr/sbin/sshd | sed -e 's/^ *//'|cut -d ' ' -f 1)
+#curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /tmp/getinstalldisk
+#python3 /tmp/getinstalldisk
+#sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml

confluent_osdeploy/debian/profiles/default/scripts/prechroot.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+mount -o bind /sys /target/sys
+mount -o bind /dev /target/dev
+mount -o bind /dev/pts /target/dev/pts
+mount -o bind /proc /target/proc
+mount -o bind /dev/pts /target/dev/pts
+mount -o bind /run /target/run
+cp -a /etc/confluent /target/etc/confluent
+cp -a /opt/confluent /target/opt/confluent
+mv /tmp/post.sh /target/tmp/
+cp -a /ssh /tls /target/tmp
+cat /tls/*.pem >> /target/etc/confluent/ca.pem
+cp -a /etc/ssh/ssh_host_* /target/etc/ssh/
+grep HostCertificate /etc/ssh/sshd_config  >> /target/etc/ssh/sshd_config
+echo Port 2222 >> /etc/ssh/sshd_config
+kill -HUP $(ps |grep -v grep|grep sshd|grep /usr|sed -e s/' root.*//')
+cp /tls/* /target/etc/ssl/certs/
+cat /tls/*.pem >> /target/etc/ssl/certs/ca-certificates.crt
+chroot /target bash /tmp/post.sh

confluent_osdeploy/debian/profiles/default/scripts/proxmox/proxmoxve.firstboot

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Add this to firstboot.d
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y install proxmox-ve postfix open-iscsi chrony < /dev/null
+

confluent_osdeploy/debian/profiles/default/scripts/proxmox/proxmoxve.post

@@ -0,0 +1,20 @@
+#!/bin/bash
+# This script would run in post.d
+#
+export DEBIAN_FRONTEND=noninteractive
+echo "deb [arch=amd64] http://download.proxmox.com/debian/pve bookworm pve-no-subscription" > /etc/apt/sources.list.d/pve-install-repo.list
+wget https://enterprise.proxmox.com/debian/proxmox-release-bookworm.gpg -O /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg
+sum=$(sha512sum /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg)
+if [ "$sum" -ne "7da6fe34168adc6e479327ba517796d4702fa2f8b4f0a9833f5ea6e6b48f6507a6da403a274fe201595edc86a84463d50383d07f64bdde2e3658108db7d6dc87" ]; then
+	echo "Mismatch in fingerprint!"
+	rm /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg
+	exit 1
+fi
+apt-get update && apt-get -y full-upgrade < /dev/null
+
+apt-get -y install proxmox-default-kernel < /dev/null
+apt-get -y remove linux-image-amd64 'linux-image-6.1*' < /dev/null
+update-grub
+apt-get -y remove os-prober < /dev/null
+
+

confluent_osdeploy/debian/profiles/default/scripts/setupssh

@@ -0,0 +1,43 @@
+[ -f /lib/confluent/functions ] && . /lib/confluent/functions
+[ -f /etc/confluent/functions ] && . /etc/confluent/functions
+[ -f /opt/confluent/bin/apiclient ] && confapiclient=/opt/confluent/bin/apiclient
+[ -f /etc/confluent/apiclient ] && confapiclient=/etc/confluent/apiclient
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    if [ "$pubkey" = /etc/ssh/ssh_host_key.pub ]; then
+        continue
+    fi
+    certfile=${pubkey/.pub/-cert.pub}
+    rm $certfile
+    confluentpython $confapiclient /confluent-api/self/sshcert $pubkey -o $certfile
+done
+if [ -d /etc/ssh/sshd_config.d/ -a ! -e /etc/ssh/sshd_config.d/90-confluent.conf ]; then
+	for cert in /etc/ssh/ssh*-cert.pub; do
+		echo HostCertificate $cert >> /etc/ssh/sshd_config.d/90-confluent.conf
+	done
+	echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/90-confluent.conf
+	echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/90-confluent.conf
+	echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/90-confluent.conf
+fi
+
+TMPDIR=$(mktemp -d)
+cd $TMPDIR
+confluentpython $confapiclient /confluent-public/site/initramfs.tgz -o initramfs.tgz
+tar xf initramfs.tgz
+for ca in ssh/*.ca; do
+	LINE=$(cat $ca)
+	cp -af /etc/ssh/ssh_known_hosts /etc/ssh/ssh_known_hosts.new
+	grep -v "$LINE" /etc/ssh/ssh_known_hosts > /etc/ssh/ssh_known_hosts.new
+	echo '@cert-authority *' $LINE >> /etc/ssh/ssh_known_hosts.new
+	mv /etc/ssh/ssh_known_hosts.new /etc/ssh/ssh_known_hosts
+done
+for pubkey in ssh/*.*pubkey; do
+	LINE=$(cat $pubkey)
+	cp -af /root/.ssh/authorized_keys /root/.ssh/authorized_keys.new
+	grep -v "$LINE" /root/.ssh/authorized_keys > /root/.ssh/authorized_keys.new
+	echo "$LINE" >> /root/.ssh/authorized_keys.new
+	mv /root/.ssh/authorized_keys.new /root/.ssh/authorized_keys
+done
+confluentpython $confapiclient /confluent-api/self/nodelist | sed -e 's/^- //' > /etc/ssh/shosts.equiv
+cat /etc/ssh/shosts.equiv > /root/.shosts
+cd -
+rm -rf $TMPDIR

confluent_osdeploy/el10-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -0,0 +1,327 @@
+get_remote_apikey() {
+    while [ -z "$confluent_apikey" ]; do
+        /opt/confluent/bin/clortho $nodename $confluent_mgr > /etc/confluent/confluent.apikey
+        if grep ^SEALED: /etc/confluent/confluent.apikey > /dev/null; then
+            # we don't support remote sealed api keys anymore
+            echo > /etc/confluent/confluent.apikey
+        fi
+        confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+        if [ -z "$confluent_apikey" ]; then
+            echo "Unable to acquire node api key, set deployment.apiarmed=once on node '$nodename', retrying..."
+            if [ ! -z "$autoconsdev" ]; then echo "Unable to acquire node api key, set deployment.apiarmed=once on node '$nodename', retrying..." > $autoconsdev; fi
+            sleep 10
+        elif [ -c /dev/tpmrm0 ]; then
+            tmpdir=$(mktemp -d)
+            cd $tmpdir
+            tpm2_startauthsession --session=session.ctx
+            tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
+            tpm2_createprimary -G ecc -Q --key-context=prim.ctx
+            (echo -n "CONFLUENT_APIKEY:";cat /etc/confluent/confluent.apikey) | tpm2_create -Q --policy=pcr15.sha256.policy --public=data.pub --private=data.priv -i - -C prim.ctx
+            tpm2_load -Q --parent-context=prim.ctx --public=data.pub --private=data.priv --name=confluent.apikey --key-context=data.ctx
+            tpm2_evictcontrol -Q -c data.ctx
+            tpm2_flushcontext session.ctx
+            cd - > /dev/null
+            rm -rf $tmpdir
+        fi
+    done
+}
+root=1
+rootok=1
+netroot=confluent
+echo -ne '\033[H\033[2J\033[3J'
+mkdir -p /etc/ssh
+mkdir -p /var/tmp/
+mkdir -p /var/empty/sshd
+mkdir -p /usr/share/empty.sshd
+mkdir -p /etc/confluent
+sed -i '/^root:x/d' /etc/passwd
+echo root:x:0:0::/:/bin/bash >> /etc/passwd
+echo sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin >> /etc/passwd
+
+if ! grep console= /proc/cmdline >& /dev/null; then
+    autocons=$(/opt/confluent/bin/autocons)
+    autoconsdev=${autocons%,*}
+    autocons=${autocons##*/}
+    echo "Automatic console configured for $autocons"
+fi
+echo "Initializing confluent diskless environment"
+echo -n "udevd: "
+/usr/lib/systemd/systemd-udevd --daemon
+echo -n "Loading drivers..."
+udevadm trigger
+udevadm trigger --type=devices --action=add
+udevadm settle
+modprobe ib_ipoib
+modprobe ib_umad
+modprobe hfi1
+modprobe mlx5_ib
+echo "done"
+cat > /etc/ssh/sshd_config << EOF
+Port 2222
+Subsystem       sftp    /usr/libexec/openssh/sftp-server
+PermitRootLogin yes
+AuthorizedKeysFile      .ssh/authorized_keys
+EOF
+mkdir /root/.ssh
+mkdir /.ssh
+cat /ssh/*pubkey > /root/.ssh/authorized_keys 2>/dev/null
+cp /root/.ssh/authorized_keys /.ssh/
+cat /tls/*.pem > /etc/confluent/ca.pem
+mkdir -p /etc/pki/tls/certs
+cat /tls/*.pem > /etc/pki/tls/certs/ca-bundle.crt
+TRIES=0
+oldumask=$(umask)
+umask 0077
+tpmdir=$(mktemp -d)
+cd $tpmdir
+lasthdl=""
+if [ -c /dev/tpmrm0 ]; then
+    for hdl in $(tpm2_getcap handles-persistent|awk '{print $2}'); do
+        tpm2_startauthsession --policy-session --session=session.ctx
+        tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
+        unsealeddata=$(tpm2_unseal --auth=session:session.ctx -Q -c $hdl 2>/dev/null)
+        tpm2_flushcontext session.ctx
+        if [[ $unsealeddata == "CONFLUENT_APIKEY:"* ]]; then
+            confluent_apikey=${unsealeddata#CONFLUENT_APIKEY:}
+            echo $confluent_apikey > /etc/confluent/confluent.apikey
+            if [ -n "$lasthdl" ]; then
+                tpm2_evictcontrol -c $lasthdl
+            fi
+            lasthdl=$hdl
+        fi
+    done
+fi
+cd - > /dev/null
+rm -rf $tpmdir
+touch /etc/confluent/confluent.info
+cd /sys/class/net
+echo -n "Scanning for network configuration..."
+while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 30 ]; do
+    TRIES=$((TRIES + 1))
+    for i in *; do
+        ip link set $i up
+    done
+    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+    echo -n .
+done
+TRIES=0
+while ! grep ^NODENAME: /etc/confluent/confluent.info >& /dev/null && [ "$TRIES" -lt 300 ]; do
+    sleep 0.5
+    echo -n .
+    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+    TRIES=$((TRIES + 1))
+done
+cd /
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+hostname $nodename
+confluent_mgr=$(grep '^EXTMGRINFO:.*1$' /etc/confluent/confluent.info | head -n 1 | awk -F': ' '{print $2}' | awk -F'|' '{print $1}')
+if [ -z "$confluent_mgr" ]; then
+    confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info|head -n 1 | awk '{print $2}')
+fi
+if [[ $confluent_mgr == *%* ]]; then
+    echo $confluent_mgr | awk -F% '{print $2}' > /tmp/confluent.ifidx
+    ifidx=$(cat /tmp/confluent.ifidx)
+    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+    ifname=${ifname%:}
+fi
+
+ready=0
+while [ $ready = "0" ]; do
+    get_remote_apikey
+    if [[ $confluent_mgr == *:* ]] && [[ $confluent_mgr != "["* ]]; then
+        confluent_mgr="[$confluent_mgr]"
+    fi
+    tmperr=$(mktemp)
+    curl -sSf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg 2> $tmperr
+    if grep 401 $tmperr > /dev/null; then
+        confluent_apikey=""
+        if [ -n "$lasthdl" ]; then
+            tpm2_evictcontrol -c $lasthdl
+        fi
+        confluent_mgr=${confluent_mgr#[}
+        confluent_mgr=${confluent_mgr%]}
+    elif grep 'SSL' $tmperr > /dev/null; then
+        confluent_mgr=${confluent_mgr#[}
+        confluent_mgr=${confluent_mgr%]}
+    	echo 'Failure establishing TLS conneection to '$confluent_mgr' (try `osdeploy initialize -t` on the deployment server)'
+	if [ ! -z "$autoconsdev" ]; then echo 'Failure establishing TLS conneection to '$confluent_mgr' (try `osdeploy initialize -t` on the deployment server)' > $autoconsdev; fi
+    	sleep 10
+    else
+        ready=1
+    fi
+    rm $tmperr
+done
+if [ ! -z "$autocons" ] && grep "textconsole: true" /etc/confluent/confluent.deploycfg > /dev/null; then /opt/confluent/bin/autocons -c > /dev/null; fi
+if [ -c /dev/tpmrm0 ]; then
+    tpm2_pcrextend 15:sha256=2fbe96c50dde38ce9cd2764ddb79c216cfbcd3499568b1125450e60c45dd19f2
+fi
+umask $oldumask
+mkdir -p /run/NetworkManager/system-connections
+cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+[connection]
+EOC
+echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
+echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+linktype=$(ip link show dev ${ifname}|grep link/|awk '{print $1}')
+if [ "$linktype" = link/infiniband ]; then
+        linktype="infiniband"
+else
+        linktype="ethernet"
+fi
+echo type=$linktype >> /run/NetworkManager/system-connections/$ifname.nmconnection
+cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+autoconnect-retries=1
+EOC
+echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
+cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+multi-connect=1
+permissions=
+wait-device-timeout=60000
+
+EOC
+if [ "$linktype" = infiniband ]; then
+cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+[infiniband]
+transport-mode=datagram
+
+EOC
+fi
+autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+auto6configmethod=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+if [ "$autoconfigmethod" = "dhcp" ]; then
+    echo -n "Attempting to use dhcp to bring up $ifname..."
+    dhcpcd $ifname
+    echo "Complete:"
+    ip addr show dev $ifname
+    confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+elif [ "$autoconfigmethod" = "static" ]; then
+    confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+    v4addr=${v4addr#ipv4_address: }
+    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+    v4gw=${v4gw#ipv4_gateway: }
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ^prefix: /etc/confluent/confluent.deploycfg)
+    v4nm=${v4nm#prefix: }
+    echo "Setting up $ifname as static at $v4addr/$v4nm"
+    ip addr add dev $ifname $v4addr/$v4nm
+    if [ ! -z "$v4gw" ]; then
+        ip route add default via $v4gw
+    fi
+    echo '[ipv4]' >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    echo address1=$v4addr/$v4nm >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    if [ ! -z "$v4gw" ]; then
+        echo gateway=$v4gw >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    fi
+    nameserversec=0
+    nameservers=""
+    while read -r entry; do
+        if [ $nameserversec = 1 ]; then
+            if [[ $entry == "-"*.* ]]; then
+                nameservers="$nameservers"${entry#- }";"
+                continue
+            fi
+        fi
+        nameserversec=0
+        if [ "${entry%:*}" = "nameservers" ]; then
+            nameserversec=1
+            continue
+        fi
+    done < /etc/confluent/confluent.deploycfg
+    echo dns=$nameservers >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
+    dnsdomain=${dnsdomain#dnsdomain: }
+    echo dns-search=$dnsdomain >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+may-fail=false
+method=manual
+
+[ipv6]
+addr-gen-mode=eui64
+method=auto
+
+EOC
+elif [ "$auto6configmethod" = "static" ]; then
+    confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+    v6addr=$(grep ^ipv6_address: /etc/confluent/confluent.deploycfg)
+    v6addr=${v6addr#ipv6_address: }
+    v6gw=$(grep ^ipv6_gateway: /etc/confluent/confluent.deploycfg)
+    v6gw=${v6gw#ipv6_gateway: }
+    if [ "$v6gw" = "null" ]; then
+        v6gw=""
+    fi
+    v6nm=$(grep ^ipv6_prefix: /etc/confluent/confluent.deploycfg)
+    v6nm=${v6nm#ipv6_prefix: }
+    echo "Setting up $ifname as static at $v6addr/$v6nm"
+    ip addr add dev $ifname $v6addr/$v6nm
+
+    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+[ipv4]
+dhcp-timeout=90
+dhcp-vendor-class-identifier=anaconda-Linux
+method=disabled
+
+[ipv6]
+addr-gen-mode=eui64
+method=manual
+may-fail=false
+EOC
+    echo address1=$v6addr/$v6nm >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    if [ ! -z "$v6gw" ]; then
+        ip route add default via $v6gw
+        echo gateway=$v6gw >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    fi
+    nameserversec=0
+    nameservers=""
+    while read -r entry; do
+        if [ $nameserversec = 1 ]; then
+            if [[ $entry == "-"*:* ]]; then
+                nameservers="$nameservers"${entry#- }";"
+                continue
+            fi
+        fi
+        nameserversec=0
+        if [ "${entry%:*}" = "nameservers" ]; then
+            nameserversec=1
+            continue
+        fi
+    done < /etc/confluent/confluent.deploycfg
+    echo dns=$nameservers >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
+    dnsdomain=${dnsdomain#dnsdomain: }
+    echo dns-search=$dnsdomain >> /run/NetworkManager/system-connections/$ifname.nmconnection
+fi
+echo '[proxy]' >> /run/NetworkManager/system-connections/$ifname.nmconnection
+chmod 600 /run/NetworkManager/system-connections/*.nmconnection
+confluent_websrv=$confluent_mgr
+if [[ $confluent_websrv == *:* ]] && [[ $confluent_websrv != "["* ]]; then
+    confluent_websrv="[$confluent_websrv]"
+fi
+echo -n "Initializing ssh..."
+ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -C '' -N ''
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    privfile=${pubkey%.pub}
+    curl -sf -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d @$pubkey  https://$confluent_websrv/confluent-api/self/sshcert > $certfile
+    if [ -s $certfile ]; then
+        echo HostCertificate $certfile >> /etc/ssh/sshd_config
+    fi
+    echo HostKey $privfile >> /etc/ssh/sshd_config
+done
+/usr/sbin/sshd
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+confluent_proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+confluent_urls=""
+for addr in $(grep ^MANAGER: /etc/confluent/confluent.info|awk '{print $2}'|sed -e s/%/%25/); do
+    if [[ $addr == *:* ]]; then
+        confluent_urls="$confluent_urls $confluent_proto://[$addr]/confluent-public/os/$confluent_profile/rootimg.sfs"
+    else
+        confluent_urls="$confluent_urls $confluent_proto://$addr/confluent-public/os/$confluent_profile/rootimg.sfs"
+    fi
+done
+mkdir -p /etc/confluent
+curl -sf https://$confluent_websrv/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+. /etc/confluent/functions
+source_remote imageboot.sh

confluent_osdeploy/el10-diskless/profiles/default/scripts/add_local_repositories

@@ -0,0 +1 @@
+../../../../el9-diskless/profiles/default/scripts/add_local_repositories

confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.custom

@@ -0,0 +1 @@
+../../../../el9-diskless/profiles/default/scripts/firstboot.custom

confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.service

@@ -0,0 +1 @@
+../../../../el9-diskless/profiles/default/scripts/firstboot.service

confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.sh

@@ -0,0 +1 @@
+../../../../el9-diskless/profiles/default/scripts/firstboot.sh

confluent_osdeploy/el10-diskless/profiles/default/scripts/functions

@@ -0,0 +1 @@
+../../../../el9-diskless/profiles/default/scripts/functions

confluent_osdeploy/el10-diskless/profiles/default/scripts/getinstalldisk

@@ -0,0 +1 @@
+../../../../el9-diskless/profiles/default/scripts/getinstalldisk

confluent_osdeploy/el10-diskless/profiles/default/scripts/image2disk.py

@@ -0,0 +1 @@
+../../../../el9-diskless/profiles/default/scripts/image2disk.py

confluent_osdeploy/el10-diskless/profiles/default/scripts/imageboot.sh

@@ -0,0 +1,132 @@
+. /lib/dracut-lib.sh
+confluent_whost=$confluent_mgr
+if [[ "$confluent_whost" == *:* ]] && [[ "$confluent_whost" != "["* ]]; then
+    confluent_whost="[$confluent_mgr]"
+fi
+mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
+if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
+    mount -t tmpfs untethered /mnt/remoteimg
+    curl https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
+else
+    confluent_urls="$confluent_urls https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs"
+    /opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg
+fi
+/opt/confluent/bin/confluent_imginfo /mnt/remoteimg/rootimg.sfs > /tmp/rootimg.info
+loopdev=$(losetup -f)
+export mountsrc=$loopdev
+losetup -r $loopdev /mnt/remoteimg/rootimg.sfs
+if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then
+    while ! curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_whost/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key; do
+        echo "Unable to retrieve private key from $confluent_mgr (verify that confluent can access /var/lib/confluent/private/os/$confluent_profile/pending/rootimg.key)"
+        sleep 1
+    done
+    cipher=$(head -n 1 /tmp/rootimg.key)
+    key=$(tail -n 1 /tmp/rootimg.key)
+    len=$(wc -c /mnt/remoteimg/rootimg.sfs | awk '{print $1}')
+    len=$(((len-4096)/512))
+    dmsetup create cryptimg --table "0 $len crypt $cipher $key 0 $loopdev 8"
+    /opt/confluent/bin/confluent_imginfo /dev/mapper/cryptimg > /tmp/rootimg.info
+    mountsrc=/dev/mapper/cryptimg
+fi
+
+if grep '^Format: squashfs' /tmp/rootimg.info > /dev/null; then
+    mount -o ro $mountsrc /mnt/remote
+elif grep  '^Format: confluent_multisquash' /tmp/rootimg.info; then
+    tail -n +3 /tmp/rootimg.info  | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh
+    . /tmp/setupmount.sh
+    cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh
+    . /tmp/mountparts.sh
+fi
+
+
+#mount -t tmpfs overlay /mnt/overlay
+modprobe zram
+memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
+memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
+echo $memtot > /sys/block/zram0/disksize
+mkfs.xfs /dev/zram0 > /dev/null
+mount -o discard /dev/zram0 /mnt/overlay
+if [ ! -f /tmp/mountparts.sh ]; then
+    mkdir -p /mnt/overlay/upper /mnt/overlay/work
+    mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
+else
+    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
+        mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
+        mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
+    done
+fi
+mkdir -p /sysroot/etc/ssh
+mkdir -p /sysroot/etc/confluent
+mkdir -p /sysroot/root/.ssh
+cp /root/.ssh/* /sysroot/root/.ssh
+chmod 700 /sysroot/root/.ssh
+cp /etc/confluent/* /sysroot/etc/confluent/
+cp /etc/ssh/*key* /sysroot/etc/ssh/
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    privfile=${pubkey%.pub}
+    if [ -s $certfile ]; then
+        echo HostCertificate $certfile >> /sysroot/etc/ssh/sshd_config
+    fi
+    echo HostKey $privfile >> /sysroot/etc/ssh/sshd_config
+done
+
+mkdir -p /sysroot/dev /sysroot/sys /sysroot/proc /sysroot/run
+if [ ! -z "$autocons" ]; then
+    autocons=${autocons%,*}
+    mkdir -p /run/systemd/generator/getty.target.wants
+    ln -s /usr/lib/systemd/system/serial-getty@.service /run/systemd/generator/getty.target.wants/serial-getty@${autocons}.service
+fi
+while [ ! -e /sysroot/sbin/init ]; do
+    echo "Failed to access root filesystem or it is missing /sbin/init"
+    echo "System should be accessible through ssh at port 2222 with the appropriate key"
+    while [ ! -e /sysroot/sbin/init ]; do
+        sleep 1
+    done
+done
+rootpassword=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg)
+rootpassword=${rootpassword#rootpassword: }
+if [ "$rootpassword" = "null" ]; then
+    rootpassword=""
+fi
+
+if [ ! -z "$rootpassword" ]; then
+    sed -i "s@root:[^:]*:@root:$rootpassword:@" /sysroot/etc/shadow
+fi
+for i in /ssh/*.ca; do
+    echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
+done
+echo HostbasedAuthentication yes >> /sysroot/etc/ssh/sshd_config
+echo HostbasedUsesNameFromPacketOnly yes >> /sysroot/etc/ssh/sshd_config
+echo IgnoreRhosts no >> /sysroot/etc/ssh/sshd_config
+sshconf=/sysroot/etc/ssh/ssh_config
+if [ -d /sysroot/etc/ssh/ssh_config.d/ ]; then
+    sshconf=/sysroot/etc/ssh/ssh_config.d/01-confluent.conf
+fi
+echo 'Host *' >> $sshconf
+echo '    HostbasedAuthentication yes' >> $sshconf
+echo '    EnableSSHKeysign yes' >> $sshconf
+echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
+curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_whost/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
+cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
+chmod 640 /sysroot/etc/ssh/*_key
+cp /tls/*.pem /sysroot/etc/pki/ca-trust/source/anchors/
+chroot /sysroot/ update-ca-trust
+curl -sf https://$confluent_whost/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
+mkdir -p /sysroot/opt/confluent/bin
+curl -sf https://$confluent_whost/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh
+chmod +x /sysroot/opt/confluent/bin/onboot.sh
+cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin
+ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service
+cp /etc/confluent/functions /sysroot/etc/confluent/functions
+if grep installtodisk /proc/cmdline > /dev/null; then
+    . /etc/confluent/functions
+    run_remote installimage
+    exec reboot -f
+fi
+mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
+ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
+kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
+exec /opt/confluent/bin/start_root

confluent_osdeploy/el10-diskless/profiles/default/scripts/installimage

@@ -0,0 +1 @@
+../../../../el9-diskless/profiles/default/scripts/installimage

confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.custom

@@ -0,0 +1 @@
+../../../../el9-diskless/profiles/default/scripts/onboot.custom

confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.service

@@ -0,0 +1 @@
+../../../../el9-diskless/profiles/default/scripts/onboot.service

confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.sh

@@ -0,0 +1 @@
+../../../../el9-diskless/profiles/default/scripts/onboot.sh

confluent_osdeploy/el10-diskless/profiles/default/scripts/post.sh

@@ -0,0 +1 @@
+../../../../el9-diskless/profiles/default/scripts/post.sh

confluent_osdeploy/el10-diskless/profiles/default/scripts/syncfileclient

@@ -0,0 +1 @@
+../../../../el9-diskless/profiles/default/scripts/syncfileclient

confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.sh

@@ -5,11 +5,37 @@
 # noted below so custom commands are executed before
 # the script notifies confluent that install is fully complete.
 
+
+ntpsrvs=""
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 timedatectl set-timezone $(grep ^timezone: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+
+
+if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
+    for ntpsrv in $(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //'); do
+        echo "server ${ntpsrv} iburst " >> /tmp/timeservers
+    done
+fi
+
+if [ -f /tmp/timeservers ]; then
+
+ntpsrvs=$(cat /tmp/timeservers)
+
+sed -i "1,/^pool * /c\\
+
+${ntpsrvs//$'\n'/\\$'\n'}" /etc/chrony.conf
+
+
+systemctl restart chronyd
+
+rm -f /tmp/timeservers
+fi
+
+
+
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
 mkdir -p /var/log/confluent

confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh

@@ -5,6 +5,7 @@
 # noted below so custom commands are executed before
 # the script notifies confluent that install is fully complete.
 
+ntpsrvs=""
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 v4meth=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg|awk '{print $2}')
@@ -17,6 +18,33 @@ fi
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 timedatectl set-timezone $(grep ^timezone: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 hostnamectl set-hostname $nodename
+
+
+if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
+    for ntpsrv in $(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //'); do
+       echo "server ${ntpsrv} iburst " >> /tmp/timeservers
+    done
+fi
+
+if [ -f /tmp/timeservers ]; then
+
+ntpsrvs=$(cat /tmp/timeservers)
+
+sed -i "1,/^pool * /c\\
+${ntpsrvs//$'\n'/\\$'\n'}" /etc/chrony.conf
+
+
+systemctl restart chronyd
+
+rm -f /tmp/timeservers
+
+fi
+
+
+
+
+
+
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
 mkdir -p /var/log/confluent

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 echo -n "" >> /tmp/net.ifaces
 echo -n "" > /tmp/01-autocons.devnode
-cat /tls/*.0 >> /etc/pki/tls/certs/ca-bundle.crt
+BUNDLENAME=/etc/pki/tls/certs/ca-bundle.crt
+if [ ! -e "$BUNDLENAME" ]; then
+    BUNDLENAME=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
+    mkdir -p /etc/pki/tls/certs
+    ln -s $BUNDLENAME /etc/pki/tls/certs/ca-bundle.crt
+fi
+cat /tls/*.0 >> $BUNDLENAME
 if ! grep console= /proc/cmdline >& /dev/null; then
     autocons=$(/opt/confluent/bin/autocons)
     if [ -n "$autocons" ]; then

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 BUNDLENAME=/sysroot/etc/pki/tls/certs/ca-bundle.crt
+if [ ! -e "$BUNDLENAME" ]; then
+    BUNDLENAME=/sysroot/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
+fi
 while [ -h $BUNDLENAME ]; do
     BUNDLENAME=/sysroot/$(readlink $BUNDLENAME)
 done

confluent_osdeploy/el8/profiles/default/initprofile.sh

@@ -1,5 +1,12 @@
 #!/bin/sh
-sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/; s/oraclelinux/Oracle Linux/; s/alma/AlmaLinux/' $2/profile.yaml
+sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/; s/oraclelinux/Oracle Linux/; s/alma/AlmaLinux/;s/fedora/Fedora Linux/' $2/profile.yaml
+if grep Fedora $2/profile.yaml > /dev/null; then
+	sed -i 's/@^minimal-environment/#/' $2/packagelist
+fi
+if grep ^label: $2/profile.yaml | grep 10 > /dev/null; then
+	echo 'echo openssh-keysign >> /tmp/addonpackages' > $2/scripts/pre.d/enablekeysign
+	chmod 644 $2/scripts/pre.d/enablekeysign
+fi
 ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
 ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution
 mkdir -p $2/boot/efi/boot

confluent_osdeploy/el8/profiles/default/scripts/add_local_repositories

@@ -6,6 +6,7 @@ except ImportError:
 import importlib.util
 import importlib.machinery
 import sys
+import glob
 modloader = importlib.machinery.SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient')
 modspec = importlib.util.spec_from_file_location('apiclient', '/opt/confluent/bin/apiclient', loader=modloader)
 apiclient = importlib.util.module_from_spec(modspec)
@@ -41,6 +42,7 @@ try:
 except AttributeError:
     f = cStringIO.StringIO(cfgdata)
     c.readfp(f)
+gpgkeys = glob.glob('/etc/pki/rpm-gpg/RPM-GPG-KEY-*')
 for sec in c.sections():
     if sec.startswith('variant-'):
         try:
@@ -56,3 +58,6 @@ for sec in c.sections():
                 repopath = repopath[1:]
             repout.write('baseurl=https://{}/confluent-public/os/{}/distribution/{}\n'.format(server, profile, repopath))
             repout.write('enabled=1\n')
+            if gpgkeys:
+                gpgkeyvals = ['file://{}'.format(x) for x in gpgkeys]
+                repout.write('gpgkey=' + ' '.join(gpgkeyvals) + '\n')

confluent_osdeploy/el8/profiles/default/scripts/post.sh

@@ -34,6 +34,11 @@ run_remote_python add_local_repositories
 # run_remote_python will use the appropriate python interpreter path to run the specified script
 # A post.custom is provided to more conveniently hold customizations, see the post.custom file.
 
+# This will do some serial console fixup for bad grub configuration when serial is firmware
+# managed. See script for details
+run_remote_python autoconsole
+
+
 # This will induce server side processing of the syncfile contents if
 # present
 run_remote_python syncfileclient

confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -42,7 +42,9 @@ if ! grep console= /proc/cmdline >& /dev/null; then
     autocons=$(/opt/confluent/bin/autocons)
     autoconsdev=${autocons%,*}
     autocons=${autocons##*/}
-    echo "Automatic console configured for $autocons"
+    if [ ! -z "$autocons" ]; then
+        echo "Automatic console configured for $autocons"
+    fi
 fi
 echo "Initializing confluent diskless environment"
 echo -n "udevd: "
@@ -102,6 +104,14 @@ while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}'
         ip link set $i up
     done
     /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+    echo -n .
+done
+TRIES=0
+while ! grep ^NODENAME: /etc/confluent/confluent.info >& /dev/null && [ "$TRIES" -lt 300 ]; do
+    sleep 0.5
+    echo -n .
+    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+    TRIES=$((TRIES + 1))
 done
 cd /
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
@@ -292,7 +302,7 @@ if [[ $confluent_websrv == *:* ]] && [[ $confluent_websrv != "["* ]]; then
     confluent_websrv="[$confluent_websrv]"
 fi
 echo -n "Initializing ssh..."
-ssh-keygen -A
+ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -C '' -N ''
 for pubkey in /etc/ssh/ssh_host*key.pub; do
     certfile=${pubkey/.pub/-cert.pub}
     privfile=${pubkey%.pub}
@@ -302,7 +312,10 @@ for pubkey in /etc/ssh/ssh_host*key.pub; do
     fi
     echo HostKey $privfile >> /etc/ssh/sshd_config
 done
-/usr/sbin/sshd
+if grep "debugssh" /proc/cmdline > /dev/null; then
+    /usr/sbin/sshd
+fi
+echo "done"
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg| awk '{print $2}')
 confluent_proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg| awk '{print $2}')
 confluent_urls=""

confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py

@@ -373,7 +373,7 @@ def install_to_disk(imgpath):
         if fs['device'].startswith('/dev/mapper'):
             oldvgname = fs['device'].rsplit('/', 1)[-1]
             # if node has - then /dev/mapper will double up the hypen
-            if '_' in oldvgname and '-' in oldvgname.split('_')[-1]:
+            if '_' in oldvgname and '-' in oldvgname.split('_', 1)[-1]:
                 oldvgname = oldvgname.rsplit('-', 1)[0].replace('--', '-')
                 osname = oldvgname.split('_')[0]
                 nodename = socket.gethostname().split('.')[0]

confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh

@@ -4,7 +4,7 @@
 # completed. It is best to edit the middle of the file as
 # noted below so custom commands are executed before
 # the script notifies confluent that install is fully complete.
-
+ntpsrvs=""
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 v4meth=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg|awk '{print $2}')
@@ -17,6 +17,28 @@ fi
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 timedatectl set-timezone $(grep ^timezone: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 hostnamectl set-hostname $nodename
+
+
+if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
+    for ntpsrv in $(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //'); do
+        echo "server ${ntpsrv} iburst " >> /tmp/timeservers
+    done
+fi
+
+if [ -f /tmp/timeservers ]; then
+
+ntpsrvs=$(cat /tmp/timeservers)
+
+sed -i "1,/^pool * /c\\
+
+${ntpsrvs//$'\n'/\\$'\n'}" /etc/chrony.conf
+
+
+systemctl restart chronyd
+
+rm -f /tmp/timeservers
+fi
+
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
 mkdir -p /var/log/confluent

confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel

@@ -6,6 +6,7 @@ if [ ! -f /var/run/vmware/show-esx-shell-login ]; then
     chvt 2
     /etc/init.d/ESXShell start
 fi
+/opt/confluent/bin/apiclient -f
 uuid=$(vsish -e get /hardware/bios/dmiInfo|grep -A15 UUID|sed -e 's/.*://'|sed -e ':a;N;$!ba;s/\n//g' | sed -e 's/ *0x//g')
 uuid=${uuid:0:8}-${uuid:8:4}-${uuid:12:4}-${uuid:16:4}-${uuid:20:12}
 kargs=$(vsish -e get /system/bootCmdLine|grep "command line:")

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -1,5 +1,14 @@
 #!/bin/bash
+mkdir -p /usr/libexec /run/sshd
+if [ ! -x /usr/libexec/platform-python ]; then
+       ln -s /usr/bin/python3 /usr/libexec/platform-python
+fi
 export LANG=en_US.utf8
+
+(while :; do TERM=linux tmux a <> /dev/tty1 >&0 2>&1; done) &
+
+
+
 mkdir -p /etc/confluent
 if ! grep console= /proc/cmdline >& /dev/null; then
     autocons=$(/opt/confluent/bin/autocons)
@@ -9,13 +18,9 @@ if ! grep console= /proc/cmdline >& /dev/null; then
     if [ ! -z "$autocons" ]; then
         echo "Using $(cat /tmp/01-autocons.conf)"
         (while :; do TERM=xterm-256color tmux a <> $autocons >&0 2>&1; done) &
-        (while :; do TERM=linux tmux a <> /dev/tty1 >&0 2>&1; done) &
-    else
-        (while :; do TERM=linux tmux a <> /dev/console >&0 2>&1; done) &
     fi
 else
     (while :; do TERM=xterm-256color tmux a <> /dev/console >&0 2>&1; done) &
-    (while :; do TERM=linux tmux a <> /dev/tty1 >&0 2>&1; done) &
 fi
 (while :; do TERM=linux tmux <> /dev/tty2 >&0 2>&1; done) &
 echo -n "udevd: "
@@ -29,6 +34,13 @@ modprobe ib_umad
 modprobe hfi1
 modprobe mlx5_ib
 echo "done"
+if [ -x /usr/bin/seatd-launch -a -x /usr/bin/sway ]; then
+    export XDG_RUNTIME_DIR=/run/users/0
+    mkdir -p $XDG_RUNTIME_DIR
+    sed -i '/^output /d' /etc/sway/config
+    echo 'exec foot -t xterm -T Terminal tmux a' > /etc/sway/config.d/genesis
+    (while :; do seatd-launch sway <> /dev/tty1 >& /dev/null; done) &
+fi
 cat > /etc/ssh/sshd_config << EOF
 Port 22
 Port 3389
@@ -43,23 +55,48 @@ mkdir -p /etc/pki/tls/certs
 cat /tls/*.pem > /etc/pki/tls/certs/ca-bundle.crt
 TRIES=0
 touch /etc/confluent/confluent.info
+TRIES=5
+echo -n "Waitiing for disks..."
+while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
+    sleep 1
+    TRIES=$((TRIES - 1))
+done
+echo "Done"
 if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
     mkdir -p /media/ident
     mount /dev/disk/by-label/CNFLNT_IDNT /media/ident
     if [ -e /media/ident/genesis_bootstrap.sh ]; then
-        exec sh /media/ident/genesis_bootstrap.sh
+        exec bash /media/ident/genesis_bootstrap.sh
     fi
 fi
 if [ -e /dev/disk/by-label/GENESIS-X86 ]; then
     mkdir -p /media/genesis
     mount /dev/disk/by-label/GENESIS-X86 /media/genesis
     if [ -e /media/genesis/genesis_bootstrap.sh ]; then
-        exec sh /media/genesis/genesis_bootstrap.sh
+        exec bash /media/genesis/genesis_bootstrap.sh
     fi
 fi
 cd /sys/class/net
 echo -n "Scanning for network configuration..."
 while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 30 ]; do
+    if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+	echo ""
+	echo "Media bootstrap detected, switching to media based boot"
+        mkdir -p /media/ident
+        mount /dev/disk/by-label/CNFLNT_IDNT /media/ident
+        if [ -e /media/ident/genesis_bootstrap.sh ]; then
+            exec bash /media/ident/genesis_bootstrap.sh
+        fi
+    fi
+    if [ -e /dev/disk/by-label/GENESIS-X86 ]; then
+	echo ""
+	echo "Media bootstrap detected, switching to media based boot"
+        mkdir -p /media/genesis
+        mount /dev/disk/by-label/GENESIS-X86 /media/genesis
+        if [ -e /media/genesis/genesis_bootstrap.sh ]; then
+            exec bash /media/genesis/genesis_bootstrap.sh
+        fi
+    fi
     TRIES=$((TRIES + 1))
     for i in *; do
         ip link set $i up

confluent_osdeploy/genesis/initramfs/usr/lib/dracut/hooks/cmdline/10-genesis.sh

@@ -2,15 +2,17 @@ root=1
 rootok=1
 netroot=genesis
 clear
+mount -t cgroup2 cgroup2 /sys/fs/cgroup
+mount -t efivarfs efivarfs /sys/firmware/efi/efivars
 echo PS1="'"'[genesis running on \H \w]$ '"'" >> ~/.bashrc
 echo PS1="'"'[genesis running on \H \w]$ '"'" >> ~/.bash_profile
 mkdir -p /etc/ssh
 mkdir -p /var/tmp/
 mkdir -p /var/empty/sshd
-sed -i '/^root:x/d' /etc/passwd
+sed -i '/^root:/d' /etc/passwd
 echo root:x:0:0::/:/bin/bash >> /etc/passwd
 echo sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin >> /etc/passwd
-tmux new-session -d sh /opt/confluent/bin/rungenesis
+tmux new-session -d bash /opt/confluent/bin/rungenesis
 while :; do
 	sleep 86400
 done

confluent_osdeploy/genesis/profiles/default/scripts/configbmc

@@ -210,16 +210,28 @@ def set_port_tsm(s, port, model):
 
 
 def set_port_xcc(s, port, model):
+    if '_' in port:
+        port_type = port.split('_')[0]
+        port_number = port.split('_')[1]  # this could be the number or the form sfp28 || rj45
+        port = port_type
     oport = port
     if port.lower() == 'dedicated':
         port = b'\x01'
     elif port.lower() in ('ml2', 'ocp'):
         port = b'\x02\x00'
-    elif port.lower() == 'lom':
-        if model == '7x58':
-            port = b'\x00\x02'
+    elif port.lower() == 'lom':  # potentially have to change to port_form.lower() == 'sfp28 || or rj45  // if it is two port sfp28 it is usually the first port that is ncsi enabled'
+        if port_number == '1' or port_number.lower() == 'sfp28':
+            port = b'\x05\x00'
+        elif port_number == '3' or port_number.lower() == 'rj45':
+            if model == '7x58':
+                port = b'\x00\x02'
+            else:
+                port = b'\x00\x00'
         else:
-            port = b'\x00\x00'
+            if model == '7x58':
+                port = b'\x00\x02'
+            else:
+                port = b'\x00\x00'
     else:
         port = port.split(' ')
         port = bytes(bytearray([int(x) for x in port]))

confluent_osdeploy/genesis/profiles/default/scripts/onboot.sh

@@ -10,6 +10,9 @@
 # present
 run_remote_python syncfileclient
 
+run_remote_parts onboot.d
+
+
 # Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/
 run_remote_config onboot
 

confluent_osdeploy/ubuntu18.04/initramfs/lib/debian-installer-startup.d/S25confluentinit

@@ -55,25 +55,28 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
         if [ ! -z "$v4gw" ]; then
             setdebopt netcfg/get_gateway $v4gw string
         fi
-        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
-            ip addr add dev $NICGUESS $v4addr
-            if [ ! -z "$v4gw" ]; then
-                ip route add default via $v4gw
-            fi
-            for dsrv in $deploysrvs; do
-                if wget https://$dsrv/confluent-public/ --tries=1 --timeout=1 -O /dev/null > /dev/null 2>&1; then
-                    deploysrvs=$dsrv
-                    NIC=$NICGUESS
-                    setdebopt netcfg/choose_interface $NIC select
+        NIC=""
+        while [ -z "$NIC" ]; do
+            for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+                ip addr add dev $NICGUESS $v4addr
+                if [ ! -z "$v4gw" ]; then
+                    ip route add default via $v4gw
+                fi
+                for dsrv in $deploysrvs; do
+                    if wget https://$dsrv/confluent-public/ --tries=1 --timeout=1 -O /dev/null > /dev/null 2>&1; then
+                        deploysrvs=$dsrv
+                        NIC=$NICGUESS
+                        setdebopt netcfg/choose_interface $NIC select
+                        break
+                    fi
+                done
+                if [ -z "$NIC" ]; then
+                    ip -4 a flush dev $NICGUESS
+                else
                     break
                 fi
             done
-            if [ -z "$NIC" ]; then
-                ip -4 a flush dev $NICGUESS
-            else
-                break
-            fi
-        done 
+        done
         #TODO: nameservers
     elif [ "$v4cfgmeth" = "dhcp" ]; then
         setdebopt netcfg/disable_dhcp false boolean

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/pre.sh

@@ -85,3 +85,4 @@ wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apike
 #curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /tmp/getinstalldisk
 #python3 /tmp/getinstalldisk
 #sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml
+umount /media

confluent_osdeploy/ubuntu20.04/initramfs/scripts/init-premount/confluent

@@ -1,75 +1,79 @@
 cd /sys/class/net
+cp /tls/* /etc/ssl/certs/
 for nic in *; do
     ip link set $nic up
 done
 mkdir -p /custom-installation
 cp -a /opt/confluent /custom-installation
 touch /custom-installation/confluent/confluent.info
-TRIES=5
-while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
-    sleep 2
-    TRIES=$((TRIES - 1))
-done
-if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
-    tmnt=/tmp/idntmnt
-    mkdir -p /tmp/identdata/
-    mkdir -p $tmnt
-    tcfg=/tmp/idnttmp
-    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
-    cp -a $tmnt/* /tmp/identdata/
-    cd $tmnt
-    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
-    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
-    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
-    autoconfigmethod=${autoconfigmethod#ipv4_method: }
-    . /scripts/functions
-    if [ "$autoconfigmethod" = "static" ]; then
-        MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
-        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
-        MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
-        if [ "$MYGW" = "null" ]; then
-            MYGW=""
-        fi
-        MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
-        NIC=""
-        while [ -z "$NIC" ]; do
-            for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
-                ip addr add dev $NICGUESS $v4addr
-                if [ ! -z "$MYGW" ]; then
-                    ip route add default via $MYGW
-                fi
-                for dsrv in $deploysrvs; do
-                    if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
-                        deploysrvs=$dsrv
-                        NIC=$NICGUESS
+while ! grep NODENAME /custom-installation/confluent/confluent.info; do
+    if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+        tmnt=/tmp/idntmnt
+        mkdir -p /tmp/identdata/
+        mkdir -p $tmnt
+        tcfg=/tmp/idnttmp
+        mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+        cp -a $tmnt/* /tmp/identdata/
+        cd $tmnt
+        deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+        sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+        autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+        autoconfigmethod=${autoconfigmethod#ipv4_method: }
+        . /scripts/functions
+        if [ "$autoconfigmethod" = "static" ]; then
+            MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
+            v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+            MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
+            if [ "$MYGW" = "null" ]; then
+                MYGW=""
+            fi
+            MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
+            NIC=""
+            while [ -z "$NIC" ]; do
+                for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+                    ip addr add dev $NICGUESS $v4addr
+                    if [ ! -z "$MYGW" ]; then
+                        ip route add default via $MYGW
+                    fi
+                    for dsrv in $deploysrvs; do
+                        if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                            deploysrvs=$dsrv
+                            NIC=$NICGUESS
+                            break
+                        fi
+                    done
+                    if [ -z "$NIC" ]; then
+                        ip -4 a flush dev $NICGUESS
+                    else
                         break
                     fi
                 done
-                if [ -z "$NIC" ]; then
-                    ip -4 a flush dev $NICGUESS
-                else
+            done
+            ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
+            echo $NIC > /tmp/autodetectnic
+        else
+            configure_networking
+            for dsrv in $deploysrvs; do
+                if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                    deploysrvs=$dsrv
                     break
                 fi
             done
-        done
-        ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
-        echo $NIC > /tmp/autodetectnic
+        fi
+        MGR=$deploysrvs
+        NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
+        echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
+        echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
+        echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
+        hmackeyfile=/tmp/cnflnthmackeytmp
+        echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+        cd -
+        umount $tmnt
     else
-        configure_networking
-    fi
-    MGR=$deploysrvs
-    NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
-    echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
-    echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
-    echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
-    hmackeyfile=/tmp/cnflnthmackeytmp
-    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
-    cd -
-    umount $tmnt
-else
-    while ! grep NODENAME /custom-installation/confluent/confluent.info; do
         /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
-    done
+    fi
+done
+if [ -z "$MGR" ]; then
     MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
 fi
 osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline)
@@ -82,8 +86,25 @@ if [ ! -z "$cons" ]; then
 fi
 echo "Preparing to deploy $osprofile from $MGR"
 echo $osprofile > /custom-installation/confluent/osprofile
-echo URL=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf
-fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso"
+mv /usr/bin/openssl /usr/bin/ossl
+cat > /usr/bin/openssl << 'EOF'
+#!/bin/sh
+AMENDARGS=0
+nargs=""
+for arg in $*; do
+    if [ "$arg" == "-servername"  -o "$arg" == "-verify" ]; then
+        AMENDARGS=1
+    fi
+    if [ "$AMENDARGS" == "1" ]; then
+        arg=$(echo $arg|sed -e 's/:443$//' -e 's/\[//' -e 's/\]//' -e 's/%.*//')
+    fi
+    nargs="$nargs $arg"
+done
+exec /usr/bin/ossl $nargs
+EOF
+chmod +x /usr/bin/openssl
+echo URL=https://${MGR}:443/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf
+fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=https://${MGR}:443/confluent-public/os/$osprofile/distribution/install.iso"
 if [ ! -z "$cons" ]; then
     fcmdline="$fcmdline console=${cons#/dev/}"
 fi

confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent

@@ -28,7 +28,15 @@ if [ -e /tmp/cnflnthmackeytmp ]; then
     DEVICE=$(cat /tmp/autodetectnic)
     IP=done
 else
+    APIKEY=
     chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
+    APIKEY=$(cat /root/custom-installation/confluent/confluent.apikey)
+    while [ -z "$APIKEY" ]; do
+	echo "Failure trying to get confluent node token registered, check nodedeploy status, retrying in 5 seconds..."
+        sleep 5
+        chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
+        APIKEY=$(cat /root/custom-installation/confluent/confluent.apikey)
+    done
     MGR=[$MGR]
     nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
     nic=$(ip link |grep ^$nic:|awk '{print $2}')

confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent

@@ -1,3 +1,4 @@
+cp /tls/* /etc/ssl/certs/
 cd /sys/class/net
 for nic in *; do
     ip link set $nic up
@@ -5,71 +6,75 @@ done
 mkdir -p /custom-installation
 cp -a /opt/confluent /custom-installation
 touch /custom-installation/confluent/confluent.info
-TRIES=5
-while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
-    sleep 2
-    TRIES=$((TRIES - 1))
-done
-if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
-    tmnt=/tmp/idntmnt
-    mkdir -p /tmp/identdata/
-    mkdir -p $tmnt
-    tcfg=/tmp/idnttmp
-    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
-    cp -a $tmnt/* /tmp/identdata/
-    cd $tmnt
-    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
-    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
-    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
-    autoconfigmethod=${autoconfigmethod#ipv4_method: }
-    . /scripts/functions
-    if [ "$autoconfigmethod" = "static" ]; then
-        MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
-        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
-        MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
-        if [ "$MYGW" = "null" ]; then
-            MYGW=""
-        fi
-        MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
-        NIC=""
-        while [ -z "$NIC" ]; do
-            for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
-                ip addr add dev $NICGUESS $v4addr
-                if [ ! -z "$MYGW" ]; then
-                    ip route add default via $MYGW
-                fi
-                for dsrv in $deploysrvs; do
-                    if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
-                        deploysrvs=$dsrv
-                        NIC=$NICGUESS
+MGR=""
+while ! grep NODENAME /custom-installation/confluent/confluent.info; do
+    if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+        tmnt=/tmp/idntmnt
+        mkdir -p /tmp/identdata/
+        mkdir -p $tmnt
+        tcfg=/tmp/idnttmp
+        mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+        cp -a $tmnt/* /tmp/identdata/
+        cd $tmnt
+        deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+        sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+        autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+        autoconfigmethod=${autoconfigmethod#ipv4_method: }
+        . /scripts/functions
+        if [ "$autoconfigmethod" = "static" ]; then
+            MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
+            v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+            MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
+            if [ "$MYGW" = "null" ]; then
+                MYGW=""
+            fi
+            MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
+            NIC=""
+            while [ -z "$NIC" ]; do
+                for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+                    ip addr add dev $NICGUESS $v4addr
+                    if [ ! -z "$MYGW" ]; then
+                        ip route add default via $MYGW
+                    fi
+                    for dsrv in $deploysrvs; do
+                        if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                            deploysrvs=$dsrv
+                            NIC=$NICGUESS
+                            break
+                        fi
+                    done
+                    if [ -z "$NIC" ]; then
+                        ip -4 a flush dev $NICGUESS
+                    else
                         break
                     fi
                 done
-                if [ -z "$NIC" ]; then
-                    ip -4 a flush dev $NICGUESS
-                else
+            done
+            ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
+            echo $NIC > /tmp/autodetectnic
+        else
+            configure_networking
+            for dsrv in $deploysrvs; do
+                if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                    deploysrvs=$dsrv
                     break
                 fi
             done
-        done
-        ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
-        echo $NIC > /tmp/autodetectnic
+        fi
+        MGR=$deploysrvs
+        NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
+        echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
+        echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
+        echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
+        hmackeyfile=/tmp/cnflnthmackeytmp
+        echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+        cd -
+        umount $tmnt
     else
-        configure_networking
-    fi
-    MGR=$deploysrvs
-    NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
-    echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
-    echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
-    echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
-    hmackeyfile=/tmp/cnflnthmackeytmp
-    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
-    cd -
-    umount $tmnt
-else
-    while ! grep NODENAME /custom-installation/confluent/confluent.info; do
         /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
-    done
+    fi
+done
+if [ -z "$MGR" ]; then
     MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
 fi
 osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline)
@@ -85,8 +90,25 @@ echo $osprofile > /custom-installation/confluent/osprofile
 . /etc/os-release
 DIRECTISO=$(blkid -t TYPE=iso9660 |grep -Ei ' LABEL="Ubuntu-Server '$VERSION_ID)
 if [ -z "$DIRECTISO" ]; then
-    echo URL=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf
-    fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso"
+    mv /usr/bin/openssl /usr/bin/ossl
+    cat > /usr/bin/openssl << 'EOF'
+#!/bin/sh
+AMENDARGS=0
+nargs=""
+for arg in $*; do
+    if [ "$arg" == "-servername" -o "$arg" == "-verify" ]; then
+        AMENDARGS=1
+    fi
+    if [ "$AMENDARGS" == "1" ]; then
+        arg=$(echo $arg|sed -e 's/:443$//' -e 's/\[//' -e 's/\]//' -e 's/%.*//')
+    fi
+    nargs="$nargs $arg"
+done
+exec /usr/bin/ossl $nargs
+EOF
+    chmod +x /usr/bin/openssl
+    echo URL=https://${MGR}:443/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf
+    fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=https://${MGR}:443/confluent-public/os/$osprofile/distribution/install.iso"
 fi
 if [ ! -z "$cons" ]; then
     fcmdline="$fcmdline console=${cons#/dev/}"

confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data

@@ -1,5 +1,13 @@
 #cloud-config
 autoinstall:
+#  The following can help an Ubuntu system skip install-time updates
+#  Only uncomment if you know you really want to do this or plan to manage the updates
+#  a different way.
+#
+#  updates: security
+#  apt:
+#    disable_suites: [security]
+#    fallback: offline-install
   version: 1
   early-commands:
     - /custom-installation/pre.sh

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh

@@ -84,6 +84,10 @@ cat /target/etc/confluent/tls/*.pem > /target/etc/confluent/ca.pem
 cat /target/etc/confluent/tls/*.pem > /target/usr/local/share/ca-certificates/confluent.crt
 cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem
 chroot /target update-ca-certificates
+
+# Ubuntu mangles grub function for serial users, undo that mangling
+chroot /target bash -c "source /etc/confluent/functions; run_remote_python autoconsole"
+
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python confignet"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d"

confluent_osdeploy/utils/clortho.c

@@ -246,6 +246,6 @@ int main(int argc, char* argv[]) {
             buffer[0] = 255;
             ret = read(sock, buffer, 2);
         }
-        fprintf(stderr, "Password was not accepted\n");
+        fprintf(stderr, "Confluent API token grant denied by server\n");
         exit(1);
 }

confluent_osdeploy/utils/confusebox/Makefile

@@ -0,0 +1,8 @@
+all: confusebox
+
+confusebox: *.go
+	go build -ldflags "-w -s" -gcflags=all="-l" -trimpath
+	upx --brute confusebox
+
+clean:
+	rm confusebox

confluent_osdeploy/utils/confusebox/apiclient.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"net/http"
+	"crypto/x509"
+	"crypto/tls"
+	"strings"
+	"errors"
+)
+type ApiClient struct {
+	server string
+	urlserver string
+	apikey string
+	nodename string
+	webclient *http.Client
+}
+
+func NewApiClient(cafile string, keyfile string, nodename string, server string) (*ApiClient, error) {
+	currcacerts, err := os.ReadFile(cafile)
+	if err != nil {
+		return nil, err
+	}
+	cacerts  := x509.NewCertPool()
+	cacerts.AppendCertsFromPEM(currcacerts)
+	apikey := []byte("")
+	if keyfile != "" {
+		apikey, err = os.ReadFile(keyfile)
+		if err != nil {
+			return nil, err
+		}
+		if apikey[len(apikey) - 1] == 0xa {
+			apikey = apikey[:len(apikey)-1]
+		}
+	}
+	if nodename == "" {
+		cinfo, err := os.ReadFile("/etc/confluent/confliuent.info")
+		if err != nil {
+			nodename, err = os.Hostname()
+			if err != nil { return nil, err }
+		}
+		cinfolines := bytes.Split(cinfo, []byte("\n"))
+		if bytes.Contains(cinfolines[0], []byte("NODENAME")) {
+			cnodebytes := bytes.Split(cinfolines[0], []byte(" "))
+			nodename = string(cnodebytes[0])
+		}
+	}
+	urlserver := server
+	if strings.Contains(server, ":") {
+		if strings.Contains(server, "%") && !strings.Contains(server, "%25") {
+			server = strings.Replace(server, "%", "%25", 1)
+		}
+		urlserver = fmt.Sprintf("[%s]", server)
+		if strings.Contains(server, "%") {
+			server = server[:strings.Index(server, "%")]
+		}
+	}
+	webclient := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: cacerts,
+				ServerName: server,
+			},
+		},
+	}
+	vc := ApiClient{server, urlserver, string(apikey), nodename, webclient}
+	return &vc, nil
+}
+
+func (apiclient *ApiClient) RegisterKey(crypted string, hmac string) (error) {
+	cryptbytes := []byte(crypted)
+	cryptbuffer := bytes.NewBuffer(cryptbytes)
+	_, err := apiclient.request("/confluent-api/self/registerapikey", "", cryptbuffer, "", hmac)
+	return err
+}
+
+func (apiclient *ApiClient) Fetch(url string, outputfile string, mime string, body io.Reader) (error)  {
+	outp, err := os.Create(outputfile)
+	if err != nil { return err }
+	defer outp.Close()
+	rsp, err := apiclient.request(url, mime, body, "", "")
+	if err != nil { return err }
+	_, err = io.Copy(outp, rsp)
+	return err
+}
+
+func (apiclient *ApiClient) GrabText(url string, mime string, body io.Reader) (string, error){
+	rsp, err := apiclient.request(url, mime, body, "", "")
+	if err != nil { return "", err }
+	rspdata, err := io.ReadAll(rsp)
+	if err != nil { return "", err }
+	rsptxt := string(rspdata)
+	return rsptxt, nil
+}
+
+func (apiclient *ApiClient) request(url string, mime string, body io.Reader, method string, hmac string) (io.ReadCloser, error) {
+	if ! strings.Contains(url, "https://") {
+		url = fmt.Sprintf("https://%s%s", apiclient.urlserver, url)
+	}
+	if method == "" {
+		if body != nil {
+			method = http.MethodPost
+		} else {
+			method = http.MethodGet
+		}
+	}
+	var err error
+	var rq *http.Request
+	if body == nil {
+		rq, err = http.NewRequest(method, url, nil)
+	} else {
+		rq, err = http.NewRequest(method, url, body)
+	}
+	if err != nil { return nil, err }
+	if (mime != "") { rq.Header.Set("Accept", mime) }
+	rq.Header.Set("CONFLUENT_NODENAME", apiclient.nodename)
+	if len(hmac) > 0 {
+		rq.Header.Set("CONFLUENT_CRYPTHMAC", hmac)
+	} else {
+
+		rq.Header.Set("CONFLUENT_APIKEY", apiclient.apikey)
+	}
+	rsp, err := apiclient.webclient.Do(rq)
+	if err != nil { return nil, err }
+	if rsp.StatusCode >= 300 {
+		err = errors.New(rsp.Status)
+		return nil, err
+	}
+	return rsp.Body, err
+}
+

confluent_osdeploy/utils/confusebox/genpasshmac.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"bytes"
+	"github.com/go-crypt/crypt/algorithm/shacrypt"
+	"os"
+	"crypto/rand"
+	"encoding/base64"
+	"crypto/hmac"
+	"crypto/sha256"
+)
+
+func genpasshmac(hmackeyfile string) (string, string, string, error)  {
+	randbytes := make([]byte, 36)
+	_, err := rand.Read(randbytes)
+	if err != nil {
+		panic(err)
+	}
+	password := base64.StdEncoding.EncodeToString(randbytes)
+	hasher, err := shacrypt.New(shacrypt.WithVariant(shacrypt.VariantSHA256), shacrypt.WithIterations(5000))
+	if err != nil {
+		panic(err)
+	}
+
+	digest, err := hasher.Hash(password)
+	if err != nil {
+		panic(err)
+	}
+	cryptpass := digest.Encode()
+	hmackey, err := os.ReadFile(hmackeyfile)
+	if err != nil { return "", "", "", err }
+	keylines := bytes.Split(hmackey, []byte("\n"))
+	if bytes.Contains(keylines[0], []byte("apitoken:")) {
+		keyparts := bytes.Split(keylines[0], []byte(" "))
+		hmackey = keyparts[1]
+	}
+
+	hmacer := hmac.New(sha256.New, hmackey)
+	hmacer.Write([]byte(cryptpass))
+	hmacresult := hmacer.Sum(nil)
+	hmacout := base64.StdEncoding.EncodeToString(hmacresult)
+	return password, cryptpass, hmacout, nil
+}
+

confluent_osdeploy/utils/confusebox/go.mod

@@ -0,0 +1,7 @@
+module confusebox
+
+go 1.23.6
+
+require github.com/go-crypt/crypt v0.3.3
+
+require github.com/go-crypt/x v0.3.4 // indirect

confluent_osdeploy/utils/confusebox/go.sum

@@ -0,0 +1,4 @@
+github.com/go-crypt/crypt v0.3.3 h1:mBSh8U+vwDm3V+UHNMQqsxV0clzlvKbLcJXcafYFpCs=
+github.com/go-crypt/crypt v0.3.3/go.mod h1:ex5C1b58/tzCW6/rJfcdf5Y2TjgzmWVtX57sjpN3pUQ=
+github.com/go-crypt/x v0.3.4 h1:zgpaI55VOAbkkRup9+tLaZ02IWTV/xz63tohoY0t9+Y=
+github.com/go-crypt/x v0.3.4/go.mod h1:+uHWqfzD3S6YWxm18/Qp+4VcuBb0Le9dGUhX0zaWicU=

confluent_osdeploy/utils/confusebox/main.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"os"
+	"io"
+	"fmt"
+)
+
+func get_confluent_server() (string, error) {
+	var confluentsrv string
+	dcfg, err := os.ReadFile("/etc/confluent/confluent.deploycfg")
+	if err == nil {
+		dcfglines := bytes.Split(dcfg, []byte("\n"))
+		for _, dcfgline := range(dcfglines) {
+			dkeyval := bytes.Split(dcfgline, []byte(" "))
+			if bytes.Contains(dkeyval[0], []byte("deploy_server")) && (bytes.Contains(dkeyval[1], []byte(".")) || bytes.Contains(dkeyval[1], []byte(":"))) {
+				confluentsrv = string(dkeyval[1])
+				return confluentsrv, nil
+			}
+		}
+	} else {
+		_, err := os.ReadFile("/etc/confluent/confluent.info")
+		if err != nil {
+			return "Unable to determine Confluent server", err
+		}
+	}
+	return "", err
+}
+func main() {
+	var nodename string
+	var cacerts string
+	var apikey string
+	var usejson bool
+	var confluentsrv string
+	hmacreg := flag.NewFlagSet("hmacregister", flag.ExitOnError)
+	hmacreg.StringVar(&apikey, "k", "/etc/confluent/apikey", "Output file for the api key")
+	hmacKey := hmacreg.String("i", "", "Identity yaml file")
+	hmacreg.StringVar(&cacerts, "c", "/etc/confluent/ca.pem", "Certeficate authorities to use in PEM")
+	hmacreg.StringVar(&nodename, "n", "", "Node name")
+	hmacreg.StringVar(&confluentsrv, "s", "", "Confluent server to request from")
+
+	invokeapi := flag.NewFlagSet("invoke", flag.ExitOnError)
+	invokeapi.StringVar(&nodename, "n", "", "Node name")
+
+	invokeapi.StringVar(&cacerts, "c", "/etc/confluent/ca.pem", "Certeficate authorities to use in PEM")
+	invokeapi.StringVar(&apikey, "k", "/etc/confluent/confluent.apikey", "File containing Confluent API key")
+	invokeapi.BoolVar(&usejson, "j", false, "Request JSON formatted reply")
+	outputfile := invokeapi.String("o", "", "Filename to store download to")
+	invokeapi.StringVar(&confluentsrv, "s", "", "Confluent server to request from")
+	invokedata := invokeapi.String("d", "", "Data to submit")
+	invokedatafile := invokeapi.String("i", "", "File containing data to submit")
+
+
+
+	if len(os.Args) < 2 {
+		panic("Insufficient arguments, no subcommand")
+	}
+	switch os.Args[1] {
+		case "hmacregister":
+			var err error
+			hmacreg.Parse(os.Args[2:])
+			if confluentsrv == "" {
+				confluentsrv, err = get_confluent_server()
+			}
+			password, crypted, hmac, err := genpasshmac(*hmacKey)
+			if err != nil { panic(err) }
+			//apiclient(cacerts, "/confluent-api/self/registerapikey", apikey, nodename, usejson)
+			apiclient, err := NewApiClient(cacerts, "", nodename, confluentsrv)
+			if err != nil { panic(err) }
+			err = apiclient.RegisterKey(crypted, hmac)
+			if err != nil { panic(err) }
+			outp, err := os.Create(apikey)
+			if err != nil { panic(err) }
+			defer outp.Close()
+			outp.Write([]byte(password))
+		case "invoke":
+			var err error
+			var body io.Reader
+			body = nil
+			invokeapi.Parse(os.Args[2:])
+			if *invokedata != "" {
+				body = bytes.NewBuffer([]byte(*invokedata))
+			}
+			if *invokedatafile != "" {
+				body, err = os.Open(*invokedatafile)
+				if err != nil { panic(err) }
+			}
+			if confluentsrv == "" {
+				confluentsrv, err = get_confluent_server()
+			}
+			apiclient, err := NewApiClient(cacerts, apikey, nodename, confluentsrv)
+			if err != nil { panic(err) }
+			mime := ""
+			if usejson {
+				mime = "application/json"
+			}
+			if *outputfile != "" {
+				err := apiclient.Fetch(invokeapi.Arg(0), *outputfile, mime, body)
+				if err != nil { panic(err) }
+			} else {
+				rsp, err := apiclient.GrabText(invokeapi.Arg(0), mime, body)
+				if err != nil { panic(err) }
+				fmt.Println(rsp)
+			}
+		default:
+			panic("Unrecognized subcommand")
+	}
+}

confluent_osdeploy/utils/goapiclient/go.mod

@@ -0,0 +1,5 @@
+module confluentapiclient
+
+go 1.22
+
+toolchain go1.23.6

confluent_osdeploy/utils/goapiclient/go.sum

@@ -0,0 +1,4 @@
+github.com/go-crypt/crypt v0.3.2 h1:I4i0u2g8X9bxCXIjvv19BDVXqQbddDQrURCJrOyyJos=
+github.com/go-crypt/crypt v0.3.2/go.mod h1:U0YhpCizEtaVC4gVfUUN0qGn1Z6+e3at+B5uLYx/sV0=
+github.com/go-crypt/x v0.3.2 h1:m2wn2+8tp28V4yDiW5NSTiyNSXnCoTs1R1+H+cAJA3M=
+github.com/go-crypt/x v0.3.2/go.mod h1:uelN9rbD2e2eqE8KA26B9R6OQ0TdM6msWdPsoMM1ZFk=

confluent_osdeploy/utils/goapiclient/main.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"net/http"
+	"crypto/x509"
+	"crypto/tls"
+)
+
+func main() {
+	certauthority := flag.String("c", "/etc/confluent/ca.pem", "Certificate authorities to use, in PEM format")
+	targurl := flag.String("u", "", "Url to connect to")
+	keyfile := flag.String("k", "/etc/confluent/confluent.apikey", "Confluent API key file")
+	nodename := flag.String("n", "", "Node Name")
+	usejson := flag.Bool("j", false, "Use JSON")
+	flag.Parse()
+	certpool := x509.NewCertPool()
+	currcacerts, err := os.ReadFile(*certauthority)
+	if err != nil {
+		panic(err)
+	}
+	confluentapikey, err := os.ReadFile(*keyfile)
+	if confluentapikey[len(confluentapikey) - 1] == 0xa {
+		confluentapikey = confluentapikey[:len(confluentapikey)-1]
+	}
+	if err != nil {
+		panic(err)
+	}
+	certpool.AppendCertsFromPEM(currcacerts)
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: certpool,
+			},
+		},
+	}
+	rq, err := http.NewRequest(http.MethodGet, *targurl, nil)
+	if err != nil { panic(err )}
+	if *usejson { rq.Header.Set("Accept", "application/json") }
+	if *nodename == "" {
+		*nodename, err = os.Hostname()
+	}
+	rq.Header.Set("CONFLUENT_NODENAME", *nodename)
+	fmt.Println(string(confluentapikey))
+	rq.Header.Set("CONFLUENT_APIKEY", string(confluentapikey))
+	if err != nil { panic(err )}
+	rsp, err := client.Do(rq)
+	if err != nil { panic(err )}
+	rspdata, err := io.ReadAll(rsp.Body)
+	rsptxt := string(rspdata)
+	fmt.Println(rsptxt)
+}
+

confluent_osdeploy/utils/gopasshmac/go.mod

@@ -0,0 +1,10 @@
+module genpasshmac
+
+go 1.22
+
+toolchain go1.23.6
+
+require (
+	github.com/go-crypt/crypt v0.3.2 // indirect
+	github.com/go-crypt/x v0.3.2 // indirect
+)

confluent_osdeploy/utils/gopasshmac/go.sum

@@ -0,0 +1,4 @@
+github.com/go-crypt/crypt v0.3.2 h1:I4i0u2g8X9bxCXIjvv19BDVXqQbddDQrURCJrOyyJos=
+github.com/go-crypt/crypt v0.3.2/go.mod h1:U0YhpCizEtaVC4gVfUUN0qGn1Z6+e3at+B5uLYx/sV0=
+github.com/go-crypt/x v0.3.2 h1:m2wn2+8tp28V4yDiW5NSTiyNSXnCoTs1R1+H+cAJA3M=
+github.com/go-crypt/x v0.3.2/go.mod h1:uelN9rbD2e2eqE8KA26B9R6OQ0TdM6msWdPsoMM1ZFk=

confluent_osdeploy/utils/gopasshmac/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"flag"
+	//"fmt"
+	"github.com/go-crypt/crypt/algorithm/shacrypt"
+	"os"
+	"crypto/rand"
+	"encoding/base64"
+	"crypto/hmac"
+	"crypto/sha256"
+)
+
+func main() {
+	hmackeyfile := flag.String("k", "", "Key file for HMAC calculation")
+	passfile := flag.String("p", "", "File to write generated password to")
+	cryptfile := flag.String("c", "", "File to write crypted form of key to")
+	hmacfile := flag.String("m", "", "File to write HMAC value to")
+	flag.Parse()
+	randbytes := make([]byte, 36)
+	_, err := rand.Read(randbytes)
+	if err != nil {
+		panic(err)
+	}
+	newpasswd := base64.StdEncoding.EncodeToString(randbytes)
+	hasher, err := shacrypt.New(shacrypt.WithVariant(shacrypt.VariantSHA256), shacrypt.WithIterations(5000))
+	if err != nil {
+		panic(err)
+	}
+
+	digest, err := hasher.Hash(newpasswd)
+	if err != nil {
+		panic(err)
+	}
+	cryptdata := []byte(digest.Encode())
+	err = os.WriteFile(*passfile, []byte(newpasswd), 0600)
+	if err != nil { panic(err )}
+	err = os.WriteFile(*cryptfile, cryptdata, 0600)
+	if err != nil { panic(err )}
+	keydata, err := os.ReadFile(*hmackeyfile)
+	if err != nil { panic(err )}
+	hmacer := hmac.New(sha256.New, keydata)
+	hmacer.Write(cryptdata)
+	hmacresult := hmacer.Sum(nil)
+	hmacout := []byte(base64.StdEncoding.EncodeToString(hmacresult))
+	err = os.WriteFile(*hmacfile, hmacout, 0600)
+	if err != nil { panic(err )}
+}
+

confluent_server/bin/confluent_selfcheck

@@ -27,6 +27,15 @@ import signal
 import confluent.collective.manager as collective
 import confluent.noderange as noderange
 
+def check_neigh_overflow():
+    dmesgout = subprocess.check_output(['dmesg'])
+    if b'_cache: neighbor table overflow!' in subprocess.check_output(['dmesg']):
+        return True
+    return False
+    #dmesg snippets
+    #[1203637.865870] neighbour: ndisc_cache: neighbor table overflow!
+    #[1205244.122606] neighbour: arp_cache: neighbor table overflow!
+
 
 def fprint(txt):
     sys.stdout.write(txt)
@@ -202,6 +211,11 @@ if __name__ == '__main__':
                     emprint('Failed access, if selinux is enabled, `setsebool -P httpd_can_network_connect=1`, otherwise check web proxy configuration')
         else:
             emprint('Not Running (Example resolution: systemctl enable httpd --now)')
+        fprint('IP neighbor table issue check:')
+        if check_neigh_overflow():
+            emprint('ARP/Neighbor table problem detected, evaluate and increase net.ipv*.neigh.default.gc_thresh*')
+        else:
+            print('OK')
         fprint('TFTP Status: ')
         if tftp_works():
             print('OK')
@@ -213,19 +227,34 @@ if __name__ == '__main__':
         else:
             emprint('No trusted ssh keys for root user, passwordless SSH from managers to nodes may not work (Example resolution: osdeploy initialize -u)')
         if sshutil.sshver() > 7.6:
-            fprint('Checking SSH Certificate authority: ')
-            try:
-                sshutil.prep_ssh_key('/etc/confluent/ssh/ca')
-                print('OK')
-            except Exception:
-                emprint('Failed to load SSH authority key, deployed servers will not have host certificates for known_hosts and users may be unable to ssh between nodes without a password (Example resolution: osdeploy initialize -s)')            
-            fprint('Checking confluent SSH automation key: ')
-            try:
-                sshutil.prep_ssh_key('/etc/confluent/ssh/automation')
-                print('OK')
-            except subprocess.CalledProcessError:
-                emprint('Failed to load confluent automation key, syncfiles and profile ansible plays will not work (Example resolution: osdeploy initialize -a)')
-            os.kill(int(sshutil.agent_pid), signal.SIGTERM)
+            child = os.fork()
+            if child > 0:
+                pid, extcode = os.waitpid(child, 0)
+            else:
+                sshutil.ready_keys = {}
+                sshutil.agent_pid = None
+                cuser = pwd.getpwnam('confluent')
+                os.setgid(cuser.pw_gid)
+                os.setuid(cuser.pw_uid)
+                fprint('Checking SSH Certificate authority: ')
+                try:
+                    sshutil.prep_ssh_key('/etc/confluent/ssh/ca')
+                    print('OK')
+                except Exception as e:
+                    if type(e).__name__ == 'CalledProcessError' and 'UNPROTECTED' in e.stderr.decode():
+                        emprint('Permissions incorrect on /etc/confluent/ssh/ca (Example resolution: chmod 600 /etc/confluent/ssh/ca)')
+                    else:
+                        emprint('Failed to load SSH authority key, deployed servers will not have host certificates for known_hosts and users may be unable to ssh between nodes without a password (Example resolution: osdeploy initialize -s)')
+                fprint('Checking confluent SSH automation key: ')
+                try:
+                    sshutil.prep_ssh_key('/etc/confluent/ssh/automation')
+                    print('OK')
+                except Exception as e:
+                    if type(e).__name__ == 'CalledProcessError' and 'UNPROTECTED' in e.stderr.decode():
+                        emprint('Permissions incorrect on /etc/confluent/ssh/automation (Example resolution: chmod 600 /etc/confluent/ssh/automation)')
+                    else:
+                        emprint('Failed to load confluent automation key, syncfiles and profile ansible plays will not work (Example resolution: osdeploy initialize -a)')
+                sys.exit(0)
         fprint('Checking for blocked insecure boot: ')
         if insecure_boot_attempts():
             emprint('Some nodes are attempting network boot using PXE or HTTP boot, but the node is not configured to allow this (Example resolution: nodegroupattrib everything deployment.useinsecureprotocols=firmware)')

confluent_server/bin/confluentdbutil

@@ -50,6 +50,8 @@ argparser.add_option('-s', '--skipkeys', action='store_true',
                            'protected keys.json file, and only the protected '
                            'data is needed.  keys do not change and as such '
                            'they do not require incremental backup')
+argparser.add_option('-y', '--yaml', action='store_true',
+                      help='Use YAML instead of JSON as file format')
 (options, args) = argparser.parse_args()
 if len(args) != 2 or args[0] not in ('dump', 'restore', 'merge'):
     argparser.print_help()
@@ -73,9 +75,16 @@ if args[0] in ('restore', 'merge'):
         cfm.init(stateless)
         cfm.statelessmode = stateless
         skipped = {'nodes': [], 'nodegroups': []}
+        
+        # Use the format parameter based on the --yaml option
+        format = 'yaml' if options.yaml else 'json'
+        
         cfm.restore_db_from_directory(
             dumpdir, password,
-            merge="skip" if args[0] == 'merge' else False, skipped=skipped)
+            merge="skip" if args[0] == 'merge' else False, 
+            skipped=skipped,
+            format=format)
+            
         if skipped['nodes']:
             skippedn = ','.join(skipped['nodes'])
             print('The following nodes were skipped during merge: '
@@ -114,8 +123,11 @@ elif args[0] == 'dump':
     main._initsecurity(conf.get_config())
     if not os.path.exists(dumpdir):
         os.makedirs(dumpdir)
+    
+    # Use the format parameter based on the --yaml option
+    format = 'yaml' if options.yaml else 'json'
     cfm.dump_db_to_directory(dumpdir, password, options.redact,
-                             options.skipkeys)
+                             options.skipkeys, format=format)
 
 
 

confluent_server/confluent/config/attributes.py

@@ -215,6 +215,13 @@ node = {
                         'Using this requires that collective members be '
                         'defined as nodes for noderange expansion')
     },
+    'deployment.lock': {
+        'description': ('Indicates whether deployment actions should be impeded. '
+                        'If locked, it indicates that a pending profile should not be applied. '
+                        'If "autolock", then locked will be set when current pending deployment completes. '
+                         ),
+        'validlist':    ('autolock', 'locked')
+    },
     'deployment.pendingprofile': {
         'description': ('An OS profile that is pending deployment.  This indicates to '
                         'the network boot subsystem what should be offered when a potential '
@@ -265,8 +272,7 @@ node = {
     },
     'discovery.policy': {
         'description':  'Policy to use for auto-configuration of discovered '
-                        'and identified nodes. Valid values are "manual", '
-                        '"permissive", or "open". "manual" means nodes are '
+                        'and identified nodes. "manual" means nodes are '
                         'detected, but not autoconfigured until a user '
                         'approves. "permissive" indicates to allow discovery, '
                         'so long as the node has no existing public key. '
@@ -361,9 +367,8 @@ node = {
 #                        'to suppress serial console configuration')
 #    },
     'console.logging': {
-        'description': ('Indicate logging level to apply to console.  Valid '
-                        'values are currently "full", "interactive", "memory", and '
-                        '"none". Defaults to "full".'),
+        'description': ('Indicate logging level to apply to console. '
+                        'Defaults to "full".'),
         'validvalues': ('full', 'memory', 'interactive', 'none'),
     },
     'console.method': {
@@ -371,7 +376,7 @@ node = {
                         'the managed node.  If not specified, then console '
                         'is disabled.  "ipmi" should be specified for most '
                         'systems if console is desired.'),
-        'validvalues': ('ssh', 'ipmi', 'openbmc', 'tsmsol'),
+        'validvalues': ('ssh', 'ipmi', 'openbmc', 'tsmsol', 'vcenter'),
     },
 #    'virtualization.host': {
 #        'description': ('Hypervisor where this node does/should reside'),
@@ -557,7 +562,7 @@ node = {
         'description': 'Indicates that this interface should be a team and what mode or runner to use when teamed. '
                        'If this covers a deployment interface, one of the member interfaces may be brought up as '
                        'a standalone interface until deployment is complete, as supported by the OS deployment profile. '
-                       'To support this scenario, the switch should be set up to allow independent operation of member ports123654 (e.g. lacp bypass mode or fallback mode).',
+                       'To support this scenario, the switch should be set up to allow independent operation of member ports (e.g. lacp bypass mode or fallback mode).',
         'validvalues': ('lacp', 'loadbalance', 'roundrobin', 'activebackup', 'none')
     },
     'power.pdu': {
@@ -639,6 +644,9 @@ node = {
         'description':  ('Fingerprint of the TLS certificate recognized as'
                          'belonging to the hardware manager of the server'),
     },
+    'pubkeys.tls': {
+        'description': ('Fingerprint of the TLS certificate for service running on host.'),
+    },
     'pubkeys.ssh': {
         'description': ('Fingerprint of the SSH key of the OS running on the '
                         'system.'),

confluent_server/confluent/config/configmanager.py

@@ -102,7 +102,7 @@ try:
     unicode
 except NameError:
     unicode = str
-
+import yaml
 
 _masterkey = None
 _masterintegritykey = None
@@ -132,11 +132,24 @@ _validroles = ['Administrator', 'Operator', 'Monitor', 'Stub']
 
 membership_callback = None
 
+
+class ExpressionChecker(string.Formatter):
+    def format_field(self, val, format_spec):
+        if len(format_spec) > 8:
+            raise Exception(f'Format specification {format_spec} exceeds maximum supported length of 8')
+        return '1'
+
+    def get_field(self, field_name, args, kwargs):
+        return field_name, field_name
+
+    def get_value(self, first, args, kwargs):
+        return 1
+
 def attrib_supports_expression(attrib):
     if not isinstance(attrib, str):
         attrib = attrib.decode('utf8')
     attrib = _attraliases.get(attrib, attrib)
-    if attrib.startswith('secret.') or attrib.startswith('crypted.'):
+    if attrib.startswith('secret.') or attrib.startswith('crypted.') or attrib.startswith('custom.nodesecret.'):
         return False
     return True
 
@@ -1108,7 +1121,14 @@ class _ExpressionFormat(string.Formatter):
             field_name = val
         parsed = ast.parse(field_name)
         val = self._handle_ast_node(parsed.body[0].value)
-        return format(val, format_spec)
+        try:
+            val = int(val)
+        except Exception:
+            pass
+        formatted = format(val, format_spec)
+        if len(formatted) > 16384:
+            raise Exception('Field length exceeded during formatting')
+        return formatted
 
     def _handle_ast_node(self, node):
         if isinstance(node, ast.Num):
@@ -1373,7 +1393,7 @@ class ConfigManager(object):
             attribute, match = expression.split('=')
         else:
             raise Exception('Invalid Expression')
-        if attribute.startswith('secret.'):
+        if attribute.startswith('secret.') or attribute.startswith('custom.nodesecret.'):
             raise Exception('Filter by secret attributes is not supported')
         if attribute_name_is_invalid(attribute):
             raise ValueError(
@@ -1918,6 +1938,8 @@ class ConfigManager(object):
                                 curr[attrib]['value'])
                     if 'value' in curr[attrib]:
                         del curr[attrib]['value']
+                if 'expression' in curr[attrib]:
+                    ExpressionChecker().format(curr[attrib]['expression'])
         if cfgleader:  # currently config slave to another
             return exec_on_leader('_rpc_master_set_group_attributes',
                                   self.tenant, attribmap, autocreate)
@@ -2023,10 +2045,10 @@ class ConfigManager(object):
                     newdict = {'value': attribmap[group][attr]}
                 else:
                     newdict = attribmap[group][attr]
-                if keydata and attr.startswith('secret.') and 'cryptvalue' in newdict:
+                if keydata and (attr.startswith('secret.') or attr.startswith('custom.nodesecret.')) and 'cryptvalue' in newdict:
                     newdict['value'] = decrypt_value(newdict['cryptvalue'], keydata['cryptkey'], keydata['integritykey'])
                     del newdict['cryptvalue']
-                if 'value' in newdict and attr.startswith("secret."):
+                if 'value' in newdict and (attr.startswith('secret.') or attr.startswith('custom.nodesecret.')):
                     newdict['cryptvalue'] = crypt_value(newdict['value'])
                     del newdict['value']
                 if 'value' in newdict and attr.startswith("crypted."):
@@ -2367,6 +2389,8 @@ class ConfigManager(object):
         for node in attribmap:
             curr = attribmap[node]
             for attrib in curr:
+                if 'expression' in curr[attrib]:
+                    ExpressionChecker().format(curr[attrib]['expression'])
                 if attrib.startswith('crypted.'):
                     if not isinstance(curr[attrib], dict):
                         curr[attrib] = {'value': curr[attrib]}
@@ -2485,10 +2509,10 @@ class ConfigManager(object):
                 # add check here, skip None attributes
                 if newdict is None:
                     continue
-                if keydata and attrname.startswith('secret.') and 'cryptvalue' in newdict:
+                if keydata and (attrname.startswith('secret.') or attrname.startswith('custom.nodesecret.')) and 'cryptvalue' in newdict:
                     newdict['value'] = decrypt_value(newdict['cryptvalue'], keydata['cryptkey'], keydata['integritykey'])
                     del newdict['cryptvalue']
-                if 'value' in newdict and attrname.startswith("secret."):
+                if 'value' in newdict and (attrname.startswith('secret.') or attrname.startswith('custom.nodesecret.')):
                     newdict['cryptvalue'] = crypt_value(newdict['value'])
                     del newdict['value']
                 if 'value' in newdict and attrname.startswith("crypted."):
@@ -2945,12 +2969,30 @@ def _dump_keys(password, dojson=True):
     return keydata
 
 
-def restore_db_from_directory(location, password, merge=False, skipped=None):
+def restore_db_from_directory(location, password, merge=False, skipped=None, format='json'):
+    """Restore database from a directory
+    
+    :param location: Directory containing the configuration
+    :param password: Password to decrypt sensitive data
+    :param merge: If True, merge with existing configuration
+    :param skipped: List of elements to skip during restore
+    :param format: Format of the files ('json' [default] or 'yaml')
+    """
+    if format not in ('json', 'yaml'):
+        raise ValueError("Format must be 'json' or 'yaml'")
+
     kdd = None
     try:
-        with open(os.path.join(location, 'keys.json'), 'r') as cfgfile:
+        keys_file = os.path.join(location, f'keys.{format}')
+        with open(keys_file, 'r') as cfgfile:
             keydata = cfgfile.read()
-            kdd = json.loads(keydata)
+            if format == 'json':
+                kdd = json.loads(keydata)
+            else:
+                kdd = yaml.safe_load(keydata)
+                if kdd is None:
+                    raise ValueError(f"Invalid or empty YAML content in {keys_file}")
+            
             if merge:
                 if 'cryptkey' in kdd:
                     kdd['cryptkey'] = _parse_key(kdd['cryptkey'], password)
@@ -2959,59 +3001,122 @@ def restore_db_from_directory(location, password, merge=False, skipped=None):
                 else:
                     kdd['integritykey'] = None  # GCM
             else:
+                if format == 'json':
+                    _restore_keys(keydata, password)
+                else:
+                    # Convert YAML to JSON string for _restore_keys
+                    _restore_keys(json.dumps(kdd), password)
                 kdd = None
-                _restore_keys(keydata, password)
     except IOError as e:
         if e.errno == 2:
             raise Exception("Cannot restore without keys, this may be a "
                             "redacted dump")
     if not merge:
         try:
-            moreglobals = json.load(open(os.path.join(location, 'globals.json')))
-            for globvar in moreglobals:
-                set_global(globvar, moreglobals[globvar])
+            globals_file = os.path.join(location, f'globals.{format}')
+            with open(globals_file, 'r') as globin:
+                if format == 'json':
+                    moreglobals = json.load(globin)
+                else:
+                    moreglobals = yaml.safe_load(globin)
+                    if moreglobals is None:
+                        raise ValueError(f"Invalid or empty YAML content in {globals_file}")
+                
+                for globvar in moreglobals:
+                    set_global(globvar, moreglobals[globvar])
         except IOError as e:
             if e.errno != 2:
                 raise
         try:
-            collective = json.load(open(os.path.join(location, 'collective.json')))
-            _cfgstore['collective'] = {}
-            for coll in collective:
-                add_collective_member(coll, collective[coll]['address'],
-                                    collective[coll]['fingerprint'])
+            collective_file = os.path.join(location, f'collective.{format}')
+            with open(collective_file, 'r') as collin:
+                if format == 'json':
+                    collective = json.load(collin)
+                else:
+                    collective = yaml.safe_load(collin)
+                    if collective is None:
+                        raise ValueError(f"Invalid or empty YAML content in {collective_file}")
+                
+                _cfgstore['collective'] = {}
+                for coll in collective:
+                    add_collective_member(coll, collective[coll]['address'],
+                                        collective[coll]['fingerprint'])
         except IOError as e:
             if e.errno != 2:
                 raise
-    with open(os.path.join(location, 'main.json'), 'r') as cfgfile:
+    main_file = os.path.join(location, f'main.{format}')
+    with open(main_file, 'r') as cfgfile:
         cfgdata = cfgfile.read()
+        if format == 'yaml':
+            # Convert YAML to JSON string for _load_from_json
+            yaml_data = yaml.safe_load(cfgdata)
+            if yaml_data is None:
+                raise ValueError(f"Invalid or empty YAML content in {main_file}")
+            cfgdata = json.dumps(yaml_data)
         ConfigManager(tenant=None)._load_from_json(cfgdata, merge=merge, keydata=kdd, skipped=skipped)
     ConfigManager.wait_for_sync(True)
 
-
-def dump_db_to_directory(location, password, redact=None, skipkeys=False):
+def dump_db_to_directory(location, password, redact=None, skipkeys=False, format='json'):
+    """Dump database to a directory
+    
+    :param location: Directory to store the configuration
+    :param password: Password to protect sensitive data
+    :param redact: If True, redact sensitive data
+    :param skipkeys: If True, skip dumping keys
+    :param format: Format to use for dumping ('json' [default] or 'yaml')
+    """
+    if format not in ('json', 'yaml'):
+        raise ValueError("Format must be 'json' or 'yaml'")
+    
+    # Handle keys file
     if not redact and not skipkeys:
-        with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
-            cfgfile.write(_dump_keys(password))
+        with open(os.path.join(location, f'keys.{format}'), 'w') as cfgfile:
+            if format == 'json':
+                cfgfile.write(_dump_keys(password))
+            else:
+                keydata = _dump_keys(password, dojson=False)
+                yaml.dump(keydata, cfgfile, default_flow_style=False)
             cfgfile.write('\n')
-    with open(os.path.join(location, 'main.json'), 'wb') as cfgfile:
-        cfgfile.write(ConfigManager(tenant=None)._dump_to_json(redact=redact))
-        cfgfile.write(b'\n')
+    
+    # Handle main config
+    main_data = ConfigManager(tenant=None)._dump_to_json(redact=redact)
+    with open(os.path.join(location, f'main.{format}'), 'wb' if format == 'json' else 'w') as cfgfile:
+        if format == 'json':
+            cfgfile.write(main_data)
+            cfgfile.write(b'\n')
+        else:
+            # Convert JSON to Python object, then dump as YAML
+            yaml.dump(json.loads(main_data.decode('utf-8')), cfgfile, default_flow_style=False)
+    
+    # Handle collective data
     if 'collective' in _cfgstore:
-        with open(os.path.join(location, 'collective.json'), 'w') as cfgfile:
-            cfgfile.write(json.dumps(_cfgstore['collective']))
-            cfgfile.write('\n')
+        with open(os.path.join(location, f'collective.{format}'), 'w') as cfgfile:
+            if format == 'json':
+                cfgfile.write(json.dumps(_cfgstore['collective']))
+                cfgfile.write('\n')
+            else:
+                yaml.dump(_cfgstore['collective'], cfgfile, default_flow_style=False)
+    
+    # Handle globals
     bkupglobals = get_globals()
     if bkupglobals:
-        with open(os.path.join(location, 'globals.json'), 'w') as globout:
-            json.dump(bkupglobals, globout)
+        with open(os.path.join(location, f'globals.{format}'), 'w') as globout:
+            if format == 'json':
+                json.dump(bkupglobals, globout)
+            else:
+                yaml.dump(bkupglobals, globout, default_flow_style=False)
+    
+    # Handle tenants
     try:
         for tenant in os.listdir(
                 os.path.join(ConfigManager._cfgdir, '/tenants/')):
-            with open(os.path.join(location, 'tenants', tenant,
-                                   'main.json'), 'w') as cfgfile:
-                cfgfile.write(ConfigManager(tenant=tenant)._dump_to_json(
-                    redact=redact))
-                cfgfile.write('\n')
+            tenant_data = ConfigManager(tenant=tenant)._dump_to_json(redact=redact)
+            with open(os.path.join(location, 'tenants', tenant, f'main.{format}'), 'wb' if format == 'json' else 'w') as cfgfile:
+                if format == 'json':
+                    cfgfile.write(tenant_data)
+                    cfgfile.write(b'\n')
+                else:
+                    yaml.dump(json.loads(tenant_data.decode('utf-8')), cfgfile, default_flow_style=False)
     except OSError:
         pass
 

confluent_server/confluent/consoleserver.py

@@ -575,12 +575,12 @@ def disconnect_node(node, configmanager):
 
 def _nodechange(added, deleting, renamed, configmanager):
     for node in deleting:
-        disconnect_node(node, configmanager)
+        eventlet.spawn(disconnect_node, node, configmanager)
     for node in renamed:
         disconnect_node(node, configmanager)
-        connect_node(renamed[node], configmanager)
+        eventlet.spawn(connect_node, renamed[node], configmanager)
     for node in added:
-        connect_node(node, configmanager)
+        eventlet.spawn(connect_node, node, configmanager)
 
 
 def _start_tenant_sessions(cfm):

confluent_server/confluent/core.py

@@ -74,7 +74,7 @@ import uuid
 import yaml
 import shutil
 
-
+vinz = None
 pluginmap = {}
 dispatch_plugins = (b'ipmi', u'ipmi', b'redfish', u'redfish', b'tsmsol', u'tsmsol', b'geist', u'geist', b'deltapdu', u'deltapdu', b'eatonpdu', u'eatonpdu', b'affluent', u'affluent', b'cnos', u'cnos', b'enos', u'enos')
 
@@ -213,6 +213,7 @@ def handle_deployment(configmanager, inputdata, pathcomponents,
                     with open('/var/lib/confluent/public/os/{}/profile.yaml'.format(profname)) as profyaml:
                         profinfo = yaml.safe_load(profyaml)
                         profinfo['name'] = profname
+                    #check if boot.ipxe is older than profile.yaml
                     yield msg.KeyValueData(profinfo)
                     return
         elif len(pathcomponents) == 3:
@@ -434,13 +435,27 @@ def _init_core():
                 'pluginattrs': ['hardwaremanagement.method'],
                 'default': 'ipmi',
             }),
-            'ikvm': PluginRoute({'handler': 'ikvm'}),
+            'ikvm': PluginRoute({
+                'pluginattrs': ['hardwaremanagement.method'],
+                'default': 'ipmi',
+            }),
+            'ikvm_methods': PluginRoute({
+                'pluginattrs': ['hardwaremanagement.method'],
+                'default': 'ipmi',
+            }),
+            'ikvm_screenshot': PluginRoute({
+                'pluginattrs': ['hardwaremanagement.method'],
+                'default': 'ipmi',
+            }),
         },
         'description': PluginRoute({
             'pluginattrs': ['hardwaremanagement.method'],
             'default': 'ipmi',
         }),
         'deployment': {
+            'lock': PluginRoute({
+                'handler': 'attributes'
+            }),
             'ident_image': PluginRoute({
                 'handler': 'identimage'
             })
@@ -483,6 +498,10 @@ def _init_core():
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
                 }),
+                'updatestatus': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
                 'updates': {
                     'active': PluginCollection({
                             'pluginattrs': ['hardwaremanagement.method'],
@@ -957,6 +976,7 @@ def _forward_rsp(connection, res):
 
 def handle_node_request(configmanager, inputdata, operation,
                         pathcomponents, autostrip=True):
+    global vinz
     if log.logfull:
         raise exc.TargetResourceUnavailable('Filesystem full, free up space and restart confluent service')
     iscollection = False
@@ -1075,6 +1095,10 @@ def handle_node_request(configmanager, inputdata, operation,
                     plugpath = plugroute['default']
             if plugpath in dispatch_plugins:
                 cfm.check_quorum()
+                if pathcomponents == ['console', 'ikvm']:
+                    if not vinz:
+                        import confluent.vinzmanager as vinz
+                        vinz.assure_vinz()
                 manager = nodeattr[node].get('collective.manager', {}).get(
                     'value', None)
                 if manager:

confluent_server/confluent/discovery/core.py

@@ -529,13 +529,22 @@ def register_remote_addrs(addresses, configmanager):
         nd = {
             'addresses': [(addr, 443)]
         }
-        sd = ssdp.check_fish(('/DeviceDescription.json', nd))
-        if not sd:
+        try:
+            sd = ssdp.check_fish(('/DeviceDescription.json', nd))
+            if not sd:
+                return addr, False
+            if 'macaddress' in sd['attributes']:
+                sd['hwaddr'] = sd['attributes']['macaddress']
+            else:
+                sd['hwaddr'] = sd['attributes']['mac-address']
+            if 'lenovo-xcc3' in sd['services']:
+                nh = xcc3.NodeHandler(sd, configmanager)
+            elif 'lenovo-xcc' in sd['services']:
+                nh = xcc.NodeHandler(sd, configmanager)
+            nh.scan()
+            detected(nh.info)
+        except Exception:
             return addr, False
-        sd['hwaddr'] = sd['attributes']['mac-address']
-        nh = xcc.NodeHandler(sd, configmanager)
-        nh.scan()
-        detected(nh.info)
         return addr, True
     rpool = eventlet.greenpool.GreenPool(512)
     for count in iterate_addrs(addresses, True):
@@ -1087,7 +1096,7 @@ def get_nodename(cfg, handler, info):
         # Ok, see if it is something with a chassis-uuid and discover by
         # chassis
         nodename = get_nodename_from_enclosures(cfg, info)
-    if not nodename and handler.devname == 'SMM':
+    if not nodename and handler.devname in ('SMM', 'SMM3'):
         nodename = get_nodename_from_chained_smms(cfg, handler, info)
     if not nodename:  # as a last resort, search switches for info
         # This is the slowest potential operation, so we hope for the
@@ -1095,7 +1104,7 @@ def get_nodename(cfg, handler, info):
         nodename, macinfo = macmap.find_nodeinfo_by_mac(info['hwaddr'], cfg)
         maccount = macinfo['maccount']
         if nodename:
-            if handler.devname == 'SMM':
+            if handler.devname in ('SMM', 'SMM3'):
                 nl = list(cfg.filter_node_attributes(
                             'enclosure.extends=' + nodename))
                 if nl:
@@ -1118,7 +1127,7 @@ def get_nodename(cfg, handler, info):
                         return None, None
         if (nodename and
                 not handler.discoverable_by_switch(macinfo['maccount'])):
-            if handler.devname == 'SMM':
+            if handler.devname in ('SMM', 'SMM3'):
                 errorstr = 'Attempt to discover SMM by switch, but chained ' \
                            'topology or incorrect net attributes detected, ' \
                            'which is not compatible with switch discovery ' \
@@ -1186,7 +1195,9 @@ def search_smms_by_cert(currsmm, cert, cfg):
         cd = cfg.get_node_attributes(currsmm, ['hardwaremanagement.manager',
                                                'pubkeys.tls_hardwaremanager'])
         smmaddr = cd.get(currsmm, {}).get('hardwaremanagement.manager', {}).get('value', None)
-        wc = webclient.SecureHTTPConnection(currsmm, verifycallback=cv)
+        if not smmaddr:
+            smmaddr = currsmm
+        wc = webclient.SecureHTTPConnection(smmaddr, verifycallback=cv)
         neighs = wc.grab_json_response('/scripts/neighdata.json')
     except Exception:
         return None

confluent_server/confluent/discovery/handlers/redfishbmc.py

@@ -276,7 +276,25 @@ class NodeHandler(generic.NodeHandler):
                     continue
                 actualnics.append(candnic)
             if len(actualnics) != 1:
-                raise Exception("Multi-interface BMCs are not supported currently")
+                compip = self.ipaddr
+                if ':' in compip:
+                    compip = compip.split('%')[0]
+                    ipkey = 'IPv6Addresses'
+                else:
+                    ipkey = 'IPv6Addresses'
+                actualnic = None
+                for curractnic in actualnics:
+                    currnicinfo = wc.grab_json_response(curractnic)
+                    for targipaddr in currnicinfo.get(ipkey, []):
+                        targipaddr = targipaddr.get('Address', 'Z')
+                        if compip == targipaddr:
+                            actualnic = curractnic
+                            break
+                    if actualnic:
+                        break
+                else:
+                    raise Exception("Unable to detect active NIC of multi-nic bmc")
+                actualnics = [actualnic]
             currnet = wc.grab_json_response(actualnics[0])
             netconfig = netutil.get_nic_config(self.configmanager, nodename, ip=newip)
             newconfig = {