Fix type of maxnodes

Do not aggressively respawn buffer daemon.
Start buffer daemon only when needed. Limit restarts to once every 30 seconds.
2026-09-28 16:20:54 +00:00 · 2026-08-21 14:58:13 -04:00 · 2026-08-20 07:23:40 -04:00 · 2026-07-16 12:39:51 -04:00 · 2026-07-14 11:01:32 -04:00 · 2026-07-14 09:52:27 -04:00
58 changed files with 1078 additions and 269 deletions
@@ -133,6 +133,8 @@ def print_help():

 def updatestatus(stateinfo={}):
    global powerstate, powertime, clearpowermessage
+    if opts.headless:
+        return
    status = consolename
    info = []
    for statekey in stateinfo:
@@ -455,8 +457,10 @@ def do_command(command, server):
        currconsole = targpath
        startrequest = {'operation': 'start', 'path': targpath,
                        'parameters': {}}
-        height, width = struct.unpack(
-            'hh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, b'....'))[:2]
+        height, width = 31, 100
+        if not opts.headless:
+            height, width = struct.unpack(
+                'hh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, b'....'))[:2]
        startrequest['parameters']['width'] = width
        startrequest['parameters']['height'] = height
        for param in argv[2:]:
@@ -624,17 +628,19 @@ def startconsole(nodename):
    signal.signal(signal.SIGWINCH, do_resize)
    didconsole = True
    consolename = nodename
-    tty.setraw(sys.stdin.fileno())
-    currfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
-    fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl | os.O_NONBLOCK)
+    if not opts.headless:
+        tty.setraw(sys.stdin.fileno())
+        currfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
+        fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl | os.O_NONBLOCK)
    inconsole = True
+    check_automation('')  # give any leading 'sends' a chance


 def quitconfetty(code=0, fullexit=False, fixterm=True):
    global inconsole
    global currconsole
    global didconsole
-    if fixterm or didconsole:
+    if (fixterm or didconsole) and not opts.headless:
        currfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
        fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl & ~os.O_NONBLOCK)
        if oldtcattr is not None:
@@ -844,6 +850,20 @@ def check_escape_seq(currinput, filehandle):
        currinput += filehandle.read()
    return currinput

+automation_directives = []
+current_automation_directive = None
+automation_map = {
+    '<up>': '\x1b[A',
+    '<down>': '\x1b[B',
+    '<right>': '\x1b[C',
+    '<left>': '\x1b[D',
+    '<enter>': '\r',
+    '<esc>': '\x1b',
+    '<tab>': '\t',
+}
+
+
+
 parser = optparse.OptionParser()
 parser.add_option("-s", "--server", dest="netserver",
                  help="Confluent instance to connect to",
@@ -851,12 +871,64 @@ parser.add_option("-s", "--server", dest="netserver",
 parser.add_option("-c", "--control", dest="controlpath",
                  help="Path to offer terminal control",
                  metavar="PATH")
+parser.add_option('-a', '--automation', type='string', default=None,
+                  help='Specify an automation script to run', metavar='SCRIPT')
+parser.add_option('-e', '--headless', action='store_true', default=False,
+                  help='Run in headless mode, which is designed for use with '
+                       'automation scripts and disables interactive features')
 parser.add_option(
    '-m', '--mintime', default=0,
    help='Minimum time to run or else pause for input (used to keep a '
         'terminal from closing quickly on error)')
 opts, shellargs = parser.parse_args()

+def parse_automation_script(script, session_node):
+    global current_automation_directive
+    for line in script.splitlines():
+        line = line.strip()
+        if not line or line.startswith('#'):
+            continue
+        if line.startswith('exit'):
+            automation_directives.append(('exit', None))
+            continue
+        if ' ' not in line:
+            sys.stderr.write("Invalid line in automation script: %s\n" % line)
+            continue
+        directive, arg = line.split(' ', 1)
+        directive = directive.strip().lower()
+        if directive not in ('expect', 'send', 'forget'):
+            sys.stderr.write("Unknown directive in automation script: %s\n" % directive)
+            continue
+        arg = arg.strip()
+        origarg = arg
+        if arg[0] not in ('"', "'"):
+            arg = '"' + arg + '"'
+        if arg[0] == "'" and arg[-1] == "'":
+            # do not process '<>' sequences in single quotes
+            arg = arg[1:-1]
+            arg = bytes(arg, "utf-8").decode("unicode_escape")
+        elif arg[0] == '"' and arg[-1] == '"':
+            arg = bytes(arg[1:-1], "utf-8").decode("unicode_escape")
+            for key, value in automation_map.items():
+                arg = arg.replace(key, value)
+            arg = re.sub(r'<env:(\w+)>', lambda m: os.environ[m.group(1)], arg)
+        if '{' in arg:  # support confluent expressions
+            for res in session.create('/nodes/{0}/attributes/expression'.format(session_node),
+                            {'expression': arg}):
+                if 'error' in res:
+                    sys.stderr.write(res['error'] + '\n')
+                    sys.exit(1)
+                if 'value' in res:
+                    arg = res['value']
+        automation_directives.append((directive, arg, origarg))
+    if automation_directives:
+        current_automation_directive = automation_directives.pop(0)
+        if opts.headless and current_automation_directive[0] == 'expect':
+            sys.stdout.write(f'Expecting {repr(current_automation_directive[2])}\r\n')
+            sys.stdout.flush()
+
+
+
 username = None
 passphrase = None
 def server_connect():
@@ -909,6 +981,7 @@ def main():
    # sys.stdout.write('\x1b[H\x1b[J')
    # sys.stdout.flush()
    global powerstate, powertime, clearpowermessage
+    


    if sys.stdout.isatty():
@@ -925,6 +998,9 @@ def main():
    targ, session_node = get_session_node(shellargs)
    if session_node is not None:
        consoleonly = True
+        if opts.automation:
+            with open(opts.automation) as f:
+                parse_automation_script(f.read(), session_node)
        do_command("start %s" % targ, netserver)
        doexit = True
    elif shellargs:
@@ -936,9 +1012,13 @@ def main():

    while inconsole or not doexit:
        if inconsole:
+            if opts.headless:
+                handles = [session.connection]
+            else:
+                handles = (sys.stdin, session.connection)
            try:
                rdylist, _, _ = select.select(
-                    (sys.stdin, session.connection), (), (), 10)
+                    handles, (), (), 10)
            except select.error:
                rdylist = ()
            for fh in rdylist:
@@ -976,6 +1056,72 @@ fgcolor = None
 bgcolor = None
 fgshifted = False
 pendseq = ''
+automation_check = ''
+
+def check_automation(data):
+    global automation_check
+    global current_automation_directive
+    if type(data) != str:
+        data = data.decode('utf-8', errors='ignore')
+    while current_automation_directive:
+        if current_automation_directive[0] == 'forget' and data:
+            current_automation_directive = None
+            automation_check = ''
+            if automation_directives:
+                current_automation_directive = automation_directives.pop(0)
+                if opts.headless and current_automation_directive[0] == 'expect':
+                    sys.stdout.write(f'Expecting {repr(current_automation_directive[2])}\n')
+                    sys.stdout.flush()
+            return
+        if current_automation_directive[0] == 'expect':
+            expected = current_automation_directive[1]
+            combined = automation_check + data
+            if expected and expected in combined:
+                data = data[combined.rindex(expected) + len(expected):]
+
+                if opts.headless:
+                    sys.stdout.write(f'Detected {repr(current_automation_directive[2])}\r\n')
+                    sys.stdout.flush()
+                current_automation_directive = None
+                automation_check = ''
+                if automation_directives:
+                    current_automation_directive = automation_directives.pop(0)
+                    if opts.headless and current_automation_directive[0] == 'expect':
+                        sys.stdout.write(f'Expecting {repr(current_automation_directive[2])}\r\n')
+                        sys.stdout.flush()
+            else:
+                # Check if there's potential start of expected data in the incoming data
+                combined = automation_check + data
+                automation_check = ''
+                for i in range(1, min(len(expected), len(combined)) + 1):
+                    if expected.startswith(combined[-i:]):
+                        automation_check = combined[-i:]
+                return # wait for next check
+        elif current_automation_directive[0] == 'send':
+            data = ''
+            automation_check = ''
+            if opts.headless:
+                sys.stdout.write(f'Sending {repr(current_automation_directive[2])}\r\n')
+                sys.stdout.flush()
+            if current_automation_directive[1]:
+                tlvdata.send(session.connection, current_automation_directive[1])
+            current_automation_directive = None
+            if automation_directives:
+                current_automation_directive = automation_directives.pop(0)
+                if opts.headless and current_automation_directive[0] == 'expect':
+                    sys.stdout.write(f'Expecting {repr(current_automation_directive[2])}\r\n')
+                    sys.stdout.flush()
+        elif current_automation_directive[0] == 'exit':
+            if opts.headless:
+                sys.stdout.write('Automation completed\r\n')
+                sys.stdout.flush()
+            return True
+        if opts.headless and not current_automation_directive:
+            sys.stdout.write('Automation completed\r\n')
+            sys.stdout.flush()
+            return True
+    return False
+
 def consume_termdata(fh, bufferonly=False):
    global clearpowermessage
    global fgcolor, bgcolor, fgshifted, pendseq
@@ -987,6 +1133,11 @@ def consume_termdata(fh, bufferonly=False):
        updatestatus(data)
        return ''
    if data is not None:
+        shouldexit = check_automation(data)
+        if opts.headless:
+            if shouldexit:
+                quitconfetty(fullexit=True)
+            return ''
        indata = pendseq + client.stringify(data)
        pendseq = ''
        data = ''
@@ -1066,6 +1217,8 @@ def consume_termdata(fh, bufferonly=False):
            # this scenario comfortable that it
            # will come out soon enough
            pass
+        if shouldexit:
+            quitconfetty(fullexit=True)
    else:
        deadline = 5
        connected = False
@@ -18,6 +18,7 @@
 import base64
 import optparse
 import os
+import shlex
 import subprocess
 import sys
 path = os.path.dirname(os.path.realpath(__file__))
@@ -62,8 +63,13 @@ argparser = optparse.OptionParser(
           "ctrl-'e', then release ctrl, then 'c', then '?' for a full list. "
           "For example, ctrl-'e', then 'c', then '.' will exit the current "
           "console")
+argparser.add_option('-a', '--automation', type='string', default=None,
+                     help='Specify an automation script')
 argparser.add_option('-t', '--tile', action='store_true', default=False,
                     help='Tile console windows in the terminal')
+argparser.add_option('-e', '--headless', action='store_true', default=False,
+                     help='Run in headless mode, which is designed for use with '
+                          'automation scripts and disables interactive features')
 argparser.add_option('-l', '--log', action='store_true', default=False,
                     help='Enter log replay mode instead of showing a live console')

@@ -98,6 +104,12 @@ argparser.add_option('-w','--windowed', action='store_true', default=False,

 (options, args) = argparser.parse_args()

+automation_args = []
+if options.automation:
+    automation_args = ['-a', options.automation]
+if options.headless:
+    automation_args += ['--headless']
+
 oldtcattr = None
 oldfl = None

@@ -655,7 +667,7 @@ if options.windowed:
    firstnode=nodes[0]
    nodes.pop(0)
    with open(os.devnull, 'wb') as devnull:
-        xopen=subprocess.Popen(envlist + [confettypath, '-c', '/tmp/controlpath-{0}'.format(firstnode), '-m', '5', 'start', '/nodes/{0}/console/session'.format(firstnode) ] , stdin=devnull)
+        xopen=subprocess.Popen(envlist + [confettypath, '-c', '/tmp/controlpath-{0}'.format(firstnode)] + automation_args + ['-m', '5', 'start', '/nodes/{0}/console/session'.format(firstnode)] , stdin=devnull)
        time.sleep(2)
        s=socket.socket(socket.AF_UNIX)
        winid=''
@@ -727,7 +739,7 @@ if options.windowed:
        else:
            pass 
        with open(os.devnull, 'wb') as devnull:
-            xopen=subprocess.Popen(envlist + [confettypath, '-c', '/tmp/controlpath-{0}'.format(node), '-m', '5', 'start', '/nodes/{0}/console/session'.format(node)] , stdin=devnull)
+            xopen=subprocess.Popen(envlist + [confettypath, '-c', '/tmp/controlpath-{0}'.format(node)] + automation_args + ['-m', '5', 'start', '/nodes/{0}/console/session'.format(node)] , stdin=devnull)
    sys.exit(0)
 #end of wcons
 if options.tile:
@@ -752,18 +764,18 @@ if options.tile:
        panename = '{0}:{1}'.format(sessname, pane)
        if initial:
            initial = False
+            confetty_cmd = [confettypath] + automation_args + ['-m', '5', 'start', '/nodes/{0}/console/session'.format(node)]
            subprocess.call(
                ['tmux', 'new-session', '-d', '-s',
                 sessname, '-x', '800', '-y',
-                 '800', '{0} -m 5 start /nodes/{1}/console/session'.format(
-                    confettypath, node)])
+                 '800', ' '.join(shlex.quote(arg) for arg in confetty_cmd)])
        else:
            subprocess.call(['tmux', 'select-pane', '-t', sessname])
            subprocess.call(['tmux', 'set-option', '-t', panename, 'pane-border-status', 'top'], stderr=null)
+            confetty_cmd = [confettypath] + automation_args + ['-m', '5', 'start', '/nodes/{0}/console/session'.format(node)]
            subprocess.call(
                    ['tmux', 'split', '-h', '-t', sessname,
-                 '{0} -m 5 start /nodes/{1}/console/session'.format(
-                     confettypath, node)])
+                 ' '.join(shlex.quote(arg) for arg in confetty_cmd)])
            subprocess.call(['tmux', 'select-layout', '-t', sessname, 'tiled'], stdout=null)
            pane += 1
    subprocess.call(['tmux', 'select-pane', '-t', sessname])
@@ -771,5 +783,6 @@ if options.tile:
    if not in_tmux:
        os.execlp('tmux', 'tmux', 'attach', '-t', sessname)
 else:
-    os.execl(confettypath, confettypath, 'start',
-             '/nodes/{0}/console/session'.format(args[0]))
+    execl_args = [confettypath] + automation_args + ['start',
+             '/nodes/{0}/console/session'.format(args[0])]
+    os.execl(confettypath, *execl_args)
@@ -78,6 +78,7 @@ def main(args):
    ap = argparse.ArgumentParser(description='Deploy OS to nodes')
    ap.add_argument('-c', '--clear', help='Clear any pending deployment action', action='store_true')
    ap.add_argument('-n', '--network', help='Initiate deployment over PXE/HTTP', action='store_true')
+    ap.add_argument('-b', '--bootmethod', help='Specify network boot method (e.g., network, http)', default='network')
    ap.add_argument('-p', '--prepareonly', help='Prepare only, skip any interaction with a BMC associated with this deployment action', action='store_true')
    ap.add_argument('-m', '--maxnodes', help='Specifiy a maximum nodes to be deployed')
    ap.add_argument('-r', '--redeploy', help='Redeploy nodes with the current or pending profile', action='store_true')
@@ -213,7 +214,7 @@ def main(args):
                print('{0}: {1}{2}'.format(node, profile, armed))
        sys.exit(0)
    if not args.clear and args.network and not args.prepareonly:
-        rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
+        rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', args.bootmethod,
                                        bootmode='uefi',
                                        persistent=False,
                                        errnodes=errnodes)
@@ -59,6 +59,8 @@ argparser = optparse.OptionParser(
          "%prog <noderange> [list][updatestatus][update [--backup <file>]]|[<components>]")
 argparser.add_option('-b', '--backup', action='store_true',
                     help='Target a backup bank rather than primary')
+argparser.add_option('-p', '--parameterfile', type='string',
+                     help='When updating, use the specified parameter file')
 argparser.add_option('-m', '--maxnodes', type='int',
                     help='When updating, prompt if more than the specified '
                          'number of servers will be affected')
@@ -112,6 +114,10 @@ def update_firmware(session, filename):
    upargs = {'filename': filename}
    if options.backup:
        upargs['bank'] = 'backup'
+    if options.parameterfile:
+        with open(options.parameterfile, 'rb') as pf:
+            pfdata = pf.read()
+        upargs['parameterdata'] = pfdata
    noderrs = {}
    if session.unixdomain:
        filesbynode = {}
@@ -32,7 +32,7 @@ if path.startswith('/opt'):
 import confluent.client as client

 argparser = optparse.OptionParser(
-    usage='Usage: %prog [options] <noderange> [default|cd|network|setup|hd|usb|floppy]')
+    usage='Usage: %prog [options] <noderange> [default|cd|network|http|setup|hd|usb|floppy]')
 argparser.add_option('-b', '--bios', dest='biosmode',
                     action='store_true', default=False,
                     help='Request BIOS style boot (rather than UEFI)')
@@ -281,6 +281,7 @@ class Command(object):
        if maxnodes is None:
            return
        nsize = self.get_noderange_size(noderange)
+        maxnodes = int(maxnodes)
        if nsize > maxnodes:
            if nsize == 1:
                nodename = list(self.read(
@@ -391,8 +392,13 @@ class Command(object):
            cacert = None
            certreqs = ssl.CERT_NONE
            knownhosts = True
-        self.connection = ssl.wrap_socket(self.connection, ca_certs=cacert,
-                                          cert_reqs=certreqs)
+        tlsctx = ssl.create_default_context()
+        if certreqs == ssl.CERT_NONE:
+            tlsctx.check_hostname = False
+        tlsctx.verify_mode = certreqs
+        if cacert:
+            tlsctx.load_verify_locations(cacert)
+        self.connection = tlsctx.wrap_socket(self.connection, server_hostname=server)
        if knownhosts:
            certdata = self.connection.getpeercert(binary_form=True)
            fingerprint = 'sha512$' + hashlib.sha512(certdata).hexdigest()
@@ -35,7 +35,10 @@ the out of band facilities.  Firmware updates can end in one of three states:
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:
  When updating, prompt if more than the specified number of servers will 
  be affected  
-  
+
+* `-p PARAMETERFILE`, `--paramaterfile=PARAMETERFILE`:
+  For updating, a parameter file to provife along with the update payload
+
 * `-h`, `--help`:
  Show help message and exit    

@@ -703,4 +703,7 @@ if __name__ == '__main__':
    elif checkonly:
        sys.stdout.write(mclient.check_connections())
    else:
-        sys.stdout.write(mclient.grab_url(sys.argv[1], data).decode())
+        try:
+            sys.stdout.buffer.write(mclient.grab_url(sys.argv[1], data))
+        except AttributeError:
+            sys.stdout.write(mclient.grab_url(sys.argv[1], data))
@@ -10,11 +10,13 @@
 #   serial port is not reporting DCD, then it doesn't look like a comfortable enough scenario

 import fcntl
+import glob
 import os
 import os.path
 import struct
 import subprocess
 import termios
+import platform


 addrtoname = {
@@ -74,9 +76,8 @@ def fixup_ubuntu_grub_serial():
            grubout.write(grubline + '\n')
    subprocess.check_call(['update-grub'])

-def get_serial_config():
-    if not os.path.exists('/sys/firmware/efi'):
-        return None
+
+def get_spcr_config():
    if not os.path.exists('/sys/firmware/acpi/tables/SPCR'):
        return None
    spcr = open("/sys/firmware/acpi/tables/SPCR", "rb")
@@ -99,14 +100,39 @@ def get_serial_config():
        currattr = termios.tcgetattr(ttyf)
        currattr[4:6] = [0, termiobaud[retval['speed']]]
        termios.tcsetattr(ttyf, termios.TCSANOW, currattr)
-    retval['connected'] = bool(struct.unpack('<I', fcntl.ioctl(
-        ttyf, termios.TIOCMGET, '\x00\x00\x00\x00'))[0] & termios.TIOCM_CAR)
-    os.close(ttyf)
    return retval

+def get_serial_config():
+    if platform.machine() != 'x86_64':
+        return None  # Trust non-x86 to do the right thing
+    if not os.path.exists('/sys/firmware/efi'):
+        return None  # BIOS might fail at grub output, defer to stock OS behavior
+    retval = get_spcr_config()
+    if retval:
+        return retval
+    firstfound = None
+    numpossible = 0
+    numconnected = 0
+    for serdev in glob.glob('/dev/ttyS*'):
+        ttyf = os.open(serdev, os.O_RDWR | os.O_NOCTTY)
+        try:
+            statusreg = fcntl.ioctl(ttyf, termios.TIOCMGET, '\x00\x00\x00\x00')
+            numpossible += 1
+            if not firstfound:
+                firstfound = serdev
+        except Exception:
+            continue
+        finally:
+            os.close(ttyf)
+        if struct.unpack('<I', statusreg)[0] & termios.TIOCM_CAR:
+            numconnected += 1
+            firstfound = serdev
+    if numpossible == 1 or numconnected == 1: 
+        return { 'tty': firstfound, 'speed': 115200 }
+
 def main():
    autoconscfg = get_serial_config()
-    if not autoconscfg or not autoconscfg['connected']:
+    if not autoconscfg:
        return
    if os.path.exists('/etc/redhat-release'):  # redhat family
        deserialize_grub_rh()
@@ -80,14 +80,18 @@ def await_tentative():
        time.sleep(1)

 def map_idx_to_name():
-    map = {}
+    map_dict = {}
    devtype = {}
    prevdev = None
+
    for line in subprocess.check_output(['ip', 'l']).decode('utf8').splitlines():
-        if line.startswith(' ') and 'link/' in line:
-            typ = line.split()[0].split('/')[1]
-            devtype[prevdev] = typ if typ != 'ether' else 'ethernet'
        if line.startswith(' '):
+            if 'link/' in line and prevdev and prevdev not in devtype:
+                for word in line.split():
+                    if word.startswith('link/'):
+                        typ = word.split('/')[1]
+                        devtype[prevdev] = typ if typ != 'ether' else 'ethernet'
+                        break # Stop detection after the first type hit
            continue
        idx, iface, rst = line.split(':', 2)
        prevdev = iface.strip()
@@ -99,9 +103,8 @@ def map_idx_to_name():
            pass
        idx = int(idx)
        iface = iface.strip()
-        map[idx] = iface
-    return map, devtype
-
+        map_dict[idx] = iface
+    return map_dict, devtype

 def get_interface_name(iname, settings):
    explicitname = settings.get('interface_names', None)
@@ -114,8 +117,10 @@ def get_interface_name(iname, settings):
 class NetplanManager(object):
    def __init__(self, deploycfg):
        self.cfgbydev = {}
+        self.cfgbybond = {}
        self.read_connections()
        self.deploycfg = deploycfg
+        self.teamidx = 0

    def read_connections(self):
        for plan in glob.glob('/etc/netplan/*.y*ml'):
@@ -124,29 +129,61 @@ class NetplanManager(object):
                if not planinfo:
                    continue
            nicinfo = planinfo.get('network', {}).get('ethernets', {})
-            for devname in nicinfo:
-                if devname == 'lo':
-                    continue
-                if 'gateway4' in nicinfo[devname]:
-                    # normalize deprecated syntax on read in
-                    gw4 = nicinfo[devname]['gateway4']
-                    del nicinfo[devname]['gateway4']
-                    routeinfo = nicinfo[devname].get('routes', [])
-                    for ri in routeinfo:
-                        if ri.get('via', None) == gw4 and ri.get('to', None) in ('default', '0.0.0.0/0', '0/0'):
-                            break
-                    else:
-                        routeinfo.append({
-                            'to': 'default',
-                            'via': gw4
-                        })
-                    nicinfo[devname]['routes'] = routeinfo
-                self.cfgbydev[devname] = nicinfo[devname]
+            bondinfo = planinfo.get('network', {}).get('bonds', {})
+            for currinfo in (nicinfo, bondinfo):
+                currcfg = self.cfgbydev if currinfo is nicinfo else self.cfgbybond
+                for devname in currinfo:
+                    if devname == 'lo':
+                        continue
+                    if 'gateway4' in currinfo[devname]:
+                        # normalize deprecated syntax on read in
+                        gw4 = currinfo[devname]['gateway4']
+                        del currinfo[devname]['gateway4']
+                        routeinfo = currinfo[devname].get('routes', [])
+                        for ri in routeinfo:
+                            if ri.get('via', None) == gw4 and ri.get('to', None) in ('default', '0.0.0.0/0', '0/0'):
+                                break
+                        else:
+                            routeinfo.append({
+                                'to': 'default',
+                                'via': gw4
+                            })
+                        currinfo[devname]['routes'] = routeinfo
+                    currcfg[devname] = currinfo[devname]
    
    def apply_configuration(self, cfg):
        devnames = cfg['interfaces']
-        if len(devnames) != 1:
-            raise Exception('Multi-nic team/bonds not yet supported')
+        if len(devnames) > 1:
+            teammode = cfg['settings'].get('team_mode', None)
+            if not teammode:
+                sys.stderr.write("Warning, multiple interfaces ({0}) without a team_mode, skipping setup\n".format(','.join(devnames)))
+                return
+            if teammode == 'lacp':
+                teammode = '802.3ad'
+            elif teammode == 'activebackup':
+                teammode = 'active-backup'
+            for currdev in self.cfgbybond:
+                for iface in self.cfgbybond[currdev].get('interfaces', []):
+                    if iface in devnames:
+                        break
+                    else:
+                        continue
+                else:
+                    continue
+                # this bond is identified as matching
+                self.cfgbybond[currdev]['interfaces'] = list(devnames)
+                self.cfgbybond[currdev]['parameters']['mode'] = teammode
+                devnames = [currdev]
+                break
+            # no current bond, make a new one
+            connname = cfg['settings'].get('connection_name', None)
+            if not connname:
+                connname = 'bond{0}'.format(self.teamidx)
+                while connname in self.cfgbybond:
+                    self.teamidx += 1
+                    connname = 'bond{0}'.format(self.teamidx)
+            self.cfgbybond[connname] = {'interfaces': list(devnames), 'parameters': {'mode': teammode, 'mii-monitor-interval': 100}}
+            devnames = [connname]
        stgs = cfg['settings']
        needcfgapply = False
        for devname in devnames:
@@ -171,7 +208,7 @@ class NetplanManager(object):
            gws.append(stgs.get('ipv4_gateway', None))
            gws.append(stgs.get('ipv6_gateway', None))
            for gwaddr in gws:
-                if gwaddr:
+                if gwaddr and gwaddr != '0.0.0.0':
                    cfgroutes = self.getcfgarrpath([devname, 'routes'])
                    for rinfo in cfgroutes:
                        if rinfo.get('via', None) == gwaddr:
@@ -192,18 +229,56 @@ class NetplanManager(object):
                if dnsdomain not in currdnsdomain:
                    needcfgwrite = True
                    currdnsdomain.append(dnsdomain)
+            prune_from_cloudinit = []
            if needcfgwrite:
                needcfgapply = True
-                newcfg = {'network': {'version': 2, 'ethernets': {devname: self.cfgbydev[devname]}}}
                oumask = os.umask(0o77)
-                with open('/etc/netplan/{0}-confluentcfg.yaml'.format(devname), 'w') as planout:
+                if devname in self.cfgbydev:
+                    prune_from_cloudinit.append(devname)
+                    newcfg = {'network': {'version': 2, 'ethernets': {devname: self.cfgbydev[devname]}}}
+                    cfgfile = '/etc/netplan/10-{0}-confluentcfg.yaml'.format(devname)
+                elif devname in self.cfgbybond:
+                    newcfg = {'network': {'version': 2, 'bonds': {devname: self.cfgbybond[devname]}}}
+                    for iface in newcfg['network']['bonds'][devname]['interfaces']:
+                        prune_from_cloudinit.append(iface)
+                        with open('/etc/netplan/10-{0}-confluentcfg.yaml'.format(iface), 'w') as planout:
+                            planout.write(yaml.dump({'network': {'version': 2, 'ethernets': {iface: {'dhcp4': False}}}}))
+                    cfgfile = '/etc/netplan/30-{0}-confluentcfg.yaml'.format(devname)
+                with open(cfgfile, 'w') as planout:
                    planout.write(yaml.dump(newcfg))
                os.umask(oumask)
+        if prune_from_cloudinit:
+            prunecfgs = ['/etc/netplan/00-installer-config.yaml',
+                         '/etc/netplan/50-cloud-init.yaml.dist-subiquity',
+                         '/etc/netplan/50-cloud-init.yaml']
+            prunecfgs.extend(glob.glob('/etc/cloud/cloud.cfg.d/*.cfg'))
+            for defcfg in prunecfgs:
+                if not os.path.exists(defcfg):
+                    continue
+                with open(defcfg, 'r') as cloudinit:
+                    cloudinfo = yaml.safe_load(cloudinit)
+                    if 'network' not in cloudinfo:
+                        continue
+                    for clouddev in list(cloudinfo.get('network', {}).get('ethernets', {})):
+                        if clouddev in prune_from_cloudinit:
+                            del cloudinfo['network']['ethernets'][clouddev]
+                    if not cloudinfo['network'].get('ethernets', {}):
+                        os.remove(defcfg)
+                        if '/etc/cloud/cloud.cfg.d/' in defcfg:
+                            # need to also change datasource, probably
+                            if os.path.exists('/var/lib/cloud/instances/iid-datasource-none/network-config.json'):
+                                os.remove('/var/lib/cloud/instances/iid-datasource-none/network-config.json')
+                    else:
+                        oumask = os.umask(0o77)
+                        with open(defcfg, 'w') as cloudinit:
+                            cloudinit.write(yaml.dump(cloudinfo))
+                        os.umask(oumask)
        if needcfgapply:
+            subprocess.check_call(['netplan', 'generate'])
            subprocess.call(['netplan', 'apply'])

    def getcfgarrpath(self, devpath):
-        currptr = self.cfgbydev
+        currptr = self.cfgbybond if devpath[0] in self.cfgbybond else self.cfgbydev
        for k in devpath[:-1]:
            if k not in currptr:
                currptr[k] = {}
@@ -55,4 +55,9 @@ confluentpython $confapiclient /confluent-api/self/nodelist | sed -e 's/^- //' >
 cat /etc/ssh/shosts.equiv > /root/.shosts
 cd -
 rm -rf $TMPDIR
-systemctl try-restart sshd
+# ssh may be sshd or ssh, depending
+if systemctl list-unit-files | grep -q '^sshd\.service'; then
+    systemctl try-restart sshd
+elif systemctl list-unit-files | grep -q '^ssh\.service'; then
+    systemctl try-restart ssh
+fi
@@ -26,12 +26,12 @@ mkdir -p opt/confluent/bin
 mkdir -p stateless-bin
 cp -a el8bin/* .
 ln -s el8 el9
-ln -s el8 el10
+cp -a el8 el10
 cp -a debian debian13
 mkdir -p debian13/initramfs/usr
 mv debian13/initramfs/lib debian13/initramfs/usr/
 mv el10/initramfs/usr el10/initramfs/var
-for os in rhvh4 el7 genesis el8 suse15 debian debian13 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9 el10; do
+for os in rhvh4 el7 genesis el8 suse15 debian debian13 ubuntu20.04 ubuntu22.04 ubuntu24.04 ubuntu26.04 coreos el9 el10; do
    mkdir ${os}out
    cd ${os}out
    if [ -d ../${os}bin ]; then 
@@ -47,7 +47,7 @@ for os in rhvh4 el7 genesis el8 suse15 debian debian13 ubuntu20.04 ubuntu22.04 u
    mv ../addons.cpio .
    cd ..
 done
-for os in el7 el8 suse15 el9 el10 ubuntu20.04; do
+for os in el7 el8 suse15 el9 el10 ubuntu20.04 ubuntu22.04 ubuntu24.04 ubuntu26.04; do
    mkdir ${os}disklessout
    cd ${os}disklessout
    if [ -d ../${os}bin ]; then
@@ -88,7 +88,7 @@ mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
 mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/common
 cp common/initramfs/opt/confluent/bin/apiclient %{buildroot}/opt/confluent/lib/osdeploy/common

-for os in rhvh4 el7 el8 el9 el10 debian debian13 genesis suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 esxi6 esxi7 esxi8 coreos; do
+for os in rhvh4 el7 el8 el9 el10 debian debian13 genesis suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 ubuntu26.04 esxi6 esxi7 esxi8 coreos; do
    mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs/aarch64/
    cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs/aarch64/
    if [ -d ${os}disklessout ]; then
@@ -33,7 +33,7 @@ cp -a debian debian13
 mkdir -p debian13/initramfs/usr
 mv debian13/initramfs/lib debian13/initramfs/usr/
 mv el10/initramfs/usr el10/initramfs/var
-for os in rhvh4 el7 genesis el8 suse15 debian debian13 ubuntu18.04 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9 el10; do
+for os in rhvh4 el7 genesis el8 suse15 debian debian13 ubuntu18.04 ubuntu20.04 ubuntu22.04 ubuntu24.04 ubuntu26.04 coreos el9 el10; do
    mkdir ${os}out
    cd ${os}out
    if [ -d ../${os}bin ]; then 
@@ -49,7 +49,7 @@ for os in rhvh4 el7 genesis el8 suse15 debian debian13 ubuntu18.04 ubuntu20.04 u
    mv ../addons.cpio .
    cd ..
 done
-for os in el7 el8 suse15 el9 el10 ubuntu20.04 ubuntu22.04 ubuntu24.04; do
+for os in el7 el8 suse15 el9 el10 ubuntu20.04 ubuntu22.04 ubuntu24.04 ubuntu26.04; do
    mkdir ${os}disklessout
    cd ${os}disklessout
    if [ -d ../${os}bin ]; then
@@ -89,7 +89,7 @@ cp -a esxi7 esxi9
 %install
 mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
 cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
-for os in rhvh4 el7 el8 el9 el10 genesis suse15 ubuntu20.04 debian debian13 ubuntu18.04 ubuntu22.04 ubuntu24.04 esxi6 esxi7 esxi8 esxi9 coreos; do
+for os in rhvh4 el7 el8 el9 el10 genesis suse15 ubuntu20.04 debian debian13 ubuntu18.04 ubuntu22.04 ubuntu24.04 ubuntu26.04 esxi6 esxi7 esxi8 esxi9 coreos; do
    mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
    mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
    cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
@@ -110,14 +110,22 @@ else
    for nic in $(ip link | grep mtu|grep -v LOOPBACK|cut -d: -f 2|sed -e 's/ //'); do
        ip link set $nic up
    done
-    for nic in $(ip link | grep mtu|grep -v LOOPBACK|grep LOWER_UP|cut -d: -f 2|sed -e 's/ //'); do
-        if udhcpc -i $nic -p $dhcpid -t 2 -T 2 -n -x 93:0007 -x 97:00$dhuuid -q; then
-            /opt/confluent/bin/copernicus > /etc/confluent/confluent.info
-            if grep ^MANAGER:.*\\. /etc/confluent/confluent.info ; then
-                break
+    nic=
+    while [ -z "$nic" ]; do
+        for nic in $(ip link | grep mtu|grep -v LOOPBACK|grep LOWER_UP|cut -d: -f 2|sed -e 's/ //'); do
+            if udhcpc -i $nic -p $dhcpid -t 2 -T 2 -n -x 93:0007 -x 97:00$dhuuid -q; then
+                /opt/confluent/bin/copernicus > /etc/confluent/confluent.info
+                if grep ^MANAGER:.*\\. /etc/confluent/confluent.info ; then
+                    break
+                fi
            fi
+            ip -4 address flush dev $nic
+            nic=""
+        done
+        if [ -z "$nic" ]; then
+            echo "No network interface could be detected, retrying...."
+            sleep 2
        fi
-        ip -4 flush dev $nic
    done
    mgr=$(grep ^MANAGER:.*\\. /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
    nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
@@ -21,17 +21,10 @@ if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
 fi

 if [ -f /tmp/timeservers ]; then
-
-ntpsrvs=$(cat /tmp/timeservers)
-
-sed -i "1,/^pool * /c\\
-
-${ntpsrvs//$'\n'/\\$'\n'}" /etc/chrony.conf
-
-
-systemctl restart chronyd
-
-rm -f /tmp/timeservers
+    sed -i '/^[[:space:]]*\(pool\|server\)[[:space:]]/d' /etc/chrony.conf
+    cat /tmp/timeservers >> /etc/chrony.conf
+    systemctl restart chronyd
+    rm -f /tmp/timeservers
 fi


@@ -27,17 +27,10 @@ if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
 fi

 if [ -f /tmp/timeservers ]; then
-
-ntpsrvs=$(cat /tmp/timeservers)
-
-sed -i "1,/^pool * /c\\
-${ntpsrvs//$'\n'/\\$'\n'}" /etc/chrony.conf
-
-
-systemctl restart chronyd
-
-rm -f /tmp/timeservers
-
+    sed -i '/^[[:space:]]*\(pool\|server\)[[:space:]]/d' /etc/chrony.conf
+    cat /tmp/timeservers >> /etc/chrony.conf
+    systemctl restart chronyd
+    rm -f /tmp/timeservers
 fi


@@ -26,17 +26,10 @@ if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
 fi

 if [ -f /tmp/timeservers ]; then
-
-ntpsrvs=$(cat /tmp/timeservers)
-
-sed -i "1,/^pool * /c\\
-
-${ntpsrvs//$'\n'/\\$'\n'}" /etc/chrony.conf
-
-
-systemctl restart chronyd
-
-rm -f /tmp/timeservers
+    sed -i '/^[[:space:]]*\(pool\|server\)[[:space:]]/d' /etc/chrony.conf
+    cat /tmp/timeservers >> /etc/chrony.conf
+    systemctl restart chronyd
+    rm -f /tmp/timeservers
 fi

 export nodename confluent_mgr confluent_profile
@@ -0,0 +1,29 @@
+# It is advised to avoid /var/lib/confluent/public as a source for syncing.  /var/lib/confluent/public
+# is served without authentication and thus any sensitive content would be a risk. If wanting to host
+# syncfiles on a common share, it is suggested to have /var/lib/confluent be the share and use some other
+# subdirectory other than public.
+#
+# Syncing is performed as the 'confluent' user, so all source files must be accessible by the confluent user.
+#
+# This file lists files to synchronize or merge to the deployed systems from the deployment server
+# To specify taking /some/path/hosts on the deployment server and duplicating it to /etc/hosts:
+# Note particularly the use of '->' to delineate source from target.
+# /some/path/hosts -> /etc/hosts
+
+# If wanting to simply use the same path for source and destinaiton, the -> may be skipped:
+# /etc/hosts
+
+# More function is available, for example to limit the entry to run only on n1 through n8, and to set
+# owner, group, and permissions in octal notation:
+# /example/source -> n1-n8:/etc/target (owner=root,group=root,permissions=600)
+
+# Entries under APPENDONCE: will be added to specified target, only if the target does not already
+# contain the data in the source already in its entirety. This allows append in a fashion that
+# is friendly to being run repeatedly
+
+# Entries under MERGE: will attempt to be intelligently merged. This supports /etc/group and /etc/passwd
+# Any supporting entries in /etc/shadow or /etc/gshadow are added automatically, with password disabled
+# It also will not inject 'system' ids (under 1,000 usually) as those tend to be local and rpm managed.
+MERGE:
+# /etc/passwd
+# /etc/group
@@ -8,7 +8,9 @@ for addr in $(grep ^MANAGER: /etc/confluent/confluent.info|awk '{print $2}'|sed
    fi
 done
 mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
-if grep confluent_imagemethod=untethered /proc/cmdline > /dev/null; then
+TETHERED=1
+if grep -q confluent_imagemethod=untethered /proc/cmdline || grep -q confluent_imagemethod=uncompressed /proc/cmdline; then
+    TETHERED=0
    mount -t tmpfs untethered /mnt/remoteimg
    curl https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
 else
@@ -44,15 +46,46 @@ fi


 #mount -t tmpfs overlay /mnt/overlay
-modprobe zram
-memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
-memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
-echo $memtot > /sys/block/zram0/disksize
-modprobe xfs
 mkdir /sysroot
-mkfs.xfs /dev/zram0 > /dev/null
-mount -o discard /dev/zram0 /mnt/overlay
-if [ ! -f /tmp/mountparts.sh ]; then
+if ! grep -q confluent_imagemethod=uncompressed /proc/cmdline; then
+    modprobe zram
+    memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
+    memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
+    echo $memtot > /sys/block/zram0/disksize
+    modprobe xfs
+    mkfs.xfs /dev/zram0 > /dev/null
+    if [ "$TETHERED" = 1 ]; then
+         mount -o discard /dev/zram0 /mnt/overlay
+    else
+         mount -o discard /dev/zram0 /sysroot
+    fi
+    mount -o discard /dev/zram0 /mnt/overlay
+elif grep -q confluent_imagemethod=uncompressed /proc/cmdline; then
+    mount -t tmpfs disklessroot /sysroot
+fi
+if [ "$TETHERED" = 0 ]; then
+    echo -en "Decrypting and extracting root filesystem: 0%\r"
+        srcsz=$(du -sk /mnt/remote | awk '{print $1}')
+        while [ -f /mnt/remoteimg/rootimg.sfs ]; do
+        dstsz=$(du -sk /sysroot | awk '{print $1}')
+        pct=$((dstsz * 100 / srcsz))
+        if [ $pct -gt 99 ]; then
+            pct=99
+        fi
+        echo -en "Decrypting and extracting root filesystem: $pct%\r"
+        sleep 0.25
+    done &
+    cp -a /mnt/remote/* /sysroot/
+    umount /mnt/remote
+    if [ -e /dev/mapper/cryptimg ]; then
+        dmsetup remove cryptimg
+    fi
+    losetup -d $loopdev
+    rm /mnt/remoteimg/rootimg.sfs
+    umount /mnt/remoteimg
+    wait
+    echo -e "Decrypting and extracting root filesystem: 100%"
+elif [ ! -f /tmp/mountparts.sh ]; then
    mkdir -p /mnt/overlay/upper /mnt/overlay/work
    mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
 else
@@ -68,6 +101,7 @@ cp /root/.ssh/* /sysroot/root/.ssh
 chmod 700 /sysroot/root/.ssh
 cp /etc/confluent/* /sysroot/etc/confluent/
 cp /etc/ssh/*key* /sysroot/etc/ssh/
+cp /tls/* /sysroot/etc/ssl/certs
 for pubkey in /etc/ssh/ssh_host*key.pub; do
    certfile=${pubkey/.pub/-cert.pub}
    privfile=${pubkey%.pub}
@@ -0,0 +1,29 @@
+# It is advised to avoid /var/lib/confluent/public as a source for syncing.  /var/lib/confluent/public
+# is served without authentication and thus any sensitive content would be a risk. If wanting to host
+# syncfiles on a common share, it is suggested to have /var/lib/confluent be the share and use some other
+# subdirectory other than public.
+#
+# Syncing is performed as the 'confluent' user, so all source files must be accessible by the confluent user.
+#
+# This file lists files to synchronize or merge to the deployed systems from the deployment server
+# To specify taking /some/path/hosts on the deployment server and duplicating it to /etc/hosts:
+# Note particularly the use of '->' to delineate source from target.
+# /some/path/hosts -> /etc/hosts
+
+# If wanting to simply use the same path for source and destinaiton, the -> may be skipped:
+# /etc/hosts
+
+# More function is available, for example to limit the entry to run only on n1 through n8, and to set
+# owner, group, and permissions in octal notation:
+# /example/source -> n1-n8:/etc/target (owner=root,group=root,permissions=600)
+
+# Entries under APPENDONCE: will be added to specified target, only if the target does not already
+# contain the data in the source already in its entirety. This allows append in a fashion that
+# is friendly to being run repeatedly
+
+# Entries under MERGE: will attempt to be intelligently merged. This supports /etc/group and /etc/passwd
+# Any supporting entries in /etc/shadow or /etc/gshadow are added automatically, with password disabled
+# It also will not inject 'system' ids (under 1,000 usually) as those tend to be local and rpm managed.
+MERGE:
+# /etc/passwd
+# /etc/group
@@ -7,7 +7,7 @@ exec >> /target/var/log/confluent/confluent-firstboot.log
 exec 2>> /target/var/log/confluent/confluent-firstboot.log
 chmod 600 /target/var/log/confluent/confluent-firstboot.log
 cp -a /etc/confluent/ssh/* /etc/ssh/
-systemctl restart sshd
+systemctl restart ssh
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 if [ ! -z "$rootpw" -a "$rootpw" != "null" ]; then
 	echo root:$rootpw | chpasswd -e
@@ -33,7 +33,7 @@ done
 echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/confluent.conf
 echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.conf
 echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf
-systemctl restart sshd
+systemctl restart ssh
 mkdir -p /etc/confluent
 export nodename confluent_profile confluent_mgr
 curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
@@ -0,0 +1,29 @@
+# It is advised to avoid /var/lib/confluent/public as a source for syncing.  /var/lib/confluent/public
+# is served without authentication and thus any sensitive content would be a risk. If wanting to host
+# syncfiles on a common share, it is suggested to have /var/lib/confluent be the share and use some other
+# subdirectory other than public.
+#
+# Syncing is performed as the 'confluent' user, so all source files must be accessible by the confluent user.
+#
+# This file lists files to synchronize or merge to the deployed systems from the deployment server
+# To specify taking /some/path/hosts on the deployment server and duplicating it to /etc/hosts:
+# Note particularly the use of '->' to delineate source from target.
+# /some/path/hosts -> /etc/hosts
+
+# If wanting to simply use the same path for source and destinaiton, the -> may be skipped:
+# /etc/hosts
+
+# More function is available, for example to limit the entry to run only on n1 through n8, and to set
+# owner, group, and permissions in octal notation:
+# /example/source -> n1-n8:/etc/target (owner=root,group=root,permissions=600)
+
+# Entries under APPENDONCE: will be added to specified target, only if the target does not already
+# contain the data in the source already in its entirety. This allows append in a fashion that
+# is friendly to being run repeatedly
+
+# Entries under MERGE: will attempt to be intelligently merged. This supports /etc/group and /etc/passwd
+# Any supporting entries in /etc/shadow or /etc/gshadow are added automatically, with password disabled
+# It also will not inject 'system' ids (under 1,000 usually) as those tend to be local and rpm managed.
+MERGE:
+# /etc/passwd
+# /etc/group
@@ -37,10 +37,20 @@ else
        chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
        APIKEY=$(cat /root/custom-installation/confluent/confluent.apikey)
    done
-    MGR=[$MGR]
-    nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
-    nic=$(ip link |grep ^$nic:|awk '{print $2}')
-    DEVICE=${nic%:}
+    if echo "$MGR" | grep -q ':'; then
+        # IPv6 manager: wrap address in brackets and resolve interface from scoped manager entry.
+        MGR=[$MGR]
+        nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
+        nic=$(ip link |grep ^$nic:|awk '{print $2}')
+        DEVICE=${nic%:}
+    else
+        # IPv4 routed deployment: use previously detected NIC, fallback to route lookup.
+        if [ -f /tmp/autodetectnic ]; then
+            DEVICE=$(cat /tmp/autodetectnic)
+        else
+            DEVICE=$(ip route get ${MGR} 2>/dev/null | head -1 | sed -n 's/.*dev \([^ ]*\).*/\1/p')
+        fi
+    fi
    IP=done
 fi
 if [ -z "$MGTIFACE" ]; then
@@ -97,6 +97,62 @@ while ! grep NODENAME /custom-installation/confluent/confluent.info; do
        echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
        cd -
        umount $tmnt
+    elif confluentsrv=$(sed -n 's/.*confluent=\([^ ]*\).*/\1/p' /proc/cmdline); [ ! -z "$confluentsrv" ]; then
+        echo "confluent= kernel arg found: $confluentsrv" > /dev/console 2>&1
+        . /scripts/functions
+
+        rmmod cdc_ether 2> /dev/null
+        rm -rf /run/net* /run/dhcpcd /var/lib/dhcpcd
+        for dev in $(ip a|grep MULTICAST|awk '{print $2}'|sed -e s/://); do
+            ip a flush $dev
+            echo 1 > /proc/sys/net/ipv6/conf/$dev/addr_gen_mode 2>/dev/null
+            echo 0 > /proc/sys/net/ipv6/conf/$dev/addr_gen_mode 2>/dev/null
+        done
+        unset DEVICE DEVICE6 IP IP6 dev
+
+        echo "Starting DHCP configure_networking..." > /dev/console 2>&1
+        configure_networking
+        echo "DHCP done, DEVICE=$DEVICE" > /dev/console 2>&1
+        echo $DEVICE > /tmp/autodetectnic
+
+        RETRIES=0
+        while [ $RETRIES -lt 5 ]; do
+            if openssl s_client -connect $confluentsrv:443 </dev/null > /dev/null 2>&1; then
+                echo "TLS connectivity to $confluentsrv OK" > /dev/console 2>&1
+                break
+            fi
+            RETRIES=$((RETRIES + 1))
+            echo "Cannot reach $confluentsrv:443, retry $RETRIES/5..." > /dev/console 2>&1
+            sleep 3
+        done
+
+        if [ $RETRIES -ge 5 ]; then
+            echo "Failed to reach $confluentsrv after 5 retries, falling back to copernicus" > /dev/console 2>&1
+            /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+            continue
+        fi
+
+        myids="uuid=$(cat /sys/devices/virtual/dmi/id/product_uuid)"
+        for mac in $(ip link | grep 'link/ether' | awk '{print $2}'); do
+            myids="$myids/mac=$mac"
+        done
+        echo "Calling whoami with IDs: $myids" > /dev/console 2>&1
+
+        myname=$( (printf "GET /confluent-api/self/whoami HTTP/1.0\r\nHost: $confluentsrv\r\nCONFLUENT_IDS: $myids\r\n\r\n"; sleep 3) \
+            | openssl s_client -connect $confluentsrv:443 -quiet 2>/dev/null \
+            | tail -1 | tr -d '\r\n')
+
+        echo "whoami returned: '$myname'" > /dev/console 2>&1
+
+        if [ ! -z "$myname" ]; then
+            MGR=$confluentsrv
+            echo "NODENAME: $myname" > /custom-installation/confluent/confluent.info
+            echo "MANAGER: $confluentsrv" >> /custom-installation/confluent/confluent.info
+            echo "EXTMGRINFO: $confluentsrv||1" >> /custom-installation/confluent/confluent.info
+        else
+            echo "whoami returned empty, retrying in 10s..." > /dev/console 2>&1
+            sleep 10
+        fi
    else
        /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
    fi
@@ -23,6 +23,7 @@ touch /etc/cloud/cloud-init.disabled
 source /etc/confluent/functions
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 export confluent_mgr confluent_profile
+run_remote_python confignet
 run_remote_parts firstboot.d
 run_remote_config firstboot.d
 curl --capath /etc/confluent/tls -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus
@@ -89,7 +89,6 @@ chroot /target update-ca-certificates
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python autoconsole"

 chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient"
-chroot /target bash -c "source /etc/confluent/functions; run_remote_python confignet"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d"
 source /target/etc/confluent/functions

@@ -28,7 +28,7 @@ done
 echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/confluent.conf
 echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.conf
 echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf
-systemctl restart sshd
+systemctl restart ssh
 mkdir -p /etc/confluent
 export nodename confluent_profile confluent_mgr
 curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
@@ -0,0 +1,29 @@
+# It is advised to avoid /var/lib/confluent/public as a source for syncing.  /var/lib/confluent/public
+# is served without authentication and thus any sensitive content would be a risk. If wanting to host
+# syncfiles on a common share, it is suggested to have /var/lib/confluent be the share and use some other
+# subdirectory other than public.
+#
+# Syncing is performed as the 'confluent' user, so all source files must be accessible by the confluent user.
+#
+# This file lists files to synchronize or merge to the deployed systems from the deployment server
+# To specify taking /some/path/hosts on the deployment server and duplicating it to /etc/hosts:
+# Note particularly the use of '->' to delineate source from target.
+# /some/path/hosts -> /etc/hosts
+
+# If wanting to simply use the same path for source and destinaiton, the -> may be skipped:
+# /etc/hosts
+
+# More function is available, for example to limit the entry to run only on n1 through n8, and to set
+# owner, group, and permissions in octal notation:
+# /example/source -> n1-n8:/etc/target (owner=root,group=root,permissions=600)
+
+# Entries under APPENDONCE: will be added to specified target, only if the target does not already
+# contain the data in the source already in its entirety. This allows append in a fashion that
+# is friendly to being run repeatedly
+
+# Entries under MERGE: will attempt to be intelligently merged. This supports /etc/group and /etc/passwd
+# Any supporting entries in /etc/shadow or /etc/gshadow are added automatically, with password disabled
+# It also will not inject 'system' ids (under 1,000 usually) as those tend to be local and rpm managed.
+MERGE:
+# /etc/passwd
+# /etc/group
@@ -0,0 +1 @@
+ubuntu22.04/
@@ -0,0 +1 @@
+ubuntu20.04-diskless/
@@ -213,97 +213,101 @@ int main(int argc, char* argv[]) {
                memset(msg, 0, 1024);
                /* Deny packet access to the last 24 bytes to assure null */
                recvfrom(n4, msg, 1000, 0, (struct sockaddr *)&dst4, &dst4size);
-                if  (nodenameidx = strstr(msg, "NODENAME: ")) {
+                if (strstr(msg, "HTTP/1.1 200 OK")) {
+                    if  (nodenameidx = strstr(msg, "NODENAME: ")) {
+                            nodenameidx += 10;
+                            strncpy(nodename, nodenameidx, 1024);
+                            nodenameidx = strstr(nodename, "\r");
+                            if (nodenameidx) { nodenameidx[0] = 0; }
+                            if (strncmp(lastnodename, nodename, 1024) != 0) {
+                                printf("NODENAME: %s\n", nodename);
+                                strncpy(lastnodename, nodename, 1024);
+                            }
+                    }
+                    if (nodenameidx = strstr(msg, "CURRTIME: ")) {
                        nodenameidx += 10;
                        strncpy(nodename, nodenameidx, 1024);
-                        nodenameidx = strstr(nodename, "\r");
-                        if (nodenameidx) { nodenameidx[0] = 0; }
-                        if (strncmp(lastnodename, nodename, 1024) != 0) {
-                            printf("NODENAME: %s\n", nodename);
-                            strncpy(lastnodename, nodename, 1024);
+                        if (nodenameidx = strstr(nodename, "\r")) {
+                            nodenameidx[0] = 0;
                        }
-                }
-                if (nodenameidx = strstr(msg, "CURRTIME: ")) {
-                    nodenameidx += 10;
-                    strncpy(nodename, nodenameidx, 1024);
-                    if (nodenameidx = strstr(nodename, "\r")) {
-                        nodenameidx[0] = 0;
+                        settime = strtol(nodename, NULL, 10);
                    }
-                    settime = strtol(nodename, NULL, 10);
-                }
-                if (nodenameidx = strstr(msg, "CURRMSECS: ")) {
-                    nodenameidx += 10;
-                    strncpy(nodename, nodenameidx, 1024);
-                    if (nodenameidx = strstr(nodename, "\r")) {
-                        nodenameidx[0] = 0;
+                    if (nodenameidx = strstr(msg, "CURRMSECS: ")) {
+                        nodenameidx += 10;
+                        strncpy(nodename, nodenameidx, 1024);
+                        if (nodenameidx = strstr(nodename, "\r")) {
+                            nodenameidx[0] = 0;
+                        }
+                        setusec = strtol(nodename, NULL, 10) * 1000;
+                    }
+                    memset(msg, 0, 1024);
+                    inet_ntop(dst4.sin_family, &dst4.sin_addr, msg, dst4size);
+                    /* Take measure from printing out the same ip twice in a row */
+                    if (strncmp(lastmsg, msg, 1024) != 0) {
+                        sendto(n4, "PING", 4, 0, (const struct sockaddr *)&dst4, dst4size);
+                        printf("MANAGER: %s\n", msg);
+                        strncpy(lastmsg, msg, 1024);
                    }
-                    setusec = strtol(nodename, NULL, 10) * 1000;
-                }
-                memset(msg, 0, 1024);
-                inet_ntop(dst4.sin_family, &dst4.sin_addr, msg, dst4size);
-                /* Take measure from printing out the same ip twice in a row */
-                if (strncmp(lastmsg, msg, 1024) != 0) {
-                    sendto(n4, "PING", 4, 0, (const struct sockaddr *)&dst4, dst4size);
-                    printf("MANAGER: %s\n", msg);
-                    strncpy(lastmsg, msg, 1024);
                }
            }
            if (FD_ISSET(ns, &rfds)) {
                memset(msg, 0, 1024);
                /* Deny packet access to the last 24 bytes to assure null */
                recvfrom(ns, msg, 1000, 0, (struct sockaddr *)&dst, &dstsize);
-                if  (nodenameidx = strstr(msg, "NODENAME: ")) {
+                if (strstr(msg, "HTTP/1.1 200 OK")) {
+                    if  (nodenameidx = strstr(msg, "NODENAME: ")) {
+                            nodenameidx += 10;
+                            strncpy(nodename, nodenameidx, 1024);
+                            nodenameidx = strstr(nodename, "\r");
+                            if (nodenameidx) { nodenameidx[0] = 0; }
+                            if (strncmp(lastnodename, nodename, 1024) != 0) {
+                                printf("NODENAME: %s\n", nodename);
+                                strncpy(lastnodename, nodename, 1024);
+                            }
+                    }
+                    if (nodenameidx = strstr(msg, "CURRTIME: ")) {
                        nodenameidx += 10;
                        strncpy(nodename, nodenameidx, 1024);
-                        nodenameidx = strstr(nodename, "\r");
-                        if (nodenameidx) { nodenameidx[0] = 0; }
-                        if (strncmp(lastnodename, nodename, 1024) != 0) {
-                            printf("NODENAME: %s\n", nodename);
-                            strncpy(lastnodename, nodename, 1024);
+                        if (nodenameidx = strstr(nodename, "\r")) {
+                            nodenameidx[0] = 0;
                        }
-                }
-                if (nodenameidx = strstr(msg, "CURRTIME: ")) {
-                    nodenameidx += 10;
-                    strncpy(nodename, nodenameidx, 1024);
-                    if (nodenameidx = strstr(nodename, "\r")) {
-                        nodenameidx[0] = 0;
+                        settime = strtol(nodename, NULL, 10);
                    }
-                    settime = strtol(nodename, NULL, 10);
-                }
-                if (nodenameidx = strstr(msg, "DEFAULTNET: 1")) {
-                    isdefault = 1;
-                }
-                if (nodenameidx = strstr(msg, "MGTIFACE: ")) {
-                    nodenameidx += 10;
-                    strncpy(mgtifname, nodenameidx, 1024);
-                    if (nodenameidx = strstr(mgtifname, "\r")) {
-                        nodenameidx[0] = 0;
+                    if (nodenameidx = strstr(msg, "DEFAULTNET: 1")) {
+                        isdefault = 1;
                    }
-                }
-                if (nodenameidx = strstr(msg, "CURRMSECS: ")) {
-                    nodenameidx += 10;
-                    strncpy(nodename, nodenameidx, 1024);
-                    if (nodenameidx = strstr(nodename, "\r")) {
-                        nodenameidx[0] = 0;
+                    if (nodenameidx = strstr(msg, "MGTIFACE: ")) {
+                        nodenameidx += 10;
+                        strncpy(mgtifname, nodenameidx, 1024);
+                        if (nodenameidx = strstr(mgtifname, "\r")) {
+                            nodenameidx[0] = 0;
+                        }
                    }
-                    setusec = strtol(nodename, NULL, 10) * 1000;
-                }
-                memset(msg, 0, 1024);
-                inet_ntop(dst.sin6_family, &dst.sin6_addr, msg, dstsize);
-                if (strncmp(last6msg, msg, 1024) != 0 || lastidx != dst.sin6_scope_id) {
-		    lastidx = dst.sin6_scope_id;
-                    sendto(ns, "PING", 4, 0, (const struct sockaddr *)&dst, dstsize);
-                    printf("MANAGER: %s", msg);
-                    if (strncmp(msg, "fe80::", 6) == 0) {
-                        printf("%%%u", dst.sin6_scope_id);
+                    if (nodenameidx = strstr(msg, "CURRMSECS: ")) {
+                        nodenameidx += 10;
+                        strncpy(nodename, nodenameidx, 1024);
+                        if (nodenameidx = strstr(nodename, "\r")) {
+                            nodenameidx[0] = 0;
+                        }
+                        setusec = strtol(nodename, NULL, 10) * 1000;
                    }
-                    printf("\n");
-                    printf("EXTMGRINFO: %s", msg);
-                    if (strncmp(msg, "fe80::", 6) == 0) {
-                        printf("%%%u", dst.sin6_scope_id);
+                    memset(msg, 0, 1024);
+                    inet_ntop(dst.sin6_family, &dst.sin6_addr, msg, dstsize);
+                    if (strncmp(last6msg, msg, 1024) != 0 || lastidx != dst.sin6_scope_id) {
+                lastidx = dst.sin6_scope_id;
+                        sendto(ns, "PING", 4, 0, (const struct sockaddr *)&dst, dstsize);
+                        printf("MANAGER: %s", msg);
+                        if (strncmp(msg, "fe80::", 6) == 0) {
+                            printf("%%%u", dst.sin6_scope_id);
+                        }
+                        printf("\n");
+                        printf("EXTMGRINFO: %s", msg);
+                        if (strncmp(msg, "fe80::", 6) == 0) {
+                            printf("%%%u", dst.sin6_scope_id);
+                        }
+                        printf("|%s|%d\n", mgtifname, isdefault);
+                        strncpy(last6msg, msg, 1024);
                    }
-                    printf("|%s|%d\n", mgtifname, isdefault);
-                    strncpy(last6msg, msg, 1024);
                }
            }
        }
@@ -132,8 +132,8 @@ static int http_read(const char *path, char *buf, size_t size, off_t offset,

    if (strcmp(path, filename) != 0) return -ENOENT;
    memset(headbuffer, 0, 512);
-    if (offset >= filesize) return 0;
-    if (offset + size - 1 >= filesize) size = filesize - offset - 1;
+    if (offset >= filesize || size == 0) return 0;
+    if (offset + size > filesize) size = filesize - offset;
    snprintf(headbuffer, 512, "%ld-%ld", offset, offset + size - 1);
    if (curl_easy_setopt(curl, CURLOPT_RANGE, headbuffer) != CURLE_OK) {
        fprintf(stderr, "Error setting range\n");
@@ -49,6 +49,7 @@ def main(args):
    wiz.add_argument('-p', help='Copy in TFTP contents required for PXE support', action='store_true')
    wiz.add_argument('-i', help='Interactively prompt for behaviors', action='store_true')
    wiz.add_argument('-l', help='Set up local management node to allow login from managed nodes', action='store_true')
+    wiz.add_argument('-r', help='Repack site contents if present', action='store_true')
    osip = sp.add_parser('importcheck', help='Check import of an OS image from an ISO image')
    osip.add_argument('imagefile', help='File to use for source of importing')
    osip = sp.add_parser('import', help='Import an OS image from an ISO image')
@@ -367,6 +368,8 @@ def initialize(cmdset):
        rc = initialize_genesis()
        if rc != 0:
            sys.exit(rc)
+    if cmdset.r:
+        didsomething = True
    if not didsomething and (cmdset.k or cmdset.l or cmdset.g or cmdset.p):
        if cmdset.g:
            updateboot('genesis-x86_64')
@@ -94,6 +94,7 @@ _allowedbyrole = {
            '/node*/power/state',
            '/node*/sensors/*',
            '/node*/attributes/current',
+            '/node*/attributes/all',
            '/node*/description',
            '/noderange/*/nodes/',
            '/nodes/',
@@ -379,7 +379,7 @@ node = {
                        'the managed node.  If not specified, then console '
                        'is disabled.  "ipmi" should be specified for most '
                        'systems if console is desired.'),
-        'validvalues': ('ssh', 'ipmi', 'openbmc', 'tsmsol', 'vcenter', 'proxmox'),
+        'validvalues': ('ssh', 'ipmi', 'openbmc', 'megasol', 'tsmsol', 'vcenter', 'proxmox'),
    },
 #    'virtualization.host': {
 #        'description': ('Hypervisor where this node does/should reside'),
@@ -373,7 +373,7 @@ def _rpc_rename_nodes(tenant, renamemap):


 def _rpc_rename_nodegroups(tenant, renamemap):
-    ConfigManager(tenant)._true_rename_nodegroups(renamemap)
+    ConfigManager(tenant)._true_rename_groups(renamemap)


 def _rpc_set_group_attributes(tenant, attribmap, autocreate):
@@ -1382,7 +1382,7 @@ class ConfigManager(object):
            return _cfgstore['main']
        return _cfgstore['tenant'][self.tenant]

-    def __init__(self, tenant, decrypt=False, username=None):
+    def __init__(self, tenant, decrypt=False, username=None, create_tenant=False):
        self.clientfiles = {}
        global _cfgstore
        self.inrestore = False
@@ -1404,12 +1404,14 @@ class ConfigManager(object):
                    self._cfgstore['nodes'] = {}
                    self._bg_sync_to_file()
                return
-            elif 'tenant' not in _cfgstore:
+            elif 'tenant' not in _cfgstore and create_tenant:
                _cfgstore['tenant'] = {tenant: {}}
                self._bg_sync_to_file()
-            elif tenant not in _cfgstore['tenant']:
+            elif tenant not in _cfgstore['tenant'] and create_tenant:
                _cfgstore['tenant'][tenant] = {}
                self._bg_sync_to_file()
+            elif tenant and tenant not in _cfgstore.get('tenant', {}):
+                raise ValueError("Tenant {0} does not exist".format(tenant))
            self.tenant = tenant
            if 'nodegroups' not in self._cfgstore:
                self._cfgstore['nodegroups'] = {'everything': {}}
@@ -1420,6 +1422,9 @@ class ConfigManager(object):
            self.wait_for_sync()

    def add_client_file(self, clientfile):
+        filename = os.path.normpath(clientfile.filename)
+        if filename.startswith('../') or filename.startswith('..\\'):
+            raise ValueError("Invalid filename: {0}".format(clientfile.filename))
        self.clientfiles[clientfile.filename] = clientfile.fileobject

    def close_client_files(self):
@@ -61,6 +61,8 @@ def chunk_output(output, n):
        yield output[i:i + n]

 def get_buffer_output(nodename):
+    if _bufferdaemon is None:
+        eventlet.spawn(run_buffer_daemon)
    out = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    out.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
    out.connect("\x00confluent-vtbuffer")
@@ -85,6 +87,8 @@ def get_buffer_output(nodename):
 def send_output(nodename, output):
    if not isinstance(nodename, bytes):
        nodename = nodename.encode('utf8')
+    if _bufferdaemon is None:
+        eventlet.spawn(run_buffer_daemon)
    out = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    out.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
    out.connect("\x00confluent-vtbuffer")
@@ -597,14 +601,25 @@ def _start_tenant_sessions(cfm):
                          event=log.Events.stacktrace)
    cfm.watch_nodecollection(_nodechange)

+running = True
+
+def run_buffer_daemon():
+    global _bufferdaemon
+    while running:
+        minrestartdeadline = time.time() + 30  # Do not restart more than once every 30 seconds
+        _bufferdaemon = subprocess.Popen(
+            ['/opt/confluent/bin/vtbufferd', 'confluent-vtbuffer'], bufsize=0, stdin=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL)
+        _bufferdaemon.wait()
+        # Ensure we do not restart more than once every 30 seconds
+        sleep_time = minrestartdeadline - time.time()
+        if sleep_time > 0:
+            eventlet.sleep(sleep_time)

 def initialize():
    global _tracelog
    global _bufferdaemon
    _tracelog = log.Logger('trace')
-    _bufferdaemon = subprocess.Popen(
-        ['/opt/confluent/bin/vtbufferd', 'confluent-vtbuffer'], bufsize=0, stdin=subprocess.DEVNULL,
-        stdout=subprocess.DEVNULL)

 def start_console_sessions():
    configmodule.hook_new_configmanagers(_start_tenant_sessions)
@@ -1390,17 +1390,16 @@ def handle_staging(pathcomponents, operation, configmanager, inputdata):
    '''
    if operation == 'create':
        if len(pathcomponents) == 1:
-                stage = Staging(inputdata['user'],str(uuid.uuid1()))
-                if stage.create_directory():
-                    if 'filename' in inputdata:
-                        data_file = stage.storage_folder + '/filename.txt'
-                        with open(data_file, 'w') as f:
-                            f.write(inputdata['filename'])     
-                    else:
-                        raise Exception('Error: Missing filename arg')
-                    push_url = stage.get_push_url()
-                    yield msg.CreatedResource(push_url)
-
+            if 'filename' not in inputdata:
+                raise Exception('Error: Missing filename parameter')
+            inputdata['filename'] = os.path.normpath(inputdata['filename'])
+            if '/' in inputdata['filename'] or '\\' in inputdata['filename']:
+                raise Exception('Error: Invalid filename parameter, must not contain path separators')
+            stage = Staging(inputdata['user'],str(uuid.uuid4()))
+            if stage.create_directory():
+                data_file = stage.storage_folder + '/filename.txt'
+                push_url = stage.get_push_url()
+                yield msg.CreatedResource(push_url)
        elif len(pathcomponents) == 3:
            stage = Staging(pathcomponents[1], pathcomponents[2])
            file = stage.get_file_name()
@@ -174,6 +174,7 @@ pxearchs = {
    b'\x00\x09': 'uefi-x64',
    b'\x00\x0b': 'uefi-aarch64',
    b'\x00\x10': 'uefi-httpboot',
+    b'\x00\x13': 'uefi-httpboot',  # arm httpboot
 }


@@ -17,6 +17,7 @@
 # This SCGI server provides a http wrap to confluent api
 # It additionally manages httprequest console sessions
 import base64
+import shutil
 try:
    import Cookie
 except ModuleNotFoundError:
@@ -360,11 +361,12 @@ def _authorize_request(env, operation, reqbody):
            return {'code': 401}
        sessid = _establish_http_session(env, authdata, name, cookie)
    if authdata and element and element.startswith('/sessions/current/webauthn/validate/'):
-        if webauthn:
-            for rsp in webauthn.handle_api_request(element, env, None, authdata[2], authdata[1], None, reqbody, None):
-                if rsp['verified']:
-                    sessid = _establish_http_session(env, authdata, name, cookie)
-                    break
+        if not webauthn:
+            return {'code': 501}
+        for rsp in webauthn.handle_api_request(element, env, None, authdata[2], authdata[1], None, reqbody, None):
+            if rsp['verified']:
+                sessid = _establish_http_session(env, authdata, name, cookie)
+                break
    skiplog = _should_skip_authlog(env)
    if authdata:
        auditmsg = {
@@ -846,7 +848,7 @@ def resourcehandler_backend(env, start_response):
        yield 'Our princess is in another castle!'
        return
    elif (operation == 'create' and ('/console/session' in env['PATH_INFO'] or
-            '/shell/sessions/' in env['PATH_INFO'])):
+            '/shell/sessions/' in env['PATH_INFO']) and env['PATH_INFO'].startswith(('/nodes/', '/noderange/'))):
        #hard bake JSON into this path, do not support other incarnations
        if '/console/session' in env['PATH_INFO']:
            prefix, _, _ = env['PATH_INFO'].partition('/console/session')
@@ -984,9 +986,7 @@ def resourcehandler_backend(env, start_response):
            start_response('200 OK', headers)
            yield rsp
            return
-    
-
-    elif (operation == 'create' and ('/firmware/updates/active' in env['PATH_INFO'])):    
+    elif (operation == 'create' and ('/firmware/updates/active' in env['PATH_INFO']) and env['PATH_INFO'].startswith(('/nodes/', '/noderange/'))):    
        url = env['PATH_INFO']
        if 'application/json' in reqtype:
            if not isinstance(reqbody, str):
@@ -1007,8 +1007,7 @@ def resourcehandler_backend(env, start_response):
            yield json.dumps({'data': nodeurls})
            start_response('200 OK', headers)
            return
-    
-    elif (operation == 'create' and ('/staging' in env['PATH_INFO'])):
+    elif (operation == 'create' and (env['PATH_INFO'].startswith('/staging'))):
        url = env['PATH_INFO']
        args_dict = {}
        content_length = int(env.get('CONTENT_LENGTH', 0))
@@ -1217,7 +1216,7 @@ def _assemble_json(responses, resource=None, url=None, extension=None):
        rspdata, sort_keys=True, indent=4, ensure_ascii=False).encode('utf-8'))


-def serve(bind_host, bind_port):
+def serve(bind_host, bind_port, bind_group=None, bind_perms=None):
    # TODO(jbjohnso): move to unix socket and explore
    # either making apache deal with it
    # or just supporting nginx or lighthttpd
@@ -1232,13 +1231,17 @@ def serve(bind_host, bind_port):
    while not sock:
        try:
            if '/' in bind_host:
+                oldumask = os.umask(0o777 - bind_perms)
                try:
                    os.remove(bind_host)
                except Exception:
                    pass
                sock = eventlet.listen(
                    bind_host, family=socket.AF_UNIX)
-                os.chmod(bind_host, 0o666)
+                os.umask(oldumask)
+                os.chmod(bind_host, bind_perms)
+                if bind_group:
+                    shutil.chown(bind_host, group=bind_group)
            else:
                addrinfo = socket.getaddrinfo(bind_host, bind_port)[0]
                sock = eventlet.listen(
@@ -1264,17 +1267,22 @@ def serve(bind_host, bind_port):


 class HttpApi(object):
-    def __init__(self, bind_host=None, bind_port=None):
+    def __init__(self, bind_host=None, bind_port=None, bind_group=None, bind_perms=None):
        self.server = None
        self.bind_host = bind_host or '127.0.0.1'
        self.bind_port = bind_port or 4005
+        # Ultimately, a unix socket is being used in lieu of a TCP socket,
+        # so open permissions make sense as the security is not based solely on socket access
+        # however, steering it to webserver group can be done for extra confidence
+        self.bind_group = bind_group
+        self.bind_perms = bind_perms or 0o666

    def start(self):
        global auditlog
        global tracelog
        tracelog = log.Logger('trace')
        auditlog = log.Logger('audit')
-        self.server = eventlet.spawn(serve, self.bind_host, self.bind_port)
+        self.server = eventlet.spawn(serve, self.bind_host, self.bind_port, self.bind_group, self.bind_perms)


 _cleaner = eventlet.spawn(_sessioncleaner)
@@ -314,14 +314,14 @@ def run(args):
    auth.check_for_yaml()
    collective.startup()
    consoleserver.initialize()
-    http_bind_host, http_bind_port = _get_connector_config('http')
-    sock_bind_host, sock_bind_port = _get_connector_config('socket')
+    http_bind_host, http_bind_port, http_bind_group, http_bind_perms = _get_connector_config('http')
+    sock_bind_host, sock_bind_port, sock_bind_group, sock_bind_perms = _get_connector_config('socket')
    try:
-        sockservice = sockapi.SockApi(sock_bind_host, sock_bind_port)
+        sockservice = sockapi.SockApi(sock_bind_host, sock_bind_port, sock_bind_group, sock_bind_perms)
        sockservice.start()
    except NameError:
        pass
-    webservice = httpapi.HttpApi(http_bind_host, http_bind_port)
+    webservice = httpapi.HttpApi(http_bind_host, http_bind_port, http_bind_group, http_bind_perms)
    webservice.start()
    while len(list(configmanager.list_collective())) >= 2:
        # If in a collective, stall automatic startup activity
@@ -340,7 +340,24 @@ def run(args):
 def _get_connector_config(session):
    host = conf.get_option(session, 'bindhost')
    port = conf.get_int_option(session, 'bindport')
-    return (host, port)
+    group = conf.get_option(session, 'bindgroup')
+    perms = conf.get_option(session, 'bindperms')
+    if perms:
+        if perms.startswith('0'):
+            perms = int(perms, 8)
+        else:
+            # Parse rw-rw-rw- format (user, group, other)
+            perms_value = 0
+            perm_map = {'r': 4, 'w': 2, 'x': 1}
+            for i, section in enumerate([perms[0:3], perms[3:6], perms[6:9]]):
+                for char in section:
+                    if char in perm_map:
+                        perms_value += perm_map[char] * (8 ** (2 - i))
+            perms = perms_value
+    else:
+        perms = None
+
+    return (host, port, group, perms)

 def _get_logdirectory():
    return conf.get_option('globals', 'logdirectory')
@@ -19,6 +19,7 @@
 # Things are defined here to 'encourage' developers to coordinate information
 # format.  This is also how different data formats are supported
 import base64
+import os
 import confluent.exceptions as exc
 import confluent.config.configmanager as cfm
 import confluent.config.conf as cfgfile
@@ -27,6 +28,7 @@ from datetime import datetime
 import confluent.util as util
 import msgpack
 import json
+import pwd

 try:
    unicode
@@ -569,6 +571,8 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
        return InputPowerMessage(path, nodes, inputdata)
    elif '/'.join(path).startswith('media/detach'):
        return DetachMedia(path, nodes, inputdata)
+    elif '/'.join(path).startswith('media/attach') and inputdata:
+        return InputMediaUrl(path, nodes, inputdata, configmanager)
    elif '/'.join(path).startswith('media/') and inputdata:
        return InputMedia(path, nodes, inputdata, configmanager)
    elif '/'.join(path).startswith('support/servicedata') and inputdata:
@@ -590,19 +594,75 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
        raise exc.InvalidArgumentException(
            'No known input handler for request')

-class InputFirmwareUpdate(ConfluentMessage):

+def checkaccess(user, filename, pwent):
+    """Check if a user has read access to a file.
+
+    This function checks if the specified user has read access to the given
+    filename. It returns True if the user has read access, and False otherwise.
+    """
+    child = os.fork()
+    if child == 0:
+        os.setgroups(os.getgrouplist(user, pwent.pw_gid))
+        os.setgid(pwent.pw_gid)
+        os.setuid(pwent.pw_uid)
+        if os.access(filename, os.R_OK):
+            os._exit(0)
+        os._exit(1)
+    else:
+        pid, status = os.waitpid(child, 0)
+        if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0:
+            return True
+        return False
+
+def isurl(value):
+    prefix, value = value.split('://', 1) if '://' in value else ('', value)
+    if '/' in prefix:
+        return False
+    return True if prefix else False
+
+class InputFirmwareUpdate(ConfluentMessage):
+    urlsupported = False
    def __init__(self, path, nodes, inputdata, configmanager):
        self._filename = inputdata.get('filename', inputdata.get('url', inputdata.get('dirname', None)))
        self.bank = inputdata.get('bank', None)
+        self.parameterdata = inputdata.get('parameterdata', None)
        self.nodes = nodes
        self.filebynode = {}
        self._complexname = False
+        curruser = configmanager.current_user if configmanager else None
+        # for configmanager filehandles, those are already opened by client, so 
+        # no need to check server side access
+        checkedfiles = set(list(configmanager.clientfiles))
        for expanded in configmanager.expand_attrib_expression(
                nodes, self._filename):
            node, value = expanded
            if value != self._filename:
                self._complexname = True
+            if self.urlsupported and isurl(value):
+                self.filebynode[node] = value
+                continue
+            value = os.path.normpath(value)
+            if value not in checkedfiles:
+                if value.startswith('../'):
+                    raise Exception('File transfer with ../ is not supported')
+                if value.startswith('/etc/confluent'):
+                    raise Exception(
+                        'File transfer with /etc/confluent is not supported')
+                if value.startswith('/var/log/confluent'):
+                    raise Exception(
+                        'File transfer with /var/log/confluent is not supported')
+                if curruser and not value.startswith('/var/lib/confluent/client_assets/'):
+                    try:
+                        pwent = pwd.getpwnam(curruser)
+                        if not checkaccess(curruser, value, pwent):
+                            errstr = '{0} is not readable by {1}, check the file and parent directory ownership and permissions'.format(
+                                value, curruser)
+                            raise Exception(errstr)
+                    except KeyError:
+                        pass # We can't check ownership for confluent users without system users, as is the case in a prominent container usage,
+                            # We must rely upon the banned paths to mitigate risk instead
+                checkedfiles.add(value)
            self.filebynode[node] = value

    @property
@@ -632,6 +692,11 @@ class InputMedia(InputFirmwareUpdate):
    # Use InputFirmwareUpdate
    pass

+class InputMediaUrl(InputFirmwareUpdate):
+    # Use InputFirmwareUpdate for URL-based media attachment
+    urlsupported = True
+    pass
+
 class InputLicense(InputFirmwareUpdate):
    pass

@@ -1282,6 +1347,7 @@ class BootDevice(ConfluentChoiceMessage):
        'cd',
        'floppy',
        'usb',
+        'http',
    ])

    valid_bootmodes = set([
@@ -1724,13 +1790,18 @@ class Disk(ConfluentMessage):
        'rebuilding',
        'online',
        'offline',
+        'failed',
+        'foreign',
    ])
    state_aliases = {
        'unconfigured bad': 'fault',
        'unconfigured good': 'unconfigured',
+        '(foreign) unconfigured good': 'foreign',
        'unconfiguredgood': 'unconfigured',
        'global hot spare': 'hotspare',
+        'globalhotspare': 'hotspare',
        'dedicated hot spare': 'hotspare',
+        'dedicatedhotspare': 'hotspare',
    }

    def _normalize_state(self, instate):
@@ -0,0 +1,160 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015-2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This plugin provides an ssh implementation comforming to the 'console'
+# specification.  consoleserver or shellserver would be equally likely
+# to use this.
+
+import confluent.exceptions as cexc
+import confluent.interface.console as conapi
+import confluent.log as log
+import confluent.util as util
+import pyghmi.exceptions as pygexc
+import pyghmi.redfish.command as rcmd
+import eventlet
+import eventlet.green.ssl as ssl
+try:
+    websocket = eventlet.import_patched('websocket')
+    wso = websocket.WebSocket
+except Exception:
+    wso = object
+
+
+def get_conn_params(node, configdata):
+    if 'secret.hardwaremanagementuser' in configdata:
+        username = configdata['secret.hardwaremanagementuser']['value']
+    else:
+        username = 'USERID'
+    if 'secret.hardwaremanagementpassword' in configdata:
+        passphrase = configdata['secret.hardwaremanagementpassword']['value']
+    else:
+        passphrase = 'PASSW0RD'  # for lack of a better guess
+    if 'hardwaremanagement.manager' in configdata:
+        bmc = configdata['hardwaremanagement.manager']['value']
+    else:
+        bmc = node
+    bmc = bmc.split('/', 1)[0]
+    return {
+        'username': username,
+        'passphrase': passphrase,
+        'bmc': bmc,
+    }   
+_configattributes = ('secret.hardwaremanagementuser',
+                     'secret.hardwaremanagementpassword',
+                     'hardwaremanagement.manager')
+
+
+class WrappedWebSocket(wso):
+
+    def set_verify_callback(self, callback):
+        self._certverify = callback
+
+    def connect(self, url, **options):
+        add_tls = url.startswith('wss://')
+        if add_tls:
+            hostname, port, resource, _ = websocket._url.parse_url(url)
+            if hostname[0] != '[' and ':' in hostname:
+                hostname = '[{0}]'.format(hostname)
+            if resource[0] != '/':
+                resource = '/{0}'.format(resource)
+            url = 'ws://{0}:443{1}'.format(hostname,resource)
+        else:
+            return super(WrappedWebSocket, self).connect(url, **options)
+        self.sock_opt.timeout = options.get('timeout', self.sock_opt.timeout)
+        self.sock, addrs = websocket._http.connect(url, self.sock_opt, websocket._http.proxy_info(**options),
+                                           options.pop('socket', None))
+        self.sock = ssl.wrap_socket(self.sock, cert_reqs=ssl.CERT_NONE)
+        # The above is supersedeed by the _certverify, which provides
+        # known-hosts style cert validaiton
+        bincert = self.sock.getpeercert(binary_form=True)
+        if not self._certverify(bincert):
+            raise pygexc.UnrecognizedCertificate('Unknown certificate', bincert)
+        try:
+            self.handshake_response = websocket._handshake.handshake(self.sock, url, *addrs, **options)
+            if self.handshake_response.status in websocket._handshake.SUPPORTED_REDIRECT_STATUSES:
+                options['redirect_limit'] = options.pop('redirect_limit', 3) - 1
+                if options['redirect_limit'] < 0:
+                     raise Exception('Redirect limit hit')
+                url = self.handshake_response.headers['location']
+                self.sock.close()
+                return self.connect(url, **options)
+            self.connected = True
+        except:
+            if self.sock:
+                self.sock.close()
+                self.sock = None
+            raise
+
+
+class TsmConsole(conapi.Console):
+
+    def __init__(self, node, config):
+        self.node = node
+        self.ws = None
+        configdata = config.get_node_attributes([node], _configattributes, decrypt=True)
+        connparams = get_conn_params(node, configdata[node])
+        self.username = connparams['username']
+        self.password = connparams['passphrase']
+        self.bmc = connparams['bmc']
+        self.origbmc = connparams['bmc']
+        if ':' in self.bmc:
+            self.bmc = '[{0}]'.format(self.bmc)
+        self.datacallback = None
+        self.nodeconfig = config
+        self.connected = False
+
+
+    def recvdata(self):
+        while self.connected:
+            pendingdata = self.ws.recv()
+            if pendingdata == '':
+                self.datacallback(conapi.ConsoleEvent.Disconnect)
+                return
+            self.datacallback(pendingdata)
+
+    def connect(self, callback):
+        self.datacallback = callback
+        rc = rcmd.Command(self.origbmc, self.username,
+                          self.password,
+                          verifycallback=lambda x: True)
+        wc = rc.oem.wc
+        bmc = self.bmc
+        if '%' in self.bmc:
+            prefix = self.bmc.split('%')[0]
+            bmc = prefix + ']'
+        self.ws = WrappedWebSocket(host=bmc)
+        kv = util.TLSCertVerifier(
+            self.nodeconfig, self.node, 'pubkeys.tls_hardwaremanager').verify_cert
+        self.ws.set_verify_callback(kv)
+        self.ws.connect('wss://{0}/h5sol'.format(self.bmc), host=bmc, cookie='QSESSIONID={0}; __Host-garc={1}'.format(wc.cookies['QSESSIONID'], rc.oem.csrftok))
+        self.connected = True
+        eventlet.spawn_n(self.recvdata)
+        return
+
+    def write(self, data):
+        self.ws.send(data)
+
+    def close(self):
+        if self.ws:
+            self.ws.close()
+        self.connected = False
+        self.datacallback = None
+
+def create(nodes, element, configmanager, inputdata):
+    if len(nodes) == 1:
+        return TsmConsole(nodes[0], configmanager)
+
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+import json
+
 import confluent.vinzmanager as vinzmanager
 import confluent.exceptions as exc
 import confluent.firmwaremanager as firmwaremanager
@@ -486,8 +488,16 @@ class IpmiHandler(object):
        else:
            raise Exception('Not Implemented')

+    def update_firmware(self, filename, progress, data, bank):
+        params=()
+        if self.inputdata.parameterdata:
+            params = self.inputdata.parameterdata
+        if params and isinstance(params, str):
+            params = json.loads(params)
+        return self.ipmicmd.update_firmware(filename, progress=progress, data=data, bank=bank, otherfields=params)
+
    def handle_update(self):
-        u = firmwaremanager.Updater(self.node, self.ipmicmd.update_firmware,
+        u = firmwaremanager.Updater(self.node, self.update_firmware,
                                    self.inputdata.nodefile(self.node), self.tenant,
                                    bank=self.inputdata.bank,
                                    configmanager=self.cfm)
@@ -26,6 +26,7 @@ import ctypes.util
 import errno
 import os
 import pwd
+import shutil
 import stat
 import struct
 import sys
@@ -458,7 +459,7 @@ def removesocket():
    except OSError:
        pass

-def _unixdomainhandler():
+def _unixdomainhandler(bind_group=None, bind_perms=None):
    unixsocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    try:
        os.remove("/var/run/confluent/api.sock")
@@ -466,10 +467,12 @@ def _unixdomainhandler():
        pass
    if not os.path.isdir("/var/run/confluent"):
        os.makedirs('/var/run/confluent', 0o755)
+    oldumask = os.umask(0o777 - bind_perms)
    unixsocket.bind("/var/run/confluent/api.sock")
-    os.chmod("/var/run/confluent/api.sock",
-             stat.S_IWOTH | stat.S_IROTH | stat.S_IWGRP |
-             stat.S_IRGRP | stat.S_IWUSR | stat.S_IRUSR)
+    os.chmod("/var/run/confluent/api.sock", bind_perms)
+    if bind_group:
+        shutil.chown("/var/run/confluent/api.sock", group=bind_group)
+    os.umask(oldumask)
    atexit.register(removesocket)
    unixsocket.listen(5)
    while True:
@@ -498,11 +501,13 @@ def _unixdomainhandler():


 class SockApi(object):
-    def __init__(self, bindhost=None, bindport=None):
+    def __init__(self, bindhost=None, bindport=None, bind_group=None, bind_perms=None):
        self.tlsserver = None
        self.unixdomainserver = None
        self.bind_host = bindhost or '::'
        self.bind_port = bindport or 13001
+        self.bind_group = bind_group
+        self.bind_perms = bind_perms or 0o666

    def start(self):
        global auditlog
@@ -515,7 +520,7 @@ class SockApi(object):
        else:
            eventlet.spawn_n(self.watch_for_cert)
        eventlet.spawn_n(self.watch_resolv)
-        self.unixdomainserver = eventlet.spawn(_unixdomainhandler)
+        self.unixdomainserver = eventlet.spawn(_unixdomainhandler, self.bind_group, self.bind_perms)

    def watch_resolv(self):
        while True:
@@ -283,15 +283,13 @@ class TLSCertVerifier(object):
    def verify_cert(self, certificate):
        storedprint = self.cfm.get_node_attributes(self.node, (self.fieldname,)
                                                   )
-        
-        if (self.fieldname not in storedprint[self.node] or
-                storedprint[self.node][self.fieldname]['value'] == ''):
+        storedprint = storedprint.get(self.node, {}).get(self.fieldname, {}).get('value', '')
+        newpolicy = self.cfm.get_node_attributes(self.node,
+                                                ('pubkeys.addpolicy',))
+        newpolicy = newpolicy.get(self.node, {}).get('pubkeys.addpolicy', {}).get('value', '')
+        if not storedprint:
            # no stored value, check policy for next action
-            newpolicy = self.cfm.get_node_attributes(self.node,
-                                                     ('pubkeys.addpolicy',))
-            if ('pubkeys.addpolicy' in newpolicy[self.node] and
-                    'value' in newpolicy[self.node]['pubkeys.addpolicy'] and
-                    newpolicy[self.node]['pubkeys.addpolicy']['value'] == 'manual'):
+            if newpolicy == 'manual':
                # manual policy means always raise unless a match is set
                # manually
                fingerprint = get_fingerprint(certificate, 'sha256')
@@ -300,18 +298,18 @@ class TLSCertVerifier(object):
                                         self.fieldname, 'newkey')
            # since the policy is not manual, go ahead and add new key
            # after logging to audit log
-            fingerprint = get_fingerprint(certificate, 'sha256')
-            auditlog = log.Logger('audit')
-            auditlog.log({'node': self.node, 'event': 'certautoadd',
-                          'fingerprint': fingerprint})
-            self.cfm.set_node_attributes(
-                {self.node: {self.fieldname: fingerprint}})
-            return True
-        elif cert_matches(storedprint[self.node][self.fieldname]['value'],
-                          certificate):
+            if newpolicy in ('tofu', ''):
+                fingerprint = get_fingerprint(certificate, 'sha256')
+                auditlog = log.Logger('audit')
+                auditlog.log({'node': self.node, 'event': 'certautoadd',
+                            'fingerprint': fingerprint})
+                self.cfm.set_node_attributes(
+                    {self.node: {self.fieldname: fingerprint}})
+                return True
+        elif cert_matches(storedprint, certificate) and newpolicy != 'ca-only':
            return True
        fingerprint = get_fingerprint(certificate, 'sha256')
-        # Mismatches, but try more traditional validation using the site CAs
+        # No pinned certificate match, try to validate by CA if possible
        if self.subject:
            try:
                if verification and self.verify_by_ca(certificate):
@@ -322,7 +320,8 @@ class TLSCertVerifier(object):
                        {self.node: {self.fieldname: fingerprint}})
                    return True
            except Exception:
-                pass
+                if newpolicy == 'ca-only':
+                    raise
        raise cexc.PubkeyInvalid(
            'Mismatched certificate detected', certificate, fingerprint,
            self.fieldname, 'mismatch')
@@ -21,6 +21,7 @@ fi
 dracut_install efibootmgr
 dracut_install du df ssh-keygen scp clear dhclient lldpd lldpcli tee
 dracut_install /$IMPLIBDIR/libnss_dns.so.2 /$IMPLIBDIR/libnss_dns.so.2 /$IMPLIBDIR/libnss_myhostname.so.2
+dracut_install /usr/$IMPLIBDIR/libdl.so.2 /usr/$IMPLIBDIR/libpthread.so.0
 dracut_install ldd uptime /usr/$IMPLIBDIR/libnl-3.so.200
 dracut_install poweroff date /etc/nsswitch.conf /etc/services /etc/protocols
 dracut_install  /usr/share/terminfo/x/xterm /usr/share/terminfo/l/linux /usr/share/terminfo/v/vt100 /usr/share/terminfo/x/xterm-color /usr/share/terminfo/s/screen /usr/share/terminfo/x/xterm-256color /usr/share/terminfo/p/putty-256color /usr/share/terminfo/p/putty /usr/share/terminfo/d/dumb
@@ -45,7 +46,7 @@ dracut_install /usr/lib/udev/rules.d/*-dm.rules /usr/sbin/dmsetup /usr/lib/udev/
 #dracut_install opainfo
 #dracut_install /usr/lib/opa-fm/bin/opafmd
 #dracut_install /usr/sbin/opensm /usr/libexec/opensm-launch
-dracut_install /usr/$IMPLIBDIR/libibverbs/libhfi1verbs-rdmav34.so /etc/libibverbs.d/hfi1verbs.driver /etc/libibverbs.d/mlx4.driver /etc/libibverbs.d/mlx5.driver /usr/$IMPLIBDIR/libibverbs/libmlx4-rdmav34.so /usr/$IMPLIBDIR/libibverbs/libmlx5-rdmav34.so
+dracut_install /usr/$IMPLIBDIR/libibverbs/libhfi1verbs-rdmav59.so /etc/libibverbs.d/hfi1verbs.driver /etc/libibverbs.d/mlx4.driver /etc/libibverbs.d/mlx5.driver /usr/$IMPLIBDIR/libibverbs/libmlx4-rdmav59.so /usr/$IMPLIBDIR/libibverbs/libmlx5-rdmav59.so
 if [ -x /usr/libexec/openssh/sshd-session ]; then
    dracut_install /usr/libexec/openssh/sshd-session
 fi
@@ -5,6 +5,7 @@ if grep Ubuntu /etc/os-release > /dev/null; then # must include specific drivers
    instmods nls_iso8859-1
 fi
 instmods virtio_net
+instmods xen-netfront xen-blkfront xen-pcifront
 instmods e1000 e1000e igb sfc mlx5_ib mlx5_core mlx4_en cxgb3 cxgb4 tg3 bnx2 bnx2x bna ixgb ixgbe qlge mptsas mpt2sas mpt3sas megaraid_sas ahci xhci-hcd sd_mod pmcraid be2net vfat ext3 ext4 usb_storage scsi_wait_scan ipmi_si ipmi_devintf qlcnic xfs
 instmods nvme
 instmods cdc_ether r8152
@@ -6,6 +6,7 @@ instmods cdc_ether r8152
 instmods r8169
 instmods vmxnet3 virtio_net
 instmods virtio_scsi vmw_pvscsi
+instmods xen-netfront xen-blkfront xen-pcifront
 instmods mptctl
 instmods mlx4_ib mlx5_ub ib_umad ib_ipoib
 instmods ice i40e hfi1 bnxt_en qed qede
@@ -3,6 +3,7 @@ instmods nfsv3 nfs_acl nfsv4 dns_resolver lockd fscache sunrpc
 instmods e1000 e1000e igb sfc mlx5_ib mlx5_core mlx4_en cxgb3 cxgb4 tg3 bnx2 bnx2x bna ixgb ixgbe qlge mptsas mpt2sas mpt3sas megaraid_sas ahci xhci-hcd sd_mod pmcraid be2net vfat ext3 ext4 usb_storage scsi_wait_scan ipmi_si ipmi_devintf qlcnic xfs
 instmods nvme
 instmods cdc_ether
+instmods xen-netfront xen-blkfront xen-pcifront
 instmods mptctl
 instmods mlx4_ib mlx5_ub ib_umad ib_ipoib
 instmods ice i40e hfi1 bnxt_en qed qede
@@ -6,6 +6,7 @@ instmods cdc_ether r8152
 instmods r8169
 instmods vmxnet3 virtio_net
 instmods virtio_scsi vmw_pvscsi
+instmods xen-netfront xen-blkfront xen-pcifront
 instmods mptctl
 instmods mlx4_ib mlx5_ub ib_umad ib_ipoib
 instmods ice i40e hfi1 bnxt_en qed qede
@@ -6,6 +6,7 @@ instmods cdc_ether r8152
 instmods r8169
 instmods vmxnet3 virtio_net
 instmods virtio_scsi vmw_pvscsi
+instmods xen-netfront xen-blkfront xen-pcifront
 instmods mptctl
 instmods mlx4_ib mlx5_ub ib_umad ib_ipoib
 instmods ice i40e hfi1 bnxt_en qed qede
@@ -214,7 +214,7 @@ def capture_remote(args):
    subprocess.check_call(['rsync', __file__, '{0}:/run/imgutil/capenv/'.format(targ)])
    finfo = subprocess.check_output(['ssh', targ, 'python3', '/run/imgutil/capenv/imgutil', 'getfingerprint']).decode('utf8')
    finfo = json.loads(finfo)
-    if finfo['oscategory'] not in ('el8', 'el9', 'ubuntu20.04', 'ubuntu22.04'):
+    if finfo['oscategory'] not in ('el8', 'el9', 'ubuntu20.04', 'ubuntu22.04', 'ubuntu24.04', 'ubuntu26.04'):
        sys.stderr.write('Not yet supported for capture: ' + repr(finfo) + '\n')
        sys.exit(1)
    unmet = finfo.get('unmetprereqs', [])
@@ -1481,15 +1481,19 @@ def pack_image(args):
        pass

 def gather_bootloader(outdir, rootpath='/'):
+    shimdestfilename = 'BOOTX64.EFI'
+    grubdestfilename = 'grubx64.efi'
    shimlocation = os.path.join(rootpath, 'boot/efi/EFI/BOOT/BOOTX64.EFI')
    if not os.path.exists(shimlocation):
        shimlocation = os.path.join(rootpath, 'boot/efi/EFI/BOOT/BOOTAA64.EFI')
+        shimdestfilename = os.path.basename(shimlocation)
    if not os.path.exists(shimlocation):
+        shimdestfilename = 'BOOTX64.EFI'
        shimlocation = os.path.join(rootpath, 'usr/lib64/efi/shim.efi')
    if not os.path.exists(shimlocation):
        shimlocation = os.path.join(rootpath, 'usr/lib/shim/shimx64.efi.signed')
    mkdirp(os.path.join(outdir, 'boot/efi/boot'))
-    shutil.copyfile(shimlocation, os.path.join(outdir, 'boot/efi/boot/BOOTX64.EFI'))
+    shutil.copyfile(shimlocation, os.path.join(outdir, 'boot/efi/boot/{0}'.format(shimdestfilename)))
    grubbin = None
    for candidate in glob.glob(os.path.join(rootpath, 'boot/efi/EFI/*')):
        if 'BOOT' not in candidate:
@@ -1498,6 +1502,7 @@ def gather_bootloader(outdir, rootpath='/'):
                break
            grubbin = os.path.join(candidate, 'grubaa64.efi')
            if os.path.exists(grubbin):
+                grubdestfilename = os.path.basename(grubbin)
                break
    if not grubbin:
        grubbin = os.path.join(rootpath, 'usr/lib64/efi/grub.efi')
@@ -1512,7 +1517,7 @@ def gather_bootloader(outdir, rootpath='/'):
        mkdirp(os.path.join(outdir, 'boot/EFI/ubuntu/'))
        with open(os.path.join(outdir, 'boot/EFI/ubuntu/grub.cfg'), 'w') as wo:
            wo.write('')
-    shutil.copyfile(grubbin, os.path.join(outdir, 'boot/efi/boot/grubx64.efi'))
+    shutil.copyfile(grubbin, os.path.join(outdir, 'boot/efi/boot/{0}'.format(grubdestfilename)))
    shutil.copyfile(grubbin, os.path.join(outdir, 'boot/efi/boot/grub.efi'))


@@ -3,6 +3,7 @@ instmods nfsv3 nfs_acl nfsv4 dns_resolver lockd fscache sunrpc
 instmods e1000 e1000e igb sfc mlx5_ib mlx5_core mlx4_en cxgb3 cxgb4 tg3 bnx2 bnx2x bna ixgb ixgbe qlge mptsas mpt2sas mpt3sas megaraid_sas ahci xhci-hcd sd_mod pmcraid be2net vfat ext3 ext4 usb_storage scsi_wait_scan ipmi_si ipmi_devintf qlcnic xfs
 instmods nvme
 instmods cdc_ether r8152
+instmods xen-netfront xen-blkfront xen-pcifront
 instmods mptctl
 instmods mlx4_ib mlx5_ub ib_umad ib_ipoib
 instmods ice i40e hfi1 bnxt_en qed qede
@@ -34,10 +34,12 @@ copy_exec /usr/bin/ssh-keygen
 copy_exec /usr/sbin/sshd
 copy_exec /usr/sbin/mkfs.xfs
 copy_exec /usr/lib/x86_64-linux-gnu/libfuse.so.2
+copy_exec /usr/lib/aarch64-linux-gnu/libfuse.so.2
 copy_exec /usr/bin/dirname
 [ -e $DESTDIR/usr/sbin/losetup ] && rm $DESTDIR/usr/sbin/losetup
 copy_exec /usr/sbin/losetup
 copy_exec /usr/lib/x86_64-linux-gnu/libtss2-tcti-device.so.0
+copy_exec /usr/lib/aarch64-linux-gnu/libtss2-tcti-device.so.0
 manual_add_modules e1000 e1000e igb sfc mlx5_ib mlx5_core mlx4_en cxgb3 cxgb4
 manual_add_modules tg3 bnx2 bnx2x bna ixgb ixgbe qlge mptsas mpt2sas mpt3sas 
 manual_add_modules megaraid_sas ahci xhci-hcd sd_mod pmcraid be2net vfat ext3
@@ -47,3 +49,4 @@ manual_add_modules ice i40e hfi1 bnxt_en qed qede dm-mod dm-log raid0 raid1
 manual_add_modules raid10 raid456 dm-raid dm-thin-pool dm-crypt dm-snapshot
 manual_add_modules linear dm-era fuse overlay squashfs loop zram
 manual_add_modules vmxnet3 r8169
+manual_add_modules xen-netfront xen-blkfront xen-pcifront
@@ -12,7 +12,7 @@ fi
 OLDINSECURE=$(nodeattrib $TARGNODE deployment.useinsecureprotocols -b 2> /dev/null |grep -v inherited|awk '{print $3}')
 nodedefine $TARGNODE deployment.profile=$TARGPROF deployment.useinsecureprotocols= deployment.pendingprofile=$TARGPROF
 confetty set /nodes/$TARGNODE/deployment/ident_image=create
-REMTMP=$(ssh $TARGNODE $(mktemp -d))
+REMTMP=$(ssh $TARGNODE mktemp -d)
 scp /var/lib/confluent/private/identity_files/$TARGNODE.json $TARGNODE:$REMTMP
 rm /var/lib/confluent/private/identity_files/$TARGNODE.*
 rm /var/lib/confluent/private/identity_images/$TARGNODE.*
@@ -19,7 +19,7 @@ if [ "$FWACTIVE" == 1 ]; then systemctl stop firewalld; fi
 opt/confluent/bin/copernicus  > /etc/confluent/confluent.info
 #opt/confluent/bin/clortho $TARGNODE $DEPLOYSRV > /etc/confluent/confluent.apikey
 . /etc/confluent/functions
-confluentpython opt/confluent/bin/apiclient -i $TAGRIDENT /confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
+confluentpython opt/confluent/bin/apiclient -i $TARGIDENT /confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
 if [ "$FWACTIVE" == 1 ]; then systemctl start firewalld; fi
 cp opt/confluent/bin/apiclient /opt/confluent/bin
 #curl -sg -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -H "CONFLUENT_NODENAME: $TARGNODE" https://$UDEPLOYSRV/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
Author	SHA1	Message	Date
Jarrod Johnson	a9ba385996	Fix type of maxnodes	2026-08-21 14:58:13 -04:00
Jarrod Johnson	29a9d417d6	Do not aggressively respawn buffer daemon. Start buffer daemon only when needed. Limit restarts to once every 30 seconds.	2026-08-20 07:23:40 -04:00
Jarrod Johnson	7347125b6f	Disable implicit tenant creation If we support more tenants, we will modify that branch.	2026-07-16 12:39:51 -04:00
Jarrod Johnson	66347a937e	Ensure prefix is a string	2026-07-14 11:01:32 -04:00
Jarrod Johnson	ee01dfd05e	Add UEFI HTTP boot for arm64 to recognized archs For now, we serve up the whole image, no need to distinguish arm from x86 here yet	2026-07-14 09:52:27 -04:00
Jarrod Johnson	611e1f5bc3	Fix nodemedia attach The hardening blocked all URL patters.	2026-07-14 09:48:09 -04:00
Jarrod Johnson	7622145fea	Fall back to x64 if the aarch64 location didn't pan out	2026-07-13 10:01:04 -04:00
Jarrod Johnson	678bd53857	Correct paths to aarch64 boot material in imgutil	2026-07-13 09:23:33 -04:00
Jarrod Johnson	3295778566	Improve webauthn error handling and tighten up routing If not webauthn, have authorize bail out on webauthn request instead of a sessionless authdata. For some "special" HTTP paths, tighten up routing rules.	2026-07-08 12:03:48 -04:00
Jarrod Johnson	b3acf011bd	Restart vtbufferd on exit Notably, if you strace it, it will trigger an exit(1). There was at least one documented segmentation fault as well. Buffer content is lost in such an event, but service remains running.	2026-07-02 12:20:01 -04:00
Jarrod Johnson	dc4cafc29f	Rework autoconsole logic Match autocons Skip unless EFI x86_64. If SPCR, trust it and use that unconditionally. Otherwise, if only one can respond to TIOCMGET, then use that one. If multiple can respond, but exactly one shows carrier, use that.	2026-06-25 16:41:32 -04:00
Jarrod Johnson	3340585fb4	Only count copernicus replies that have OK status	2026-06-25 15:24:31 -04:00
Jarrod Johnson	4456767122	When possible, check confluent user access to file If a confluent user is a system user, do not allow them to upload paths that their user would not have access to otherwise. For non-system users, continue with the path based banned behavior.	2026-06-25 12:15:13 -04:00
Jarrod Johnson	556bb1d0ff	Prevent staging of files from indicating path traversal	2026-06-25 10:09:59 -04:00
Jarrod Johnson	a201e9886e	Have messages force normalizing the incoming filenames This avoids downstream code that may expect specific locations from being confused.	2026-06-25 09:48:40 -04:00
Jarrod Johnson	f82993efe7	Fix debian deployment on slow network link up When network link was slow to establish, it would fall right through the network initilalization code. Now keep working it until a result is acheived.	2026-06-25 08:37:19 -04:00
Jarrod Johnson	f8366a50ef	Do not set 0.0.0.0 gateway	2026-06-24 15:59:10 -04:00
Jarrod Johnson	d369dcac55	Fix non-bonding configuration of ubuntu	2026-06-16 12:42:03 -04:00
Jarrod Johnson	3b2d92a219	Correct typo in pthread name	2026-06-08 11:00:40 -04:00
Jarrod Johnson	ffacb66c62	Update rdma vintage	2026-06-08 10:56:26 -04:00
Jarrod Johnson	2073f421da	Add libraries to genesis	2026-06-08 10:54:50 -04:00
Jarrod Johnson	ed2eed66dc	Allow nodedeploy to request http boot specifically	2026-06-03 09:28:44 -04:00
Jarrod Johnson	d8f9b6c8e6	Add support for http boot Some redfish require us to be very specific.	2026-06-03 09:12:46 -04:00
Jarrod Johnson	d9a18a7bf6	Actually use the interposer for firmware update	2026-06-01 19:52:43 -04:00
Markus Hilger	439a930188	confignet: Fix interface type detection for IB VFs IB VFs have the following "ip l" output: 4: ibp129s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 2044 qdisc mq state UP mode DEFAULT group default qlen 1000 link/infiniband 00:00:00:8d:fe:80:00:00:00:00:00:00:60:5e:65:03:00:2c:43:c8 brd 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:ff:ff:ff:ff vf 0 link/infiniband 00:00:00:8d:fe:80:00:00:00:00:00:00:60:5e:65:03:00:2c:43:c8 brd 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:ff:ff:ff:ff, spoof checking off, NODE_GUID 00:00:00:00:00:00:00:00, PORT_GUID 00:00:00:00:00:00:00:00, link-state enable, trust off, query_rss off 5: eno1: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN mode DEFAULT group default qlen 1000 link/ether 30:56:0f:17:c0:b4 brd ff:ff:ff:ff:ff:ff altname enp196s0 altname enx30560f17c0b4 This breaks the detection script because index 0 of the "vf 0 ..." line is not link/<type> anymore. This commit improves the detection logic to fix this.	2026-06-01 19:30:54 -04:00
Jarrod Johnson	00b2afd42b	Fixes for confignet for Ubuntu Try to find various layers of network config and normalize. Ultimately, after post subiquity will do some things and easiest to fix in firstboot instead.	2026-06-01 16:49:42 -04:00
Jarrod Johnson	90c2a4cf73	Fix iterating the netplan configuration	2026-06-01 12:55:11 -04:00
Jarrod Johnson	c691dc7159	Remove cloud-init netplan if redundant	2026-06-01 09:33:25 -04:00
Jarrod Johnson	a7c188b812	Add support for passing a parameterfile in updates	2026-06-01 07:51:22 -04:00
Jarrod Johnson	5ba43ecaa0	Add bonding to netplan management	2026-05-26 10:06:49 -04:00
Vinícius Ferrão	2f53d3bde6	Include xen-front drivers in confluent-curated initramfs	2026-05-23 14:21:13 -04:00
Jarrod Johnson	9fe9e8672a	Support more states	2026-05-21 10:03:08 -04:00
Jarrod Johnson	0fe60175f3	Add missing close	2026-05-21 08:36:16 -04:00
Jarrod Johnson	d411041243	Recognize more storage states	2026-05-20 16:14:24 -04:00
Jarrod Johnson	cc101d12bc	Port diskless enhancements from el9 to ubuntu	2026-05-20 15:30:23 -04:00
Jarrod Johnson	2f08ee81f2	Fix off by one in urlmount	2026-05-20 12:37:36 -04:00
Jarrod Johnson	1e9231eea6	Add megasol method	2026-05-19 09:17:50 -04:00
Jarrod Johnson	24cb05e535	Fix name of ssh in various ubuntu scripts	2026-05-13 13:52:05 -04:00
Jarrod Johnson	72b95abb4f	Add missing syncfiles examples to ubuntu profiles	2026-05-13 13:51:59 -04:00
Jarrod Johnson	57a170d0d8	Implement a headless mode For automation, this can make more sense.	2026-05-12 11:23:37 -04:00
Jarrod Johnson	daeabc6fe5	Add expression support to the nodeconsole automation	2026-05-11 16:51:01 -04:00
Jarrod Johnson	28a8f6f0d6	Provide automation facility for nodeconsole Allow nodeconsole to walk console according to a script	2026-05-11 13:55:54 -04:00
Jarrod Johnson	7542897b43	Normalize perms to int or None	2026-05-07 09:44:00 -04:00
Jarrod Johnson	c69952265f	Permit override of unix ownership/permissions on sockets If an environment knows more specifically what should have access in terms of group, allow service.cfg to indicate.	2026-05-07 08:44:16 -04:00
Jarrod Johnson	01cc86fa55	Add a '-r' argument to refresh site contents If an environment manually manages all materials, provide -r to let them request packing of those materials without trying to generate any of the content.	2026-05-06 08:46:31 -04:00
Jarrod Johnson	d6e3c7e837	Backport cert fix	2026-05-05 16:26:49 -04:00
Jarrod Johnson	dcb6aeca65	Add ca-only policy This policy forces CA validation every time. This also checks things like date validity.	2026-05-05 14:41:02 -04:00
Jarrod Johnson	7bc76b62e6	Backport CA policy changes	2026-05-05 11:31:26 -04:00
Jarrod Johnson	db313628c5	Include aarch64 names for key libraries in ubuntu diskless	2026-05-01 14:25:18 -04:00
Jarrod Johnson	f260323d2f	Fix missing ubuntu diskless content	2026-05-01 12:14:13 -04:00
Jarrod Johnson	d60bc7f524	Bring chrony fixes to other scripts	2026-04-30 11:24:20 -04:00
Jarrod Johnson	ff0d4cdadf	Fix diskless profiles for chrony.conf modification	2026-04-30 11:24:15 -04:00
Timothy Middelkoop	db6475c4da	Fix el8/el9 hook paths corrupted by symlinked el10 in aarch64 spec In confluent_osdeploy-aarch64.spec.tmpl, el10 was created as a symlink to el8, so the subsequent `mv el10/initramfs/usr el10/initramfs/var` inadvertently renamed el8's usr directory, leaving el8 and el9 (also symlinked to el8) with hooks at var/lib/dracut/hooks/ instead of usr/lib/dracut/hooks/. Rocky 9 dracut never found the hooks and dropped to the emergency shell on all aarch64 nodes. Use `cp -a el8 el10` as the x86_64 spec already does, so the rename only affects the el10 copy. Co-authored-by: Claude <noreply@anthropic.com> Signed-off-by: Timothy Middelkoop <tmiddelkoop@internet2.edu>	2026-04-30 08:09:06 -04:00
Jarrod Johnson	6d27e8a009	Allow monitor to read attributes by 'all' resource.	2026-04-29 07:51:08 -04:00
Jarrod Johnson	f363796439	Write to stdout as binary This allows better redirection. In python3, must write to sys.stdout.buffer. AttributeError for the unlikely event of a python2 based node being deployed.	2026-04-29 07:45:49 -04:00
Jarrod Johnson	dec118a985	Fix mistake in spec file	2026-04-24 09:29:31 -04:00
Jarrod Johnson	38eb0d7b10	Add Ubuntu 26.04	2026-04-24 08:35:44 -04:00
xu_ren_xian	ae338daa43	Handle confluent= boot arg and IPv4 NIC autodetect Add support for a confluent=<host> kernel argument in init-premount: configure networking, flush interfaces, autodetect the primary NIC (saved to /tmp/autodetectnic), verify TLS connectivity to the provided server, call the whoami endpoint over TLS to obtain the node name, and write results to /custom-installation/confluent/confluent.info (with fallback to copernicus on failure). Also update casper-bottom logic to handle IPv4 manager addresses: for IPv6 the manager is still bracketed and scoped interface resolved as before; for IPv4 the script now uses the previously detected NIC (/tmp/autodetectnic) or falls back to an `ip route get <mgr>` lookup to determine DEVICE. This ensures routed IPv4 deployments work correctly.	2026-04-23 17:50:41 -04:00
Jarrod Johnson	6ad3f0d70c	Fix mistakes in the node apoption samples	2026-04-20 09:46:45 -04:00
Jarrod Johnson	c0b9bb3ab1	Fix group rename in collective	2026-04-17 11:57:35 -04:00
Jarrod Johnson	b32755b0d3	Fix remote client operation with Python 3.12+	2026-04-17 09:01:12 -04:00