Merge branch 'master' into async

If deletion of a node brings quorum, notify followers
of the good news

build/arm/el10/Dockerfile

@@ -0,0 +1,8 @@
+FROM almalinux:10
+RUN ["yum", "-y","update"]
+RUN ["yum", "-y","install","gcc","make","rpm-build","python3-devel","python3-setuptools","createrepo","python3", "perl", "perl-DBI", "perl-JSON", "perl-XML-LibXML", "pinentry-tty", "rpm-sign", "git", "golang"]
+ADD rpmmacro /root/.rpmmacros
+ADD buildpackages.sh /bin/
+#VOLUME ["/rpms", "/srpms"]
+CMD ["/bin/bash","/bin/buildpackages.sh"]
+

build/arm/el10/buildpackages.sh

@@ -0,0 +1,6 @@
+for package in /srpms/*; do
+        rpmbuild --rebuild $package
+done
+find ~/rpmbuild/RPMS -type f -exec cp {} /rpms/ \;
+
+

build/arm/el10/rpmmacro

@@ -0,0 +1,3 @@
+%_gpg_digest_algo sha256
+%_gpg_name Lenovo Scalable Infrastructure
+

build/arm/el8/Dockerfile

@@ -0,0 +1,8 @@
+FROM almalinux:8
+RUN ["yum", "-y","update"]
+RUN ["yum", "-y","install","gcc","make","rpm-build","python3-devel","python3-setuptools","createrepo","python3", "perl", "perl-DBI", "perl-JSON", "perl-Net-DNS", "perl-DB_File", "perl-XML-LibXML", "rpm-sign", "git", "fuse-devel","libcurl-devel"]
+ADD rpmmacro /root/.rpmmacros
+ADD buildpackages.sh /bin/
+#VOLUME ["/rpms", "/srpms"]
+CMD ["/bin/bash","/bin/buildpackages.sh"]
+

build/arm/el8/buildpackages.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+for package in /srpms/*; do
+        rpmbuild --rebuild $package
+done
+find ~/rpmbuild/RPMS -type f -exec cp {} /rpms/ \;
+

build/arm/el8/rpmmacro

@@ -0,0 +1,2 @@
+%_gpg_digest_algo sha256
+%_gpg_name Lenovo Scalable Infrastructure

build/arm/el9/Dockerfile

@@ -0,0 +1,10 @@
+FROM almalinux:9
+RUN ["yum", "-y","update"]
+RUN ["yum", "-y","install","gcc","make","rpm-build","python3-devel","python3-setuptools","createrepo","python3", "perl", "perl-DBI", "perl-JSON", "perl-Net-DNS", "perl-DB_File", "perl-XML-LibXML", "pinentry-tty", "rpm-sign", "epel-release", "git"]
+RUN ["crb", "enable"]
+RUN ["yum", "-y","install","fuse-devel","libcurl-devel"]
+ADD rpmmacro /root/.rpmmacros
+ADD buildpackages.sh /bin/
+#VOLUME ["/rpms", "/srpms"]
+CMD ["/bin/bash","/bin/buildpackages.sh"]
+

build/arm/el9/buildpackages.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+for package in /srpms/*; do
+        rpmbuild --rebuild $package
+done
+find ~/rpmbuild/RPMS -type f -exec cp {} /rpms/ \;
+

build/arm/el9/rpmmacro

@@ -0,0 +1,2 @@
+%_gpg_digest_algo sha256
+%_gpg_name Lenovo Scalable Infrastructure

build/arm/noble/Dockerfile

@@ -0,0 +1,12 @@
+FROM ubuntu:noble
+ADD stdeb.patch /tmp/
+ADD buildapt.sh /bin/
+ADD distributions.tmpl /bin/
+RUN ["apt-get", "update"]
+RUN ["apt-get", "install", "-y", "reprepro", "python3-stdeb", "gnupg-agent", "devscripts", "debhelper", "libsoap-lite-perl", "libdbi-perl", "quilt", "git", "python3-pyparsing", "python3-dnspython", "python3-eventlet", "python3-netifaces", "python3-paramiko", "dh-python", "libjson-perl", "ronn", "alien", "gcc", "make"]
+RUN ["mkdir", "-p", "/sources/git/"]
+RUN ["mkdir", "-p", "/debs/"]
+RUN ["mkdir", "-p", "/apt/"]
+RUN ["bash", "-c", "patch -p1 < /tmp/stdeb.patch"]
+CMD ["/bin/bash", "/bin/buildapt.sh"]
+

build/arm/noble/buildapt.sh

@@ -0,0 +1,21 @@
+#cp -a /sources/git /tmp
+for builder in $(find /sources/git -name builddeb); do
+    cd $(dirname $builder)
+    ./builddeb /debs/
+done
+cp /prebuilt/* /debs/
+cp /osd/*.deb /debs/
+mkdir -p /apt/conf/
+CODENAME=$(grep VERSION_CODENAME= /etc/os-release | sed -e 's/.*=//')
+if [ -z "$CODENAME" ]; then
+    CODENAME=$(grep VERSION= /etc/os-release | sed -e 's/.*(//' -e 's/).*//')
+fi
+if ! grep $CODENAME /apt/conf/distributions; then
+    sed -e s/#CODENAME#/$CODENAME/ /bin/distributions.tmpl >> /apt/conf/distributions
+fi
+cd /apt/
+reprepro includedeb $CODENAME /debs/*.deb
+for dsc in /debs/*.dsc; do
+    reprepro includedsc $CODENAME $dsc
+done
+

build/arm/noble/distributions.tmpl

@@ -0,0 +1,7 @@
+Origin: Lenovo HPC Packages
+Label: Lenovo HPC Packages
+Codename: #CODENAME#
+Architectures: amd64 source
+Components: main
+Description: Lenovo HPC Packages
+

build/arm/noble/stdeb.patch

@@ -0,0 +1,34 @@
+diff -urN t/usr/lib/python3/dist-packages/stdeb/cli_runner.py t.patch/usr/lib/python3/dist-packages/stdeb/cli_runner.py
+--- t/usr/lib/python3/dist-packages/stdeb/cli_runner.py 2024-06-11 18:30:13.930328999 +0000
++++ t.patch/usr/lib/python3/dist-packages/stdeb/cli_runner.py   2024-06-11 18:32:05.392731405 +0000
+@@ -8,7 +8,7 @@
+     from ConfigParser import SafeConfigParser  # noqa: F401
+ except ImportError:
+     # python 3.x
+-    from configparser import SafeConfigParser  # noqa: F401
++    from configparser import ConfigParser  # noqa: F401
+ from distutils.util import strtobool
+ from distutils.fancy_getopt import FancyGetopt, translate_longopt
+ from stdeb.util import stdeb_cmdline_opts, stdeb_cmd_bool_opts
+diff -urN t/usr/lib/python3/dist-packages/stdeb/util.py t.patch/usr/lib/python3/dist-packages/stdeb/util.py
+--- t/usr/lib/python3/dist-packages/stdeb/util.py       2024-06-11 18:32:53.864776149 +0000
++++ t.patch/usr/lib/python3/dist-packages/stdeb/util.py 2024-06-11 18:33:02.063952870 +0000
+@@ -730,7 +730,7 @@
+     example.
+     """
+ 
+-    cfg = ConfigParser.SafeConfigParser()
++    cfg = ConfigParser.ConfigParser()
+     cfg.read(cfg_files)
+     if cfg.has_section(module_name):
+         section_items = cfg.items(module_name)
+@@ -801,7 +801,7 @@
+         if len(cfg_files):
+             check_cfg_files(cfg_files, module_name)
+ 
+-        cfg = ConfigParser.SafeConfigParser(cfg_defaults)
++        cfg = ConfigParser.ConfigParser(cfg_defaults)
+         for cfg_file in cfg_files:
+             with codecs.open(cfg_file, mode='r', encoding='utf-8') as fd:
+                 cfg.readfp(fd)
+

build/buildarm

@@ -0,0 +1,9 @@
+cd ~/confluent
+git pull
+rm ~/rpmbuild/RPMS/noarch/*osdeploy*
+rm ~/rpmbuild/SRPMS/*osdeploy*
+sh confluent_osdeploy/buildrpm-aarch64
+mkdir -p $HOME/el9/
+mkdir -p $HOME/el10/
+podman run --rm -it -v $HOME:/build el9build bash /build/confluent/confluent_vtbufferd/buildrpm /build/el9/
+

confluent_client/bin/confetty

@@ -45,6 +45,7 @@ import math
 import getpass
 import optparse
 import os
+import re
 import select
 import shlex
 import signal
@@ -130,6 +131,15 @@ def print_help():
     #common with the api document
 
 
+def writeout(data):
+    while True:
+        try:
+            select.select((), (sys.stdout,), ())
+            sys.stdout.write(data)
+            break
+        except BlockingIOError:
+            continue
+
 def updatestatus(stateinfo={}):
     global powerstate, powertime, clearpowermessage
     status = consolename
@@ -149,10 +159,10 @@ def updatestatus(stateinfo={}):
     if 'state' in stateinfo:  # currently only read power means anything
         newpowerstate = stateinfo['state']['value']
         if newpowerstate != powerstate and newpowerstate == 'off':
-            sys.stdout.write("\x1b[2J\x1b[;H[powered off]\r\n")
+            writeout("\x1b[2J\x1b[;H[powered off]\r\n")
             clearpowermessage = True
         if newpowerstate == 'on' and clearpowermessage:
-            sys.stdout.write("\x1b[2J\x1b[;H")
+            writeout("\x1b[2J\x1b[;H")
             clearpowermessage = False
         powerstate = newpowerstate
     if 'clientcount' in laststate and laststate['clientcount'] != 1:
@@ -170,7 +180,7 @@ def updatestatus(stateinfo={}):
     if info:
         status += ' [' + ','.join(info) + ']'
     if os.environ.get('TERM', '') not in ('linux'):
-        sys.stdout.write('\x1b]0;console: %s\x07' % status)
+        writeout('\x1b]0;console: %s\x07' % status)
     sys.stdout.flush()
 
 
@@ -450,7 +460,7 @@ def do_command(command, server):
                 print_result(res)
     elif argv[0] == 'start':
         targpath = fullpath_target(argv[1])
-        nodename = targpath.split('/')[-3]
+        nodename = targpath.split('/')[2]
         currconsole = targpath
         startrequest = {'operation': 'start', 'path': targpath,
                         'parameters': {}}
@@ -658,9 +668,10 @@ def get_session_node(shellargs):
         return targ, shellargs[0]
     if len(shellargs) == 2 and shellargs[0] == 'start':
         args = [s for s in shellargs[1].split('/') if s]
-        if len(args) == 4 and args[0] == 'nodes' and args[2] == 'console' and \
-                args[3] == 'session':
-            return shellargs[1], args[1]
+        if len(args) == 4 and args[0] == 'nodes':
+            if args[2] == 'console' and \
+                    args[3] == 'session':
+                return shellargs[1], args[1]
         if len(args) == 5 and args[0] == 'nodes' and args[2] == 'shell' and \
                 args[3] == 'sessions':
             return shellargs[1], args[1]
@@ -969,8 +980,15 @@ def main():
                     sys.stdout.write('Lost connection to server')
     quitconfetty(fullexit=True)
 
+sgr_re = re.compile(r'(\x1b\[[0-9;]*m)')
+sgr_parameters_re = re.compile(r'\x1b\[([0-9;]*)m')
+fgcolor = None
+bgcolor = None
+fgshifted = False
+pendseq = ''
 def consume_termdata(fh, bufferonly=False):
     global clearpowermessage
+    global fgcolor, bgcolor, fgshifted, pendseq
     try:
         data = tlvdata.recv(fh)
     except Exception:
@@ -979,20 +997,79 @@ def consume_termdata(fh, bufferonly=False):
         updatestatus(data)
         return ''
     if data is not None:
-        data = client.stringify(data)
+        indata = pendseq + client.stringify(data)
+        pendseq = ''
+        data = ''
+        for segment in sgr_re.split(indata):
+            if sgr_re.match(segment):  # it is an sgr, analyze, maybe replace
+                params = []
+                for parameters in sgr_parameters_re.findall(segment):
+                    for param in parameters.split(';'):
+                        params.append(param)
+                        if param == '0':
+                            fgcolor = None
+                            bgcolor = None
+                        try:
+                            ival = int(param)
+                        except ValueError:
+                            continue
+                        if 40 <= ival <= 47 or 100 <= ival <= 107:
+                            bgcolor = ival
+                        if 30 <= ival <= 37 or 90 <= ival <= 97:
+                            fgcolor = ival
+                if bgcolor is not None:
+                    fgindicated = False
+                    for idx, param in enumerate(params):
+                        try:
+                            ival = int(param)
+                        except ValueError:
+                            continue
+                        if 30 <= ival <= 37 and (bgcolor % 10 == ival % 10):
+                            fgindicated = True
+                            fgshifted = True
+                            ival += 60
+                            params[idx] = str(ival)
+                    if not fgindicated and fgcolor is not None:
+                        if bgcolor and (bgcolor % 10) == (fgcolor % 10):
+                            fgshifted = True
+                            params.append(str((fgcolor % 10) + 90))
+                        elif fgshifted:
+                            params.append(str(fgcolor))
+                segment = '\x1b[' + ';'.join(str(p) for p in params) + 'm'
+            data += segment
+        # defer any partial ansi escape sequence for a later pass
+        escidx = segment.rfind('\x1b[')
+        if escidx >= 0:
+            for chr in segment[escidx + 2:]:
+                if 0x40 <= ord(chr) <= 0x7e:
+                    break
+            else:
+                # incomplete escape sequence, don't print it yet
+                data = data[:-len(segment) + escidx]
+                pendseq = segment[escidx:]
+        if not pendseq and segment and segment[-1] == '\x1b':
+            data = data[:-1]
+            pendseq = '\x1b'
         if clearpowermessage:
             sys.stdout.write("\x1b[2J\x1b[;H")
             clearpowermessage = False
         if bufferonly:
             return data
-        try:
-            sys.stdout.write(data)
-        except UnicodeEncodeError:
-            sys.stdout.buffer.write(data.encode('utf8'))
-        except IOError:  # Some times circumstances are bad
-                         # resort to byte at a time...
-            for d in data:
-                sys.stdout.write(d)
+        data = data.encode('utf8')
+        written = False
+        while not written:
+            select.select((), (sys.stdout,), ())
+            try:
+                sys.stdout.buffer.write(data)
+                written = True
+            except BlockingIOError:
+                continue
+            except IOError:  # Some times circumstances are bad
+                            # resort to byte at a time...
+                raise
+                for d in data:
+                    sys.stdout.write(d)
+                written = True
         now = time.time()
         if ('showtime' not in laststate or
                                 (now // 60) != laststate['showtime'] // 60):

confluent_client/bin/confluent2ansible

@@ -18,11 +18,6 @@ import confluent.client as client
 import confluent.sortutil as sortutil
 
 
-def lookupdata(data, key):
-    ret = data.get(key, {}).get('value', '')
-    if ret is None:
-        ret = ''
-    return ret
 
 
 def main():
@@ -59,7 +54,6 @@ def main():
         else:
             for g in groups:
                 nodesbygroup.setdefault(g, set()).add(node.strip().lower())
-
     existing_data = {}
     if options.append and os.path.exists(options.output):
         current_group = ''

confluent_client/bin/nodeapply

@@ -36,6 +36,36 @@ import confluent.client as client
 import confluent.sortutil as sortutil
 devnull = None
 
+def run_automation(noderange, category, c):
+    automationbynode = {}
+    for res in c.update('/noderange/{0}/deployment/remote_config/run'.format(noderange), {
+            'category': category,
+        }):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                exitcode |= res.get('errorcode', 1)
+            if 'created' in res:
+                nodename = res['created'].split('/')[2]
+                automationbynode[nodename] = res['created']
+    while automationbynode:
+        for node in list(automationbynode):
+            for res in c.read(automationbynode[node]):
+                if 'error' in res:
+                    sys.stderr.write(res['error'] + '\n')
+                    exitcode |= res.get('errorcode', 1)
+                for result in res.get('results', []):
+                    sys.stdout.write('{0}: Task [{1}] {2}\n'.format(
+                        node, result['task_name'], result['state']))
+                    for warning in result.get('warnings', []):
+                        sys.stderr.write('{0}: [WARNING] {1}\n'.format(node, warning))
+                    if 'errorinfo' in result:
+                        for errorline in result['errorinfo'].splitlines():
+                            sys.stderr.write('{0}: [ERROR] {1}\n'.format(node, errorline))                
+                if res.get('complete', False):
+                    del automationbynode[node]
+                    sys.stdout.write('{0}: Automation complete\n'.format(node))
+                    
+
 def run():
     global devnull
     devnull = open(os.devnull, 'rb')
@@ -51,6 +81,8 @@ def run():
                          help='Run the syncfiles associated with the currently completed OS profile on the noderange')
     argparser.add_option('-P', '--scripts',
                          help='Re-run specified scripts, with full path under scripts, e.g. post.d/first,firstboot.d/second')
+    argparser.add_option('-A', '--automation',
+                         help='Run the automation scripts associated with the current OS profile on the noderange, specifying category (onboot.d/firstboot.d/post.d)')
     argparser.add_option('-m', '--maxnodes', type='int',
                     help='Specify a maximum number of '
                         'nodes to run remote ssh command to, '
@@ -74,16 +106,13 @@ def run():
     exitcode = 0
 
     c.stop_if_noderange_over(args[0], options.maxnodes)
+    if options.automation:
+        run_automation(args[0], options.automation, c)
+        
     nodemap = {}
     cmdparms = []
     nodes = []
-    for res in c.read('/noderange/{0}/nodes/'.format(args[0])):
-        if 'error' in res:
-            sys.stderr.write(res['error'] + '\n')
-            exitcode |= res.get('errorcode', 1)
-            break
-        node = res['item']['href'][:-1]
-        nodes.append(node)
+
     
     cmdstorun = []
     if options.security:
@@ -94,8 +123,17 @@ def run():
         for script in options.scripts.split(','):
             cmdstorun.append(['run_remote', script])
     if not cmdstorun:
+        if options.automation:
+            sys.exit(0)
         argparser.print_help()
         sys.exit(1)
+    for res in c.read('/noderange/{0}/nodes/'.format(args[0])):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode |= res.get('errorcode', 1)
+            break
+        node = res['item']['href'][:-1]
+        nodes.append(node)
     idxbynode = {}
     cmdvbase = ['bash', '/etc/confluent/functions']
     for sshnode in nodes:
@@ -145,7 +183,7 @@ def run():
                                 run_cmdv(node, cmdv, all, poller, pipedesc)
                             elif pendingexecs:
                                 node, cmdv = pendingexecs.popleft()
-                                run_cmdv(node, cmdv, all, poller. pipedesc)
+                                run_cmdv(node, cmdv, all, poller, pipedesc)
             singlepoller.close()
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:

confluent_client/bin/nodeattrib

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -17,6 +17,7 @@
 
 __author__ = 'alin37'
 
+import asyncio
 from getpass import getpass
 import optparse
 import os
@@ -34,126 +35,130 @@ path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
     sys.path.append(path)
 
-import confluent.client as client
+import confluent.asynclient as client
 
-argparser = optparse.OptionParser(
-    usage='''\n       %prog [-b] noderange [list of attributes or 'all']  \
-             \n       %prog -c noderange <list of attributes>  \
-             \n       %prog -e noderange <attribute names to set>  \
-             \n       %prog noderange attribute1=value1 attribute2=value,...
-             \n ''')
-argparser.add_option('-b', '--blame', action='store_true',
-                     help='Show information about how attributes inherited')
-argparser.add_option('-e', '--environment', action='store_true',
-                     help='Set attributes, but from environment variable of '
-                          'same name')
-argparser.add_option('-c', '--clear', action='store_true',
-                     help='Clear attributes')
-argparser.add_option('-p', '--prompt', action='store_true',
-                     help='Prompt for attribute values interactively')
-argparser.add_option('-m', '--maxnodes', type='int',
-                     help='Prompt if trying to set attributes on more '
-                          'than specified number of nodes')
-argparser.add_option('-s', '--set', dest='set', metavar='settings.batch',
-                     default=False, help='set attributes using a batch file')
-(options, args) = argparser.parse_args()
+async def main():
+    argparser = optparse.OptionParser(
+        usage='''\n       %prog [-b] noderange [list of attributes or 'all']  \
+                \n       %prog -c noderange <list of attributes>  \
+                \n       %prog -e noderange <attribute names to set>  \
+                \n       %prog noderange attribute1=value1 attribute2=value,...
+                \n ''')
+    argparser.add_option('-b', '--blame', action='store_true',
+                        help='Show information about how attributes inherited')
+    argparser.add_option('-e', '--environment', action='store_true',
+                        help='Set attributes, but from environment variable of '
+                            'same name')
+    argparser.add_option('-c', '--clear', action='store_true',
+                        help='Clear attributes')
+    argparser.add_option('-p', '--prompt', action='store_true',
+                        help='Prompt for attribute values interactively')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                        help='Prompt if trying to set attributes on more '
+                            'than specified number of nodes')
+    argparser.add_option('-s', '--set', dest='set', metavar='settings.batch',
+                        default=False, help='set attributes using a batch file')
+    (options, args) = argparser.parse_args()
 
 
-#setting minimal output to only output current information
-showtype = 'current'
-requestargs=None
-try:
-    noderange = args[0]
-    nodelist = '/noderange/{0}/nodes/'.format(noderange)
-except IndexError:
-    argparser.print_help()
-    sys.exit(1)
-client.check_globbing(noderange)
-session = client.Command()
-exitcode = 0
-
-#Sets attributes
-nodetype="noderange"
-
-if len(args) > 1:
-    if "=" in args[1] or options.clear or options.environment or options.prompt:
-        if "=" in args[1] and options.clear:
-            print("Can not clear and set at the same time!")
-            argparser.print_help()
-            sys.exit(1)
-        argassign = None
-        if options.prompt:
-            argassign = {}
-            for arg in args[1:]:
-                oneval = 1
-                twoval = 2
-                while oneval != twoval:
-                    oneval = getpass('Enter value for {0}: '.format(arg))
-                    twoval = getpass('Confirm value for {0}: '.format(arg))
-                    if oneval != twoval:
-                        print('Values did not match.')
-                argassign[arg] = twoval
-        session.stop_if_noderange_over(noderange, options.maxnodes)
-        exitcode=client.updateattrib(session,args,nodetype, noderange, options, argassign)
+    #setting minimal output to only output current information
+    showtype = 'current'
+    requestargs=None
     try:
-        # setting user output to what the user inputs
-        if args[1] == 'all':
-            showtype = 'all'
-            requestargs=args[2:]
-        elif args[1] == 'current':
-            showtype = 'current'
-            requestargs=args[2:]
-        else:
-            showtype = 'all'
-            requestargs=args[1:]
-    except:
-        pass
-elif options.clear or options.environment or options.prompt:
-    sys.stderr.write('Attribute names required with specified options\n')
-    argparser.print_help()
-    exitcode = 400
+        noderange = args[0]
+        nodelist = '/noderange/{0}/nodes/'.format(noderange)
+    except IndexError:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(noderange)
+    session = client.Command()
+    exitcode = 0
 
-elif options.set:
-    arglist = [noderange]
-    showtype='current'
-    argfile = open(options.set, 'r')
-    argset = argfile.readline()
-    while argset:
+    #Sets attributes
+    nodetype="noderange"
+
+    if len(args) > 1:
+        if "=" in args[1] or options.clear or options.environment or options.prompt:
+            if "=" in args[1] and options.clear:
+                print("Can not clear and set at the same time!")
+                argparser.print_help()
+                sys.exit(1)
+            argassign = None
+            if options.prompt:
+                argassign = {}
+                for arg in args[1:]:
+                    oneval = 1
+                    twoval = 2
+                    while oneval != twoval:
+                        oneval = getpass('Enter value for {0}: '.format(arg))
+                        twoval = getpass('Confirm value for {0}: '.format(arg))
+                        if oneval != twoval:
+                            print('Values did not match.')
+                    argassign[arg] = twoval
+            await session.stop_if_noderange_over(noderange, options.maxnodes)
+            exitcode =  await client.updateattrib(session,args,nodetype, noderange, options, argassign)
         try:
-            argset = argset[:argset.index('#')]
-        except ValueError:
+            # setting user output to what the user inputs
+            if args[1] == 'all':
+                showtype = 'all'
+                requestargs=args[2:]
+            elif args[1] == 'current':
+                showtype = 'current'
+                requestargs=args[2:]
+            else:
+                showtype = 'all'
+                requestargs=args[1:]
+        except:
             pass
-        argset = argset.strip()
-        if argset:
-            arglist += shlex.split(argset)
+    elif options.clear or options.environment or options.prompt:
+        sys.stderr.write('Attribute names required with specified options\n')
+        argparser.print_help()
+        exitcode = 400
+
+    elif options.set:
+        arglist = [noderange]
+        showtype='current'
+        argfile = open(options.set, 'r')
         argset = argfile.readline()
-    session.stop_if_noderange_over(noderange, options.maxnodes)
-    exitcode=client.updateattrib(session,arglist,nodetype, noderange, options, None)
-if exitcode != 0:
+        while argset:
+            try:
+                argset = argset[:argset.index('#')]
+            except ValueError:
+                pass
+            argset = argset.strip()
+            if argset:
+                arglist += shlex.split(argset)
+            argset = argfile.readline()
+        session.stop_if_noderange_over(noderange, options.maxnodes)
+        exitcode=client.updateattrib(session,arglist,nodetype, noderange, options, None)
+    if exitcode != 0:
+        sys.exit(exitcode)
+
+    # Lists all attributes
+    if len(args) > 0:
+        # setting output to all so it can search since if we do have something to search, we want to show all outputs even if it is blank.
+        if requestargs is None:
+            showtype = 'current'
+        elif requestargs == []:
+            #showtype already set
+            pass
+        else:
+            try:
+                requestargs.remove('all')
+                requestargs.remove('current')
+            except ValueError:
+                pass
+        exitcode = await client.printattributes(session, requestargs, showtype,nodetype, noderange, options)
+    else:
+        for res in session.read(nodelist):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                exitcode = 1
+            else:
+                print(res['item']['href'].replace('/', ''))
+
     sys.exit(exitcode)
 
-# Lists all attributes
+if __name__ == '__main__':
+    asyncio.run(main())
 
-if len(args) > 0:
-    # setting output to all so it can search since if we do have something to search, we want to show all outputs even if it is blank.
-    if requestargs is None:
-        showtype = 'current'
-    elif requestargs == []:
-        #showtype already set
-        pass
-    else:
-        try:
-            requestargs.remove('all')
-            requestargs.remove('current')
-        except ValueError:
-            pass
-    exitcode = client.printattributes(session, requestargs, showtype,nodetype, noderange, options)
-else:
-    for res in session.read(nodelist):
-        if 'error' in res:
-            sys.stderr.write(res['error'] + '\n')
-            exitcode = 1
-        else:
-            print(res['item']['href'].replace('/', ''))
-
-sys.exit(exitcode)

confluent_client/bin/nodecertutil

@@ -0,0 +1,104 @@
+#!/usr/bin/python3
+
+import os
+import sys
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+import confluent.client as client
+
+def removebmccacert(noderange, certid, cmd):
+    for res in cmd.delete(f'/noderange/{noderange}/configuration/management_controller/certificate_authorities/{certid}'):
+        print(repr(res))
+
+def listbmccacerts(noderange, cmd):
+    certids = []
+    for res in cmd.read(f'/noderange/{noderange}/configuration/management_controller/certificate_authorities'):
+        certids.append(res.get('item', {}).get('href', ''))
+    for certid in certids:
+        for res in cmd.read(f'/noderange/{noderange}/configuration/management_controller/certificate_authorities/{certid}'):
+            for node in res.get('databynode', {}):
+                certdata = res['databynode'][node].get('pem', {}).get('value', '')
+                summary = ''
+                if not certdata:
+                    continue
+                san = res['databynode'][node].get('san', {}).get('value', '')
+                if san:
+                    summary += f" SANs: {san}"
+                subject = res['databynode'][node].get('subject', {}).get('value', '')
+                if subject:
+                    summary = subject
+                try:
+                    cert = x509.load_pem_x509_certificate(certdata.encode())
+                    sha256 = cert.fingerprint(hashes.SHA256()).hex().upper()
+                except Exception as e:
+                    print(f"Error processing certificate for {node}: {e}", file=sys.stderr)
+                    continue
+                summary += f" (SHA256={sha256})"
+                print(f"{node}: {certid}: {summary}")
+
+def installbmccacert(noderange, certfile, cmd):
+    if certfile:
+        try:
+            with open(certfile, 'r') as f:
+                certdata = f.read()
+        except Exception as e:
+            print(f"Error reading certificate file: {e}", file=sys.stderr)
+            sys.exit(1)
+
+        # Simple validation: check if it starts and ends with the correct PEM markers
+        if not (certdata.startswith("-----BEGIN CERTIFICATE-----") and certdata.strip().endswith("-----END CERTIFICATE-----")):
+            print("Invalid certificate format. Must be a PEM encoded certificate.", file=sys.stderr)
+            sys.exit(1)
+        payload = {'pem': certdata}
+        for res in cmd.update(f'/noderange/{noderange}/configuration/management_controller/certificate_authorities', payload):
+            print(repr(res))
+
+    
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Node certificate utility')
+    parser.add_argument('noderange', help='Node range to operate on')
+    subparsers = parser.add_subparsers(dest='command', help='Available commands')
+
+    # installbmccacert subcommand
+    install_parser = subparsers.add_parser('installbmccacert', help='Install BMC CA certificate')
+    install_parser.add_argument('filename', help='Certificate file to install')
+
+    remove_parser = subparsers.add_parser('removebmccacert', help='Remove BMC CA certificate')
+    remove_parser.add_argument('id', help='Certificate id to remove')
+
+    list_parser = subparsers.add_parser('listbmccacerts', help='List BMC CA certificates')
+
+    sign_bmc_parser = subparsers.add_parser('signbmccert', help='Sign BMC certificate')
+    sign_bmc_parser.add_argument('--days', type=int, help='Number of days the certificate is valid for')
+    sign_bmc_parser.add_argument('--added-names', type=str, help='Additional names to include in the certificate')
+
+    args = parser.parse_args()
+    c = client.Command()
+    if args.command == 'installbmccacert':
+        installbmccacert(args.noderange, args.filename, c)
+    elif args.command == 'removebmccacert':
+        removebmccacert(args.noderange, args.id, c)
+    elif args.command == 'listbmccacerts':
+        listbmccacerts(args.noderange, c)
+    elif args.command == 'signbmccert':
+        payload = {}
+        if args.days is not None:
+            payload['days'] = args.days
+        else:
+            print("Error: --days is required for signbmccert", file=sys.stderr)
+            sys.exit(1)
+        if args.added_names:
+            payload['added_names'] = args.added_names
+        for res in c.update(f'/noderange/{args.noderange}/configuration/management_controller/certificate/sign', payload):
+            print(repr(res))
+    else:
+        parser.print_help()
+        sys.exit(1)

confluent_client/bin/nodeconfig

@@ -1,7 +1,7 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2017 Lenovo
+# Copyright 2025 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+import asyncio
 import os
 import signal
 import optparse
@@ -31,7 +31,7 @@ path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
     sys.path.append(path)
 
-import confluent.client as client
+import confluent.asynclient as client
 
 class NullOpt(object):
     blame = None
@@ -62,7 +62,7 @@ argparser.add_option('-e', '--extra', dest='extra',
                            'to be extra configuration')
 argparser.add_option('-x', '--exclude', dest='exclude',
                      action='store_true', default=False,
-                     help='Treat positional arguments as items to not '
+                     help='Treat named settings as items to not '
                           'examine, compare, or restore default')
 argparser.add_option('-a', '--advanced', dest='advanced',
                      action='store_true', default=False,
@@ -73,7 +73,7 @@ argparser.add_option('-r', '--restoredefault', default=False,
                      dest='restoredefault', metavar="COMPONENT",
                      help='Restore the configuration of the node '
                           'to factory default for given component. '
-                          'Currently only uefi is supported')
+                          'Currently "uefi" and "bmc" are supported components')
 argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '
                           'nodes to configure, '
@@ -96,6 +96,12 @@ cfgpaths = {
     'bmc.static_ipv6_gateway': (
         'configuration/management_controller/net_interfaces/management',
         'static_v6_gateway'),
+    'bmc.vlan_id': (
+        'configuration/management_controller/net_interfaces/management',
+        'vlan_id'),
+    'bmc.mac_address': (
+        'configuration/management_controller/net_interfaces/management',
+        'hw_addr'),
     'bmc.hostname': (
         'configuration/management_controller/hostname', 'hostname'),
 }
@@ -216,109 +222,114 @@ def parse_config_line(arguments, single=False):
                     queryparms[path] = {}
                 queryparms[path][attrib] = param
 
-if options.batch:
-    printsys = []
-    argfile = open(options.batch, 'r')
-    argset = argfile.readline()
-    while argset:
-        try:
-            argset = argset[:argset.index('#')]
-        except ValueError:
-            pass
-        argset = argset.strip()
-        if argset:
-            parse_config_line(shlex.split(argset), single=True)
+async def main():
+    global printsys
+    if options.batch:
+        printsys = []
+        argfile = open(options.batch, 'r')
         argset = argfile.readline()
-else:
-    parse_config_line(args[1:])
-session = client.Command()
-rcode = 0
-if options.restoredefault:
-    session.stop_if_noderange_over(noderange, options.maxnodes)
-    if options.restoredefault.lower() in (
-        'sys', 'system', 'uefi', 'bios'):
-        for fr in session.update(
-                '/noderange/{0}/configuration/system/clear'.format(noderange),
-                {'clear': True}):
-            rcode |= client.printerror(fr)
-        sys.exit(rcode)
-    elif options.restoredefault.lower() in (
-        'bmc', 'imm', 'xcc'):
-        for fr in session.update(
-                '/noderange/{0}/configuration/management_controller/clear'.format(noderange),
-                {'clear': True}):
-            rcode |= client.printerror(fr)
-        sys.exit(rcode)
+        while argset:
+            try:
+                argset = argset[:argset.index('#')]
+            except ValueError:
+                pass
+            argset = argset.strip()
+            if argset:
+                parse_config_line(shlex.split(argset), single=True)
+            argset = argfile.readline()
     else:
-        sys.stderr.write(
-            'Unrecognized component to restore defaults: {0}\n'.format(
-                options.restoredefault))
-        sys.exit(1)
-if setmode:
-    session.stop_if_noderange_over(noderange, options.maxnodes)
-    if options.exclude:
-        sys.stderr.write('Cannot use exclude and assign at the same time\n')
-        sys.exit(1)
-    updatebypath = {}
-    attrnamebypath = {}
-    for key in assignment:
-        if key not in cfgpaths:
-            if key.startswith('bmc.'):
-                path = 'configuration/management_controller/extended/all'
-                attrib = key.replace('bmc.', '')
-            else:
-                path = 'configuration/system/all'
-                attrib = key
+        parse_config_line(args[1:])
+    session = client.Command()
+    rcode = 0
+    if options.restoredefault:
+        await session.stop_if_noderange_over(noderange, options.maxnodes)
+        if options.restoredefault.lower() in (
+            'sys', 'system', 'uefi', 'bios'):
+            async for fr in session.update(
+                    '/noderange/{0}/configuration/system/clear'.format(noderange),
+                    {'clear': True}):
+                rcode |= client.printerror(fr)
+            sys.exit(rcode)
+        elif options.restoredefault.lower() in (
+            'bmc', 'imm', 'xcc'):
+            async for fr in session.update(
+                    '/noderange/{0}/configuration/management_controller/clear'.format(noderange),
+                    {'clear': True}):
+                rcode |= client.printerror(fr)
+            sys.exit(rcode)
         else:
-            path, attrib = cfgpaths[key]
-        if path not in updatebypath:
-            updatebypath[path] = {}
-            attrnamebypath[path] = {}
-        updatebypath[path][attrib] = assignment[key]
-        attrnamebypath[path][attrib] = key
-    # well, we want to expand things..
-    # check ipv4, if requested change method to static
-    for path in updatebypath:
-        for fr in session.update('/noderange/{0}/{1}'.format(noderange, path),
-                                updatebypath[path]):
-            rcode |= client.printerror(fr)
-            for node in fr.get('databynode', []):
-                r = fr['databynode'][node]
-                if 'value' not in r:
-                    continue
-                keyval = r['value']
-                key, val = keyval.split('=')
-                if key in attrnamebypath[path]:
-                    key = attrnamebypath[path][key]
-                print('{0}: {1}: {2}'.format(node, key, val))
-else:
-    for path in queryparms:
-        if options.comparedefault:
-            continue
-        rcode |= client.print_attrib_path(path, session, list(queryparms[path]),
-                                      NullOpt(), queryparms[path])
-    if printsys == 'all' or printextbmc or printbmc or printallbmc:
-        if printbmc or not printextbmc:
-         rcode |= client.print_attrib_path(
-                        '/noderange/{0}/configuration/management_controller/extended/all'.format(noderange),
-                        session, printbmc, options, attrprefix='bmc.')
-        if options.extra:
-            if options.advanced:
-                   rcode |= client.print_attrib_path(
-                        '/noderange/{0}/configuration/management_controller/extended/extra_advanced'.format(noderange),
-                        session, printextbmc, options)
+            sys.stderr.write(
+                'Unrecognized component to restore defaults: {0}\n'.format(
+                    options.restoredefault))
+            sys.exit(1)
+    if setmode:
+        await session.stop_if_noderange_over(noderange, options.maxnodes)
+        if options.exclude:
+            sys.stderr.write('Cannot use exclude and assign at the same time\n')
+            sys.exit(1)
+        updatebypath = {}
+        attrnamebypath = {}
+        for key in assignment:
+            if key not in cfgpaths:
+                if key.startswith('bmc.'):
+                    path = 'configuration/management_controller/extended/all'
+                    attrib = key.replace('bmc.', '')
+                else:
+                    path = 'configuration/system/all'
+                    attrib = key
             else:
-                rcode |= client.print_attrib_path(
-                        '/noderange/{0}/configuration/management_controller/extended/extra'.format(noderange),
-                        session, printextbmc, options)
-    if printsys or options.exclude:
-        if printsys == 'all':
-            printsys = []
-        if (options.comparedefault or printsys == []) and not options.advanced:
-            path = '/noderange/{0}/configuration/system/all'.format(noderange)
-        else:
-            path = '/noderange/{0}/configuration/system/advanced'.format(
-                noderange)
-        rcode = client.print_attrib_path(path, session, printsys,
-                                         options)
-sys.exit(rcode)
+                path, attrib = cfgpaths[key]
+            if path not in updatebypath:
+                updatebypath[path] = {}
+                attrnamebypath[path] = {}
+            updatebypath[path][attrib] = assignment[key]
+            attrnamebypath[path][attrib] = key
+        # well, we want to expand things..
+        # check ipv4, if requested change method to static
+        for path in updatebypath:
+            async for fr in session.update('/noderange/{0}/{1}'.format(noderange, path),
+                                    updatebypath[path]):
+                rcode |= client.printerror(fr)
+                for node in fr.get('databynode', []):
+                    r = fr['databynode'][node]
+                    if 'value' not in r:
+                        continue
+                    keyval = r['value']
+                    key, val = keyval.split('=')
+                    if key in attrnamebypath[path]:
+                        key = attrnamebypath[path][key]
+                    print('{0}: {1}: {2}'.format(node, key, val))
+    else:
+        for path in queryparms:
+            if options.comparedefault:
+                continue
+            rcode |= await client.print_attrib_path(path, session, list(queryparms[path]),
+                                        NullOpt(), queryparms[path])
+        if printsys == 'all' or printextbmc or printbmc or printallbmc:
+            if printbmc or not printextbmc:
+                rcode |= await client.print_attrib_path(
+                            '/noderange/{0}/configuration/management_controller/extended/all'.format(noderange),
+                            session, printbmc, options, attrprefix='bmc.')
+            if options.extra:
+                if options.advanced:
+                        rcode |= await client.print_attrib_path(
+                            '/noderange/{0}/configuration/management_controller/extended/extra_advanced'.format(noderange),
+                            session, printextbmc, options)
+                else:
+                    rcode |= await client.print_attrib_path(
+                            '/noderange/{0}/configuration/management_controller/extended/extra'.format(noderange),
+                            session, printextbmc, options)
+        if printsys or options.exclude:
+            if printsys == 'all':
+                printsys = []
+            if (options.comparedefault or printsys == []) and not options.advanced:
+                path = '/noderange/{0}/configuration/system/all'.format(noderange)
+            else:
+                path = '/noderange/{0}/configuration/system/advanced'.format(
+                    noderange)
+            rcode = await client.print_attrib_path(path, session, printsys,
+                                            options)
+    sys.exit(rcode)
+
+if __name__ == '__main__':
+    asyncio.run(main())

confluent_client/bin/nodeconsole

@@ -134,7 +134,9 @@ def determine_tile_size(numnodes):
     # from kitty by omitting, but:
     # then we don't know how much to move the cursor left after draw_image
     # Konsole won't scale at all with only partial scaling specified
-    cheight, cwidth, pixwidth, pixheight = sq.get_screengeom()
+    direct_console()
+    cheight, cwidth, pixwidth, pixheight = sq.get_screengeom(escfallback=True)
+    indirect_console()
     # 16:12 is to roughly account for the 'titles' of the tiles
     ratio = (pixwidth / 16) / (pixheight / 12)
     bestdeviation = None
@@ -214,7 +216,9 @@ def cursor_show():
     sys.stdout.write('\x1b[?25h')
 
 def get_pix_dimensions(width, height):
-    cheight, cwidth, pixwidth, pixheight = sq.get_screengeom()
+    direct_console()
+    cheight, cwidth, pixwidth, pixheight = sq.get_screengeom(escfallback=True)
+    indirect_console()
     imgwidth = int(pixwidth / cwidth * width)
     imgheight = int(pixheight / cheight * height)
     return imgwidth, imgheight

confluent_client/bin/nodedefine

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import asyncio
 import optparse
 import os
 import signal
@@ -30,29 +31,33 @@ path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
     sys.path.append(path)
 
-import confluent.client as client
+import confluent.asynclient as client
 
-argparser = optparse.OptionParser(
-    usage='''\n       %prog noderange attribute1=value1 attribute2=value,...
-             \n ''')
-(options, args) = argparser.parse_args()
-requestargs=None
-try:
-    noderange = args[0]
-except IndexError:
-    argparser.print_help()
-    sys.exit(1)
-client.check_globbing(noderange)
-session = client.Command()
-exitcode = 0
-attribs = {'name': noderange}
-for arg in args[1:]:
-    key, val = arg.split('=', 1)
-    attribs[key] = val
-for r in session.create('/noderange/', attribs):
-    if 'error' in r:
-        sys.stderr.write(r['error'] + '\n')
-        exitcode |= 1
-    if 'created' in r:
-        print('{0}: created'.format(r['created']))
-sys.exit(exitcode)
+async def main():
+    argparser = optparse.OptionParser(
+        usage='''\n       %prog noderange attribute1=value1 attribute2=value,...
+                \n ''')
+    (options, args) = argparser.parse_args()
+    requestargs = None
+    try:
+        noderange = args[0]
+    except IndexError:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(noderange)
+    session = client.Command()
+    exitcode = 0
+    attribs = {'name': noderange}
+    for arg in args[1:]:
+        key, val = arg.split('=', 1)
+        attribs[key] = val
+    async for r in session.create('/noderange/', attribs):
+        if 'error' in r:
+            sys.stderr.write(r['error'] + '\n')
+            exitcode |= 1
+        if 'created' in r:
+            print('{0}: created'.format(r['created']))
+    sys.exit(exitcode)
+
+if __name__ == '__main__':
+    asyncio.run(main())

confluent_client/bin/nodedeploy

@@ -48,7 +48,18 @@ def armonce(nr, cli):
         pass
 
 
-def setpending(nr, profile, cli):
+def setpending(nr, profile, profilebynodes, cli):
+    if profilebynodes:
+        for node in sortutil.natural_sort(profilebynodes):
+            prof = profilebynodes[node]
+            args = {'deployment.pendingprofile': prof, 'deployment.state': '', 'deployment.state_detail': ''}
+            if not prof.startswith('genesis-'):
+                args['deployment.stagedprofile'] = ''
+                args['deployment.profile'] = ''
+            for rsp in cli.update('/nodes/{0}/attributes/current'.format(node),
+                                  args):
+                pass
+        return
     args = {'deployment.pendingprofile': profile, 'deployment.state': '', 'deployment.state_detail': ''}
     if not profile.startswith('genesis-'):
         args['deployment.stagedprofile'] = ''
@@ -69,6 +80,7 @@ def main(args):
     ap.add_argument('-n', '--network', help='Initiate deployment over PXE/HTTP', action='store_true')
     ap.add_argument('-p', '--prepareonly', help='Prepare only, skip any interaction with a BMC associated with this deployment action', action='store_true')
     ap.add_argument('-m', '--maxnodes', help='Specifiy a maximum nodes to be deployed')
+    ap.add_argument('-r', '--redeploy', help='Redeploy nodes with the current or pending profile', action='store_true')
     ap.add_argument('noderange', help='Set of nodes to deploy')
     ap.add_argument('profile', nargs='?', help='Profile name to deploy')
     args, extra = ap.parse_known_args(args)
@@ -78,7 +90,7 @@ def main(args):
     if args.profile and not args.network:
         sys.stderr.write('-n is a required argument currently to perform an install, optionally with -p\n')
         return 1
-    if not args.profile and args.network:
+    if not args.profile and args.network and not args.redeploy:
         sys.stderr.write('Both noderange and a profile name are required arguments to request a network deployment\n')
         return 1
     if args.clear and args.profile:
@@ -96,27 +108,38 @@ def main(args):
         if 'error' in rsp:
             sys.stderr.write(rsp['error'] + '\n')
             sys.exit(1)
+    profilebynode = {}
     if args.clear:
         cleararm(args.noderange, c)
         clearpending(args.noderange, c)
-    elif args.profile:
-        profnames = []
-        for prof in c.read('/deployment/profiles/'):
-            profname = prof.get('item', {}).get('href', None)
-            if profname:
-                profname = profname.replace('/', '')
-                profnames.append(profname)
-                if profname == args.profile:
-                    break
-        else:
-            sys.stderr.write('The specified profile "{}" is not an available profile\n'.format(args.profile))
-            if profnames:
-                sys.stderr.write('The following profiles are available:\n')
-                for profname in profnames:
-                    sys.stderr.write('    ' + profname + '\n')
-            else:
-                sys.stderr.write('No deployment profiles available, try osdeploy import or imgutil capture\n')
-            sys.exit(1)
+    elif args.redeploy:
+        hadpending = {}
+        for rsp in c.read('/noderange/{0}/attributes/current'.format(args.noderange)):
+            for node in rsp.get('databynode', {}):
+                nodeinfo = rsp['databynode'][node]
+                for attr in nodeinfo:
+                    if attr == 'deployment.pendingprofile':
+                        curr = nodeinfo[attr].get('value', '')
+                        if curr:
+                            hadpending[node] = True
+                            profilebynode[node] = curr
+                    if attr == 'deployment.stagedprofile':
+                        curr = nodeinfo[attr].get('value', '')
+                        if curr and node not in hadpending:
+                            profilebynode[node] = curr
+                    if attr == 'deployment.profile':
+                        curr = nodeinfo[attr].get('value', '')
+                        if curr and node not in profilebynode:
+                            profilebynode[node] = curr
+        for lockinfo in c.read('/noderange/{0}/deployment/lock'.format(args.noderange)):
+            for node in lockinfo.get('databynode', {}):
+                lockstate = lockinfo['databynode'][node]['lock']['value']
+                if lockstate == 'locked':
+                    lockednodes.append(node)     
+    if args.profile and profilebynode:
+        sys.stderr.write('The -r/--redeploy option cannot be used with a profile, it redeploys the current or pending profile\n')
+        return 1
+    if args.profile or profilebynode:
         lockednodes = []
         for lockinfo in c.read('/noderange/{0}/deployment/lock'.format(args.noderange)):
             for node in lockinfo.get('databynode', {}):
@@ -127,8 +150,26 @@ def main(args):
             sys.stderr.write('Requested noderange has nodes with locked deployment: ' + ','.join(lockednodes))
             sys.stderr.write('\n')
             sys.exit(1)
+        if args.profile:
+            profnames = []
+            for prof in c.read('/deployment/profiles/'):
+                profname = prof.get('item', {}).get('href', None)
+                if profname:
+                    profname = profname.replace('/', '')
+                    profnames.append(profname)
+                    if profname == args.profile:
+                        break
+            else:
+                sys.stderr.write('The specified profile "{}" is not an available profile\n'.format(args.profile))
+                if profnames:
+                    sys.stderr.write('The following profiles are available:\n')
+                    for profname in profnames:
+                        sys.stderr.write('    ' + profname + '\n')
+                else:
+                    sys.stderr.write('No deployment profiles available, try osdeploy import or imgutil capture\n')
+                sys.exit(1)
         armonce(args.noderange, c)
-        setpending(args.noderange, args.profile, c)
+        setpending(args.noderange, args.profile, profilebynode, c)
     else:
         databynode = {}
         for r in c.read('/noderange/{0}/attributes/current'.format(args.noderange)):

confluent_client/bin/nodediscover

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import asyncio
 import csv
 import optparse
 import os
@@ -26,7 +27,7 @@ path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
     sys.path.append(path)
 
-import confluent.client as client
+import confluent.asynclient as client
 import confluent.sortutil as sortutil
 
 defcolumns = ['Node', 'Model', 'Serial', 'UUID', 'Mac Address', 'Type',
@@ -81,16 +82,15 @@ def subscribe_discovery(options, session, subscribe, targ):
             if 'status' in rsp:
                 print(rsp['status'])
     
-def print_disco(options, session, currmac, outhandler, columns):
+async def print_disco(options, session, currmac, outhandler, columns):
     procinfo = {}
-    for tmpinfo in session.read('/discovery/by-mac/{0}'.format(currmac)):
-
+    async for tmpinfo in session.read('/discovery/by-mac/{0}'.format(currmac)):
         procinfo.update(tmpinfo)
     if 'Switch' in columns or 'Port' in columns:
         if 'switch' in procinfo:
             procinfo['port'] = procinfo['switchport']
         else:
-            for tmpinfo in session.read(
+            async for tmpinfo in session.read(
                     '/networking/macs/by-mac/{0}'.format(currmac)):
                 if 'ports' in tmpinfo:
                     # The api sorts so that the most specific available value
@@ -160,12 +160,12 @@ def datum_complete(datum):
 searchkeys = set(['mac', 'serial', 'uuid'])
 
 
-def search_record(datum, options, session):
+async def search_record(datum, options, session):
     for searchkey in searchkeys:
         options.__dict__[searchkey] = None
     for searchkey in searchkeys & set(datum):
         options.__dict__[searchkey] = datum[searchkey]
-    return list(list_matching_macs(options, session))
+    return [x async for x in list_matching_macs(options, session)]
 
 
 def datum_to_attrib(datum):
@@ -180,7 +180,7 @@ def datum_to_attrib(datum):
 
 unique_fields = frozenset(['serial', 'mac', 'uuid'])
 
-def import_csv(options, session):
+async def import_csv(options, session):
     nodedata = []
     unique_data = {}
     exitcode = 0
@@ -213,7 +213,7 @@ def import_csv(options, session):
         for nodedatum in alldata:
             if not search_record(nodedatum, options, session) and not broken:
                 allthere = False
-                blocking_scan(session)
+                await blocking_scan(session)
                 break
         for nodedatum in alldata:
             if not allthere and not search_record(nodedatum, options, session):
@@ -262,7 +262,7 @@ def import_csv(options, session):
         sys.exit(exitcode)
 
 
-def list_discovery(options, session):
+async def list_discovery(options, session):
     orderby = None
     if options.fields:
         columns = []
@@ -278,23 +278,24 @@ def list_discovery(options, session):
             if options.order.lower() == field.lower():
                 orderby = field
     outhandler = client.Tabulator(columns)
-    for mac in list_matching_macs(options, session):
-        print_disco(options, session, mac, outhandler, columns)
+    for mac in [x async for x in list_matching_macs(options, session)]:
+        await print_disco(options, session, mac, outhandler, columns)
     if options.csv:
         outhandler.write_csv(sys.stdout, orderby)
     else:
         for row in outhandler.get_table(orderby):
             print(row)
 
-def clear_discovery(options, session):
-    for mac in list_matching_macs(options, session):
-        for res in session.delete('/discovery/by-mac/{0}'.format(mac)):
+async def clear_discovery(options, session):
+    allmacs = [x async for x in list_matching_macs(options, session)]
+    for mac in allmacs:
+        async for res in session.delete('/discovery/by-mac/{0}'.format(mac)):
             if 'deleted' in res:
                 print('Cleared info for {0}'.format(res['deleted']))
             else:
                 print(repr(res))
 
-def list_matching_macs(options, session, node=None, checknode=True):
+async def list_matching_macs(options, session, node=None, checknode=True):
     path = '/discovery/'
     if node:
         path += 'by-node/{0}/'.format(node)
@@ -314,22 +315,21 @@ def list_matching_macs(options, session, node=None, checknode=True):
         path += 'by-state/{0}/'.format(options.state).lower()
     if options.mac:
         path += 'by-mac/{0}'.format(options.mac)
-        result = list(session.read(path))[0]
+        result = list([x async for x in session.read(path)])[0]
         if 'error' in result:
-            return []
-        return [options.mac.replace(':', '-')]
+            return
+        yield options.mac.replace(':', '-')
+        return
     else:
         path += 'by-mac/'
-        ret = []
-        for x in session.read(path):
+        async for x in session.read(path):
             if 'item' in x and 'href' in x['item']:
-                ret.append(x['item']['href'])
-        return ret
+                yield x['item']['href']
 
-def assign_discovery(options, session, needid=True):
+async def assign_discovery(options, session, needid=True):
     abort = False
     if options.importfile:
-        return import_csv(options, session)
+        return await import_csv(options, session)
     if not options.node:
         sys.stderr.write("Node (-n) must be specified for assignment\n")
         abort = True
@@ -340,16 +340,16 @@ def assign_discovery(options, session, needid=True):
         abort = True
     if abort:
         sys.exit(1)
-    matches = list_matching_macs(options, session, None if needid else options.node, False)
+    matches = [x async for x in list_matching_macs(options, session, None if needid else options.node, False)]
     if not matches:
         # Do a rescan to catch missing requested data
-        blocking_scan(session)
-        matches = list_matching_macs(options, session, None if needid else options.node, False)
+        await blocking_scan(session)
+        matches = [x async for x in list_matching_macs(options, session, None if needid else options.node, False)]
     if not matches:
         sys.stderr.write("No matching discovery candidates found\n")
         sys.exit(1)
     exitcode = 0
-    for res in session.update('/discovery/by-mac/{0}'.format(matches[0]),
+    async for res in session.update('/discovery/by-mac/{0}'.format(matches[0]),
                    {'node': options.node}):
         if 'assigned' in res:
             print('Assigned: {0}'.format(res['assigned']))
@@ -361,14 +361,14 @@ def assign_discovery(options, session, needid=True):
     if exitcode:
         sys.exit(exitcode)
 
-def blocking_scan(session):
-    list(session.update('/discovery/rescan', {'rescan': 'start'}))
-    while(list(session.read('/discovery/rescan'))[0].get('scanning', False)):
+async def blocking_scan(session):
+    list([x async for x in session.update('/discovery/rescan', {'rescan': 'start'})])
+    while(list([x async for x in session.read('/discovery/rescan')])[0].get('scanning', False)):
         time.sleep(0.5)
-    list(session.update('/networking/macs/rescan', {'rescan': 'start'}))
+    list([x async for x in session.update('/networking/macs/rescan', {'rescan': 'start'})])
 
 
-def main():
+async def main():
     parser = optparse.OptionParser(
         usage='Usage: %prog [list|assign|rescan|clear|subscribe|unsubscribe|register] [options]')
     # -a for 'address' maybe?
@@ -419,13 +419,13 @@ def main():
         sys.exit(1)
     session = client.Command()
     if args[0] == 'list':
-        list_discovery(options, session)
+        await list_discovery(options, session)
     if args[0] == 'clear':
-        clear_discovery(options, session)
+        await clear_discovery(options, session)
     if args[0] == 'assign':
-        assign_discovery(options, session)
+        await assign_discovery(options, session)
     if args[0] == 'reassign':
-        assign_discovery(options, session, False)
+        await assign_discovery(options, session, False)
     if args[0] == 'register':
         register_endpoint(options, session, args[1])
     if args[0] == 'subscribe':
@@ -433,9 +433,9 @@ def main():
     if args[0] == 'unsubscribe':
         subscribe_discovery(options, session, False, args[1])
     if args[0] == 'rescan':
-        blocking_scan(session)
+        await blocking_scan(session)
         print("Rescan complete")
 
 
 if __name__ == '__main__':
-    main()
+    asyncio.run(main())

confluent_client/bin/nodefirmware

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -114,11 +114,24 @@ def update_firmware(session, filename):
         upargs['bank'] = 'backup'
     noderrs = {}
     if session.unixdomain:
-        of = open(filename, 'rb')
-        try:
-            session.add_file(filename, of.fileno(), 'rb')
-        except Exception:
-            pass
+        filesbynode = {}
+        for exp in session.create('/noderange/{0}/attributes/expression'.format(noderange),
+                        {'expression': filename}):
+            if 'error' in exp:
+                sys.stderr.write(exp['error'] + '\n')
+                exitcode |= exp.get('errorcode', 1)
+            ex = exp.get('databynode', ())
+            for node in ex:
+                filesbynode[node] = ex[node]['value']
+                if not isinstance(filesbynode[node], bytes) and not isinstance(filesbynode[node], str):
+                    filesbynode[node] = filesbynode[node].encode('utf-8')
+        for node in filesbynode:
+            endfilename = filesbynode[node]
+            of = open(endfilename, 'rb')
+            try:
+                session.add_file(endfilename, of.fileno(), 'rb')
+            except Exception:
+                pass
     for res in session.create(resource, upargs):
         if 'created' not in res:
             for nodename in res.get('databynode', ()):
@@ -153,9 +166,13 @@ def show_firmware(session):
     firmware_shown = False
     nodes_matched = False
     for component in components:
+        category = 'all'
+        if component in ('adapters', 'disks', 'misc', 'core'):
+            category = component
+            component = 'all'    
         for res in session.read(
-                '/noderange/{0}/inventory/firmware/all/{1}'.format(
-                    noderange, component)):
+                '/noderange/{0}/inventory/firmware/{2}/{1}'.format(
+                    noderange, component, category)):
             nodes_matched = True
             exitcode |= client.printerror(res)
             if 'databynode' not in res:

confluent_client/bin/nodehealth

@@ -84,13 +84,13 @@ def main():
                 healthexplanations[node] = []
                 for sensor in health[node]['sensors']:
                     explanation = sensor['name'] + ':'
-                    if sensor['value'] is not None:
+                    if sensor.get('value', None) is not None:
                         explanation += str(sensor['value'])
-                        if sensor['units'] is not None:
+                        if sensor.get('units', None) is not None:
                             explanation += sensor['units']
-                        if sensor['states']:
+                        if sensor.get('states', None):
                             explanation += ','
-                    if sensor['states']:
+                    if sensor.get('states', None):
                         explanation += ','.join(sensor['states'])
                     healthexplanations[node].append(explanation)
             if node in healthbynode and node in healthexplanations:

confluent_client/bin/nodeinventory

@@ -49,7 +49,9 @@ def pretty(text):
 
 def print_mem_info(node, prefix, meminfo):
     memdescfmt = '{0}GB PC'
-    if meminfo['memory_type'] == 'DDR3 SDRAM':
+    if meminfo['memory_type'] is None:
+        memdescfmt = '{0}GB '
+    elif meminfo['memory_type'] == 'DDR3 SDRAM':
         memdescfmt += '3-{1} '
     elif 'DDR4' in meminfo['memory_type']:
         memdescfmt += '4-{1} '
@@ -58,16 +60,21 @@ def print_mem_info(node, prefix, meminfo):
     elif 'DCPMM' in meminfo['memory_type']:
         memdescfmt = '{0}GB {1} '
         meminfo['module_type'] = 'DCPMM'
+    elif meminfo['memory_type'] == 'HBM':
+        memdescfmt = '{0}GB HBM '
     else:
         print('{0}: {1}: Unrecognized Memory'.format(node, prefix))
         return
     if meminfo.get('ecc', False):
         memdescfmt += 'ECC '
-    capacity = meminfo['capacity_mb'] / 1024
     modtype = meminfo.get('module_type', None)
     if modtype:
         memdescfmt += modtype
-    memdesc = memdescfmt.format(capacity, meminfo['speed'])
+    if meminfo.get('capacity_mb', None):
+        capacity = meminfo['capacity_mb'] // 1024
+        memdesc = memdescfmt.format(capacity, meminfo['speed'])
+    else:
+        memdesc = 'Unspecified Module'
     print('{0}: {1} description: {2}'.format(node, prefix, memdesc))
     print('{0}: {1} manufacturer: {2}'.format(
         node, prefix, meminfo['manufacturer']))

confluent_client/bin/nodelicense

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2019 Lenovo
@@ -65,16 +65,30 @@ client.check_globbing(noderange)
 
 def install_license(session, filename):
     global exitcode
+    
     resource = '/noderange/{0}/configuration/' \
                'management_controller/licenses/'.format(noderange)
     filename = os.path.abspath(filename)
     instargs = {'filename': filename}
     if session.unixdomain:
-        of = open(filename, 'rb')
-        try:
-            session.add_file(filename, of.fileno(), 'rb')
-        except Exception:
-            pass
+        filesbynode = {}
+        for exp in session.create('/noderange/{0}/attributes/expression'.format(noderange),
+                        {'expression': filename}):
+            if 'error' in exp:
+                sys.stderr.write(exp['error'] + '\n')
+                exitcode |= exp.get('errorcode', 1)
+            ex = exp.get('databynode', ())
+            for node in ex:
+                filesbynode[node] = ex[node]['value']
+                if not isinstance(filesbynode[node], bytes) and not isinstance(filesbynode[node], str):
+                    filesbynode[node] = filesbynode[node].encode('utf-8')
+        for node in filesbynode:
+            endfilename = filesbynode[node]
+            of = open(endfilename, 'rb')
+            try:
+                session.add_file(endfilename, of.fileno(), 'rb')
+            except Exception:
+                pass
     for res in session.create(resource, instargs):
         for node in res.get('databynode', []):
             if 'error' in res['databynode'][node]:

confluent_client/bin/nodelist

@@ -1,4 +1,4 @@
-#!/usr/libexec/platform-python
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -21,6 +21,7 @@ import optparse
 import os
 import signal
 import sys
+import asyncio
 
 
 
@@ -33,9 +34,9 @@ path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
     sys.path.append(path)
 
-import confluent.client as client
+import confluent.asynclient as client
 
-def main():
+async def main():
     argparser = optparse.OptionParser(
     usage="Usage: %prog noderange\n"
           "   or: %prog [options] noderange <nodeattribute>...")
@@ -59,9 +60,9 @@ def main():
     requestargs=args[1:]
     nodetype='noderange'
     if len(args) > 1:
-        exitcode=client.printattributes(session,requestargs,showtype,nodetype,noderange,options)
+        exitcode=await client.printattributes(session,requestargs,showtype,nodetype,noderange,options)
     else:
-        for res in session.read(nodelist):
+        async for res in session.read(nodelist):
             if 'error' in res:
                 sys.stderr.write(res['error'] + '\n')
                 exitcode = 1
@@ -73,4 +74,4 @@ def main():
     sys.exit(exitcode)
 
 if __name__ == '__main__':
-    main()
+    asyncio.run(main())

confluent_client/bin/nodemedia

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2018 Lenovo
@@ -101,22 +101,37 @@ def detach_media(noderange, media):
 
 def upload_media(noderange, media):
     global exitcode
-    if not os.path.exists(media):
-        sys.stderr.write('Unable to locate requested file {0}\n'.format(
-            media))
-        sys.exit(404)
+
     session = client.Command()
     output = sq.ScreenPrinter(noderange, session)
     filename = os.path.abspath(media)
     resource = '/noderange/{0}/media/uploads/'.format(noderange)
+    filename = os.path.abspath(filename)
     upargs = {'filename': filename}
     noderrs = {}
     if session.unixdomain:
-        of = open(filename, 'rb')
-        try:
-            session.add_file(filename, of.fileno(), 'rb')
-        except Exception:
-            pass
+        filesbynode = {}
+        for exp in session.create('/noderange/{0}/attributes/expression'.format(noderange),
+                        {'expression': filename}):
+            if 'error' in exp:
+                sys.stderr.write(exp['error'] + '\n')
+                exitcode |= exp.get('errorcode', 1)
+            ex = exp.get('databynode', ())
+            for node in ex:
+                filesbynode[node] = ex[node]['value']
+                if not isinstance(filesbynode[node], bytes) and not isinstance(filesbynode[node], str):
+                    filesbynode[node] = filesbynode[node].encode('utf-8')
+        for node in filesbynode:
+            endfilename = filesbynode[node]
+            if not os.path.exists(endfilename):
+                sys.stderr.write('Unable to locate requested file {0}\n'.format(
+                    endfilename))
+                sys.exit(404)
+            of = open(endfilename, 'rb')
+            try:
+                session.add_file(endfilename, of.fileno(), 'rb')
+            except Exception:
+                pass
     nodeurls = {}
     for res in session.create(resource, upargs):
         if 'created' not in res:

confluent_client/bin/nodepower

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo

confluent_client/bin/noderemove

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import asyncio
 import optparse
 import os
 import signal
@@ -32,26 +33,32 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-argparser = optparse.OptionParser(
-    usage='''\n       %prog <noderange>
+
+async def main():
+    argparser = optparse.OptionParser(
+        usage='''\n       %prog <noderange>
              \n ''')
-argparser.add_option('-m', '--maxnodes', type='int',
-                     help='Specify a maximum number of '
-                          'nodes to delete, '
-                          'prompting if over the threshold')      
-(options, args) = argparser.parse_args()
-if len(args) != 1:
-    argparser.print_help()
-    sys.exit(1)
-noderange = args[0]
-client.check_globbing(noderange)
-session = client.Command()
-exitcode = 0
-session.stop_if_noderange_over(noderange, options.maxnodes)
-for r in session.delete('/noderange/{0}'.format(noderange)):
-    if 'error' in r:
-        sys.stderr.write(r['error'] + '\n')
-        exitcode |= 1
-    if 'deleted' in r:
-        print('{0}: deleted'.format(r['deleted']))
-sys.exit(exitcode)
+    argparser.add_option('-m', '--maxnodes', type='int',
+                         help='Specify a maximum number of '
+                              'nodes to delete, '
+                              'prompting if over the threshold')
+    (options, args) = argparser.parse_args()
+    if len(args) != 1:
+        argparser.print_help()
+        sys.exit(1)
+    noderange = args[0]
+    client.check_globbing(noderange)
+    session = client.Command()
+    exitcode = 0
+    await session.stop_if_noderange_over(noderange, options.maxnodes)
+    async for r in session.delete('/noderange/{0}'.format(noderange)):
+        if 'error' in r:
+            sys.stderr.write(r['error'] + '\n')
+            exitcode |= 1
+        if 'deleted' in r:
+            print('{0}: deleted'.format(r['deleted']))
+    sys.exit(exitcode)
+
+
+if __name__ == '__main__':
+    asyncio.get_event_loop().run_until_complete(main())

confluent_client/bin/nodesensors

@@ -123,7 +123,7 @@ def sensorpass(showout=True, appendtime=False):
                 if 'sensors' not in reading[node]:
                     continue
                 for sensedata in reading[node]['sensors']:
-                    if sensedata['value'] is None and options.skipnumberless:
+                    if sensedata.get('value', None) is None and options.skipnumberless:
                         continue
                     for redundant_state in ('Non-Critical', 'Critical'):
                         try:
@@ -134,17 +134,17 @@ def sensorpass(showout=True, appendtime=False):
                     resultdata[node][sensedata['name']] = sensedata
                     sensorname = sensedata['name']
                     sensorheaders[sensorname] = sensorname
-                    if sensedata['units'] not in (None, u''):
+                    if sensedata.get('units', None) not in (None, u''):
                         sensorheaders[sensorname] += u' ({0})'.format(
                             sensedata['units'])
                     if showout:
-                        if sensedata['value'] is None:
+                        if sensedata.get('value', None) is None:
                             showval = ''
                         elif isinstance(sensedata['value'], float):
                             showval = u' {0} '.format(floatformat(sensedata['value']))
                         else:
-                            showval = u' {0} '.format(sensedata['value'])
-                        if sensedata['units'] not in (None, u''):
+                            showval = u' {0} '.format(sensedata.get('value', ''))
+                        if sensedata.get('units', None) not in (None, u''):
                             showval += sensedata['units']
                         if sensedata.get('health', 'ok') != 'ok':
                             datadescription = [sensedata['health']]

confluent_client/confluent/asynclient.py

@@ -0,0 +1,827 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2014 IBM Corporation
+# Copyright 2015-2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import ctypes
+import ctypes.util
+import dbm
+import csv
+import errno
+import fnmatch
+import hashlib
+import os
+import shlex
+import socket
+import ssl
+import sys
+import confluent.asynctlvdata as tlvdata
+import confluent.sortutil as sortutil
+libssl = ctypes.CDLL(ctypes.util.find_library('ssl'))
+libssl.SSL_CTX_set_cert_verify_callback.argtypes = [
+    ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
+
+SO_PASSCRED = 16
+
+_attraliases = {
+    'bmc': 'hardwaremanagement.manager',
+    'bmcuser': 'secret.hardwaremanagementuser',
+    'switchuser': 'secret.hardwaremanagementuser',
+    'bmcpass': 'secret.hardwaremanagementpassword',
+    'switchpass': 'secret.hardwaremanagementpassword',
+}
+
+try:
+    getinput = raw_input
+except NameError:
+    getinput = input
+
+
+class PyObject_HEAD(ctypes.Structure):
+    _fields_ = [
+        ("ob_refcnt",    ctypes.c_ssize_t),
+        ("ob_type",      ctypes.c_void_p),
+    ]
+
+
+# see main/Modules/_ssl.c, only caring about the SSL_CTX pointer
+class PySSLContext(ctypes.Structure):
+    _fields_ = [
+        ("ob_base",      PyObject_HEAD),
+        ("ctx",         ctypes.c_void_p),
+    ]
+
+
+@ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)
+def verify_stub(store, misc):
+    return 1
+
+
+class NestedDict(dict):
+    def __missing__(self, key):
+        value = self[key] = type(self)()
+        return value
+
+
+def stringify(instr):
+    # Normalize unicode and bytes to 'str', correcting for
+    # current python version
+    if isinstance(instr, bytes) and not isinstance(instr, str):
+        return instr.decode('utf-8', errors='replace')
+    elif not isinstance(instr, bytes) and not isinstance(instr, str):
+        return instr.encode('utf-8')
+    return instr
+
+
+class Tabulator(object):
+    def __init__(self, headers):
+        self.headers = headers
+        self.rows = []
+
+    def add_row(self, row):
+        self.rows.append(row)
+
+    def get_table(self, order=None):
+        i = 0
+        fmtstr = ''
+        separator = []
+        for head in self.headers:
+            if order and order == head:
+                order = i
+            neededlen = len(head)
+            for row in self.rows:
+                if len(row[i]) > neededlen:
+                    neededlen = len(row[i])
+            separator.append('-' * (neededlen + 1))
+            fmtstr += '{{{0}:>{1}}}|'.format(i, neededlen + 1)
+            i = i + 1
+        fmtstr = fmtstr[:-1]
+        yield fmtstr.format(*self.headers)
+        yield fmtstr.format(*separator)
+        if order is not None:
+            for row in sorted(
+                    self.rows,
+                    key=lambda x: sortutil.naturalize_string(x[order])):
+                yield fmtstr.format(*row)
+        else:
+            for row in self.rows:
+                yield fmtstr.format(*row)
+
+    def write_csv(self, output, order=None):
+        output = csv.writer(output)
+        output.writerow(self.headers)
+        i = 0
+        for head in self.headers:
+            if order and order == head:
+                order = i
+            i = i + 1
+        if order is not None:
+            for row in sorted(
+                    self.rows,
+                    key=lambda x: sortutil.naturalize_string(x[order])):
+                output.writerow(row)
+        else:
+            for row in self.rows:
+                output.writerow(row)
+
+
+def printerror(res, node=None):
+    exitcode = 0
+    if 'errorcode' in res:
+        exitcode = res['errorcode']
+    for node in res.get('databynode', {}):
+        exitcode = res['databynode'][node].get('errorcode', exitcode)
+        if 'error' in res['databynode'][node]:
+            sys.stderr.write(
+                '{0}: {1}\n'.format(node, res['databynode'][node]['error']))
+            if exitcode == 0:
+                exitcode = 1
+    if 'error' in res:
+        if node:
+            sys.stderr.write('{0}: {1}\n'.format(node, res['error']))
+        else:
+            sys.stderr.write('{0}\n'.format(res['error']))
+        if 'errorcode' not in res:
+            exitcode = 1
+    return exitcode
+
+
+def cprint(txt):
+    try:
+        print(txt)
+    except UnicodeEncodeError:
+        print(txt.encode('utf8'))
+    sys.stdout.flush()
+
+def _parseserver(string):
+    if ']:' in string:
+        server, port = string[1:].split(']:')
+    elif string[0] == '[':
+        server = string[1:-1]
+        port = '13001'
+    elif ':' in string:
+        server, port = string.split(':')
+    else:
+        server = string
+        port = '13001'
+    return server, port
+
+
+class Command(object):
+    def __init__(self, server=None):
+        self._prevdict = None
+        self._prevkeyname = None
+        self.connection = None
+        self._currnoderange = None
+        self.unixdomain = False
+        if server is None:
+            if 'CONFLUENT_HOST' in os.environ:
+                self.serverloc = os.environ['CONFLUENT_HOST']
+            else:
+                self.serverloc = '/var/run/confluent/api.sock'
+        else:
+            self.serverloc = server
+        self.connected = False
+
+    async def ensure_connected(self):
+        if self.connected:
+            return True
+        if os.path.isabs(self.serverloc) and os.path.exists(self.serverloc):
+            self._connect_unix()
+            self.unixdomain = True
+        elif self.serverloc == '/var/run/confluent/api.sock':
+            raise Exception('Confluent service is not available')
+        else:
+            await self._connect_tls()
+        self.protversion = int((await tlvdata.recv(self.connection)).split(
+            b'--')[1].strip()[1:])
+        authdata = await tlvdata.recv(self.connection)
+        if authdata['authpassed'] == 1:
+            self.authenticated = True
+        else:
+            self.authenticated = False
+        if not self.authenticated and 'CONFLUENT_USER' in os.environ:
+            username = os.environ['CONFLUENT_USER']
+            passphrase = os.environ['CONFLUENT_PASSPHRASE']
+            await self.authenticate(username, passphrase)
+        self.connected = True
+
+    async def add_file(self, name, handle, mode):
+        await self.ensure_connected()
+        if self.protversion < 3:
+            raise Exception('Not supported with connected confluent server')
+        if not self.unixdomain:
+            raise Exception('Can only add a file to a unix domain connection')
+        tlvdata.send(self.connection, {'filename': name, 'mode': mode}, handle)
+
+    async def authenticate(self, username, password):
+        await tlvdata.send(self.connection,
+                           {'username': username, 'password': password})
+        authdata = await tlvdata.recv(self.connection)
+        if authdata['authpassed'] == 1:
+            self.authenticated = True
+
+    def add_precede_key(self, keyname):
+        self._prevkeyname = keyname
+
+    def add_precede_dict(self, dict):
+        self._prevdict = dict
+
+    def handle_results(self, ikey, rc, res, errnodes=None, outhandler=None):
+        if 'error' in res:
+            if errnodes is not None:
+                errnodes.add(self._currnoderange)
+            sys.stderr.write('Error: {0}\n'.format(res['error']))
+            if 'errorcode' in res:
+                return res['errorcode']
+            else:
+                return 1
+        if 'databynode' not in res:
+            return 0
+        res = res['databynode']
+        for node in res:
+            if 'error' in res[node]:
+                if errnodes is not None:
+                    errnodes.add(node)
+                sys.stderr.write('{0}: Error: {1}\n'.format(
+                    node, res[node]['error']))
+                if 'errorcode' in res[node]:
+                    rc |= res[node]['errorcode']
+                else:
+                    rc |= 1
+            elif ikey in res[node]:
+                if 'value' in res[node][ikey]:
+                    val = res[node][ikey]['value']
+                elif 'isset' in res[node][ikey]:
+                    val = '********' if res[node][ikey] else ''
+                else:
+                    val = repr(res[node][ikey])
+                if self._prevkeyname and self._prevkeyname in res[node]:
+                    cprint('{0}: {2}->{1}'.format(
+                        node, val, res[node][self._prevkeyname]['value']))
+                elif self._prevdict and node in self._prevdict:
+                    cprint('{0}: {2}->{1}'.format(
+                        node, val, self._prevdict[node]))
+                else:
+                    cprint('{0}: {1}'.format(node, val))
+            elif outhandler:
+                outhandler(node, res)
+        return rc
+
+    async def simple_noderange_command(self, noderange, resource, input=None,
+                                 key=None, errnodes=None, promptover=None, outhandler=None, **kwargs):
+        try:
+            self._currnoderange = noderange
+            rc = 0
+            if resource[0] == '/':
+                resource = resource[1:]
+            # The implicit key is the resource basename
+            if key is None:
+                ikey = resource.rpartition('/')[-1]
+            else:
+                ikey = key
+            if input is None:
+                async for res in self.read('/noderange/{0}/{1}'.format(
+                        noderange, resource)):
+                    rc = self.handle_results(ikey, rc, res, errnodes, outhandler)
+            else:
+                await self.stop_if_noderange_over(noderange, promptover)
+                kwargs[ikey] = input
+                async for res in self.update('/noderange/{0}/{1}'.format(
+                        noderange, resource), kwargs):
+                    rc = self.handle_results(ikey, rc, res, errnodes, outhandler)
+            self._currnoderange = None
+            return rc
+        except KeyboardInterrupt:
+            cprint('')
+            return 0
+    
+    async def stop_if_noderange_over(self, noderange, maxnodes):
+        if maxnodes is None:
+            return
+        nsize = await self.get_noderange_size(noderange)
+        if nsize > maxnodes:
+            if nsize == 1:
+                nodename = [x async for x in self.read(
+                    '/noderange/{0}/nodes/'.format(noderange))][0].get('item', {}).get('href', None)
+                nodename = nodename[:-1]
+                p = getinput('Command is about to affect node {0}, continue (y/n)? '.format(nodename))
+            else:
+                p = getinput('Command is about to affect {0} nodes, continue (y/n)? '.format(nsize))
+            if p.lower() != 'y':
+                sys.stderr.write('Aborting at user request\n')
+                sys.exit(1)
+                raise Exception("Aborting at user request")
+        
+
+    async def get_noderange_size(self, noderange):
+        numnodes = 0
+        async for node in self.read('/noderange/{0}/nodes/'.format(noderange)):
+            if node.get('item', {}).get('href', None):
+                numnodes += 1
+            else:
+                raise Exception("Error trying to size noderange {0}".format(noderange))
+        return numnodes
+
+    async def simple_nodegroups_command(self, noderange, resource, input=None, key=None, **kwargs):
+        try:
+            rc = 0
+            if resource[0] == '/':
+                resource = resource[1:]
+            # The implicit key is the resource basename
+            if key is None:
+                ikey = resource.rpartition('/')[-1]
+            else:
+                ikey = key
+            if input is None:
+                for res in await self.read('/nodegroups/{0}/{1}'.format(
+                        noderange, resource)):
+                    rc = self.handle_results(ikey, rc, res)
+            else:
+                kwargs[ikey] = input
+                for res in await self.update('/nodegroups/{0}/{1}'.format(
+                        noderange, resource), kwargs):
+                    rc = self.handle_results(ikey, rc, res)
+            return rc
+        except KeyboardInterrupt:
+            cprint('')
+            return 0
+
+    async def read(self, path, parameters=None):
+        await self.ensure_connected()
+        if not self.authenticated:
+            raise Exception('Unauthenticated')
+        async for rsp in send_request(
+                'retrieve', path, self.connection, parameters):
+            yield rsp
+
+    async def update(self, path, parameters=None):
+        await self.ensure_connected()
+        if not self.authenticated:
+            raise Exception('Unauthenticated')
+        async for rsp in send_request(
+                'update', path, self.connection, parameters):
+            yield rsp
+
+    async def create(self, path, parameters=None):
+        await self.ensure_connected()
+        if not self.authenticated:
+            raise Exception('Unauthenticated')
+        async for rsp in send_request(
+                'create', path, self.connection, parameters):
+            yield rsp
+
+    async def delete(self, path, parameters=None):
+        await self.ensure_connected()
+        if not self.authenticated:
+            raise Exception('Unauthenticated')
+        async for rsp in send_request(
+                'delete', path, self.connection, parameters):
+            yield rsp
+
+    def _connect_unix(self):
+        self.connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self.connection.setsockopt(socket.SOL_SOCKET, SO_PASSCRED, 1)
+        self.connection.connect(self.serverloc)
+
+    async def _connect_tls(self):
+        server, port = _parseserver(self.serverloc)
+        for res in socket.getaddrinfo(server, port, socket.AF_UNSPEC,
+                                      socket.SOCK_STREAM):
+            af, socktype, proto, canonname, sa = res
+            try:
+                self.connection = socket.socket(af, socktype, proto)
+                self.connection.setsockopt(
+                    socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+            except:
+                self.connection = None
+                continue
+            try:
+                self.connection.settimeout(5)
+                self.connection.connect(sa)
+                self.connection.settimeout(0)
+            except:
+                raise
+                self.connection.close()
+                self.connection = None
+                continue
+            break
+        if self.connection is None:
+            raise Exception("Failed to connect to %s" % self.serverloc)
+        #TODO(jbjohnso): server certificate validation
+        clientcfgdir = os.path.join(os.path.expanduser("~"), ".confluent")
+        try:
+            os.makedirs(clientcfgdir)
+        except OSError as exc:
+            if not (exc.errno == errno.EEXIST and os.path.isdir(clientcfgdir)):
+                raise
+        cacert = os.path.join(clientcfgdir, "ca.pem")
+        certreqs = ssl.CERT_REQUIRED
+        knownhosts = False
+        if not os.path.exists(cacert):
+            cacert = None
+            certreqs = ssl.CERT_NONE
+            knownhosts = True
+        ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
+        ssl_ctx = PySSLContext.from_address(id(ctx)).ctx
+        libssl.SSL_CTX_set_cert_verify_callback(ssl_ctx, verify_stub, 0)
+        sreader = asyncio.StreamReader()
+        sreaderprot = asyncio.StreamReaderProtocol(sreader)
+        cloop = asyncio.get_event_loop()
+        tport, _ = await cloop.create_connection(
+            lambda: sreaderprot, sock=self.connection, ssl=ctx, server_hostname='x')
+        swriter = asyncio.StreamWriter(tport, sreaderprot, sreader, cloop)
+        self.connection = (sreader, swriter)
+        #self.connection = ssl.wrap_socket(self.connection, ca_certs=cacert,
+        #                                  cert_reqs=certreqs)
+        if knownhosts:
+            certdata = tport.get_extra_info('ssl_object').getpeercert(binary_form=True)
+            # certdata = self.connection.getpeercert(binary_form=True)
+            fingerprint = 'sha512$' + hashlib.sha512(certdata).hexdigest()
+            fingerprint = fingerprint.encode('utf-8')
+            hostid = '@'.join((port, server))
+            khf = dbm.open(os.path.join(clientcfgdir, "knownhosts"), 'c', 384)
+            if hostid in khf:
+                if fingerprint == khf[hostid]:
+                    return
+                else:
+                    replace = getinput(
+                        "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
+                    if replace not in ('y', 'Y'):
+                        raise Exception("BAD CERTIFICATE")
+            cprint('Adding new key for %s:%s' % (server, port))
+            khf[hostid] = fingerprint
+
+
+async def send_request(operation, path, server, parameters=None):
+    """This function iterates over all the responses
+    received from the server.
+
+    :param operation:  The operation to request, retrieve, update, delete,
+                       create, start, stop
+    :param path: The URI path to the resource to operate on
+    :param server: The socket to send data over
+    :param parameters:  Parameters if any to send along with the request
+    """
+    payload = {'operation': operation, 'path': path}
+    if parameters is not None:
+        payload['parameters'] = parameters
+    await tlvdata.send(server, payload)
+    result = await tlvdata.recv(server)
+    while '_requestdone' not in result:
+        try:
+            yield result
+        except GeneratorExit:
+            while '_requestdone' not in result:
+                result = await tlvdata.recv(server)
+            raise
+        result = await tlvdata.recv(server)
+
+
+def attrrequested(attr, attrlist, seenattributes, node=None):
+    for candidate in attrlist:
+        truename = candidate
+        if candidate.startswith('hm'):
+            candidate = candidate.replace('hm', 'hardwaremanagement', 1)
+        if candidate in _attraliases:
+            candidate = _attraliases[candidate]
+        if fnmatch.fnmatch(attr.lower(), candidate.lower()):
+            if node is None:
+                seenattributes.add(truename)
+            else:
+                seenattributes[node][truename] = True
+            return True
+        elif attr.lower().startswith(candidate.lower() + '.'):
+            if node is None:
+                seenattributes.add(truename)
+            else:
+                seenattributes[node][truename] = 1
+            return True
+    return False
+
+
+async def printattributes(session, requestargs, showtype, nodetype, noderange, options):
+    path = '/{0}/{1}/attributes/{2}'.format(nodetype, noderange, showtype)
+    return await print_attrib_path(path, session, requestargs, options)
+
+def _sort_attrib(k):
+    if isinstance(k[1], dict) and k[1].get('sortid', None) is not None:
+        return sortutil.naturalize_string('{}'.format(k[1]['sortid']))
+    return sortutil.naturalize_string(k[0])
+
+async def print_attrib_path(path, session, requestargs, options, rename=None, attrprefix=None):
+    exitcode = 0
+    seenattributes = NestedDict()
+    allnodes = set([])
+    async for res in session.read(path):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+            continue
+        for node in sorted(res['databynode']):
+            allnodes.add(node)
+            for attr, val in sorted(res['databynode'][node].items(), key=_sort_attrib):
+                if attr == 'error':
+                    sys.stderr.write('{0}: Error: {1}\n'.format(node, val))
+                    continue
+                if attr == 'errorcode':
+                    exitcode |= val
+                    continue
+                seenattributes[node][attr] = True
+                if rename:
+                    printattr = rename.get(attr, attr)
+                else:
+                    printattr = attr
+                if attrprefix:
+                    printattr = attrprefix + printattr
+                currattr = res['databynode'][node][attr]
+                if show_attr(attr, requestargs, seenattributes, options, node):
+                    if 'value' in currattr:
+                        if currattr['value'] is not None:
+                            val = currattr['value']
+                            if isinstance(val, list):
+                                val = ','.join(val)
+                            attrout = '{0}: {1}: {2}'.format(
+                                node, printattr, val).strip()
+                        else:
+                            attrout = '{0}: {1}:'.format(node, printattr)
+                    elif 'isset' in currattr:
+                        if currattr['isset']:
+                            attrout = '{0}: {1}: ********'.format(node,
+                                                                  printattr)
+                        else:
+                            attrout = '{0}: {1}:'.format(node, printattr)
+                    elif isinstance(currattr, dict) and 'broken' in currattr:
+                        attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \
+                                  '{2}'.format(node, printattr,
+                                               currattr['broken'])
+                    elif isinstance(currattr, list) or isinstance(currattr, tuple):
+                        attrout = '{0}: {1}: {2}'.format(node, attr, ','.join(map(str, currattr)))
+                    elif isinstance(currattr, dict):
+                        dictout = []
+                        for k, v in currattr.items:
+                            dictout.append("{0}={1}".format(k, v))
+                        attrout = '{0}: {1}: {2}'.format(node, printattr, ','.join(map(str, dictout)))
+                    else:
+                        cprint("CODE ERROR" + repr(attr))
+                    try:
+                        blame = options.blame
+                    except AttributeError:
+                        blame = False
+                    if blame or (isinstance(currattr, dict) and 'broken' in currattr):
+                        blamedata = []
+                        if 'inheritedfrom' in currattr:
+                            blamedata.append('inherited from group {0}'.format(
+                                currattr['inheritedfrom']
+                            ))
+                        if 'expression' in currattr:
+                            blamedata.append(
+                                'derived from expression "{0}"'.format(
+                                    currattr['expression']))
+                        if blamedata:
+                            attrout += ' (' + ', '.join(blamedata) + ')'
+                    try:
+                        comparedefault = options.comparedefault
+                    except AttributeError:
+                        comparedefault = False
+                    if comparedefault:
+                        try:
+                            exclude = options.exclude
+                        except AttributeError:
+                            exclude = False
+                        if ((requestargs and not exclude) or
+                                (currattr.get('default', None) is not None and
+                                currattr.get('value', None) is not None and
+                                currattr['value'] != currattr['default'])):
+                            cval = ','.join(currattr['value']) if isinstance(
+                                currattr['value'], list) else currattr['value']
+                            dval = ','.join(currattr['default']) if isinstance(
+                                currattr['default'], list) else currattr['default']
+                            cprint('{0}: {1}: {2} (Default: {3})'.format(
+                                node, printattr, cval, dval))
+                    else:
+
+                        try:
+                            details = options.detail
+                        except AttributeError:
+                            details = False
+                        if details:
+                            if currattr.get('help', None):
+                                attrout += u' (Help: {0})'.format(
+                                    currattr['help'])
+                            if currattr.get('possible', None):
+                                try:
+                                    attrout += u' (Choices: {0})'.format(
+                                        ','.join(currattr['possible']))
+                                except TypeError:
+                                    pass
+                        cprint(attrout)
+    somematched = set([])
+    printmissing = set([])
+    badnodes = NestedDict()
+    if not exitcode:
+        if requestargs:
+            for attr in requestargs:
+                for node in allnodes:
+                    if attr in seenattributes[node]:
+                        somematched.add(attr)
+                    else:
+                        badnodes[node][attr] = True
+                        exitcode = 1
+        for node in sortutil.natural_sort(badnodes):
+            for attr in badnodes[node]:
+                if attr in somematched:
+                    sys.stderr.write(
+                        'Error: {0} matches no valid value for {1}\n'.format(
+                            attr, node))
+                else:
+                    printmissing.add(attr)
+        for missing in printmissing:
+            sys.stderr.write('Error: {0} not a valid attribute\n'.format(missing))
+    return exitcode
+
+
+def show_attr(attr, requestargs, seenattributes, options, node):
+    try:
+        reverse = options.exclude
+    except AttributeError:
+        reverse = False
+    if requestargs is None or requestargs == []:
+        return True
+    processattr = attrrequested(attr, requestargs, seenattributes, node)
+    if reverse:
+        processattr = not processattr
+    return processattr
+
+
+def printgroupattributes(session, requestargs, showtype, nodetype, noderange, options):
+    exitcode = 0
+    seenattributes = set([])
+    for res in session.read('/{0}/{1}/attributes/{2}'.format(nodetype, noderange, showtype)):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+            continue
+        for attr in res:
+            seenattributes.add(attr)
+            currattr = res[attr]
+            if (requestargs is None or requestargs == [] or attrrequested(attr, requestargs, seenattributes)):
+                if 'value' in currattr:
+                    if currattr['value'] is not None:
+                        attrout = '{0}: {1}: {2}'.format(
+                            noderange, attr, currattr['value'])
+                    else:
+                        attrout = '{0}: {1}:'.format(noderange, attr)
+                elif 'isset' in currattr:
+                    if currattr['isset']:
+                        attrout = '{0}: {1}: ********'.format(noderange, attr)
+                    else:
+                        attrout = '{0}: {1}:'.format(noderange, attr)
+                elif isinstance(currattr, dict) and 'broken' in currattr:
+                    attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \
+                              '{2}'.format(noderange, attr,
+                                           currattr['broken'])
+                elif 'expression' in currattr:
+                    attrout = '{0}: {1}:  (will derive from expression {2})'.format(noderange, attr, currattr['expression'])
+                elif isinstance(currattr, list) or isinstance(currattr, tuple):
+                    attrout = '{0}: {1}: {2}'.format(noderange, attr, ','.join(map(str, currattr)))
+                elif isinstance(currattr, dict):
+                    dictout = []
+                    for k, v in currattr.items:
+                        dictout.append("{0}={1}".format(k, v))
+                    attrout = '{0}: {1}: {2}'.format(noderange, attr, ','.join(map(str, dictout)))
+                else:
+                    cprint("CODE ERROR" + repr(attr))
+                cprint(attrout)
+    if not exitcode:
+        if requestargs:
+            for attr in requestargs:
+                if attr not in seenattributes:
+                    sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr))
+                    exitcode = 1
+    return exitcode
+
+async def updateattrib(session, updateargs, nodetype, noderange, options, dictassign=None):
+    # update attribute
+    exitcode = 0
+    if options.clear:
+        targpath = '/{0}/{1}/attributes/all'.format(nodetype, noderange)
+        keydata = {}
+        for attrib in updateargs[1:]:
+            keydata[attrib] = None
+        async for res in session.update(targpath, keydata):
+            for node in res.get('databynode', {}):
+                for warnmsg in res['databynode'][node].get('_warnings', []):
+                    sys.stderr.write('Warning: ' + warnmsg + '\n')
+            if 'error' in res:
+                if 'errorcode' in res:
+                    exitcode = res['errorcode']
+                sys.stderr.write('Error: ' + res['error'] + '\n')
+        sys.exit(exitcode)
+    elif hasattr(options, 'environment') and options.environment:
+        for key in updateargs[1:]:
+            key = key.replace('.', '_')
+            value = os.environ.get(
+                key, os.environ[key.upper()])
+            # Let's do one pass to make sure that there's not a usage problem
+        for key in updateargs[1:]:
+            key = key.replace('.', '_')
+            value = os.environ.get(
+                key, os.environ[key.upper()])
+            if (nodetype == "nodegroups"):
+                exitcode = await session.simple_nodegroups_command(noderange,
+                                                             'attributes/all',
+                                                             value, key)
+            else:
+                exitcode = await session.simple_noderange_command(noderange,
+                                                            'attributes/all',
+                                                            value, key)
+        sys.exit(exitcode)
+    elif dictassign:
+        for key in dictassign:
+            if nodetype == 'nodegroups':
+                exitcode = await session.simple_nodegroups_command(
+                    noderange, 'attributes/all', dictassign[key], key)
+            else:
+                exitcode = await session.simple_noderange_command(
+                    noderange, 'attributes/all', dictassign[key], key)
+    else:
+        if "=" in updateargs[1]:
+            update_ready = True
+            for arg in updateargs[1:]:
+                if not '=' in arg:
+                    update_ready = False
+                    exitcode = 1
+            if not update_ready:
+                sys.stderr.write('Error: {0} Can not set and read at the same time!\n'.format(str(updateargs[1:])))
+                sys.exit(exitcode)
+            try:
+                for val in updateargs[1:]:
+                    val = val.split('=', 1)
+                    if val[0][-1] in (',', '-', '^'):
+                        key = val[0][:-1]
+                        if val[0][-1] == ',':
+                            value = {'prepend': val[1]}
+                        elif val[0][-1] in ('-', '^'):
+                            value = {'remove': val[1]}
+                    else:
+                        key = val[0]
+                        value = val[1]
+                    if (nodetype == "nodegroups"):
+                        exitcode =  await session.simple_nodegroups_command(noderange, 'attributes/all',
+                                                                     value, key)
+                    else:
+                        exitcode = await session.simple_noderange_command(noderange, 'attributes/all',
+                                                                    value, key)
+            except Exception:
+                sys.stderr.write('Error: {0} not a valid expression\n'.format(str(updateargs[1:])))
+                exitcode = 1
+            sys.exit(exitcode)
+    return exitcode
+
+
+# So we try to prevent bad things from happening when globbing
+# We tried to head this off at the shell, but the various solutions would end
+# up breaking the shell in various ways (breaking pipe capability if using
+# DEBUG, breaking globbing if in pipe, etc)
+# Then we tried to parse the original commandline instead, however shlex isn't
+# going to parse full bourne language (e.g. knowing that '|' and '>' and
+# a world of other things would not be in our command line
+# so finally, just make sure the noderange appears verbatim in the command line
+# if we glob to something, then bash will change noderange and this should
+# detect it and save the user from tragedy
+def check_globbing(noderange):
+    if not os.path.exists(noderange):
+        return True
+    rawargs = os.environ.get('CURRENT_CMDLINE', None)
+    if rawargs:
+        rawargs = shlex.split(rawargs)
+        for arg in rawargs:
+            if arg.startswith('$'):
+                arg = arg[1:]
+                if arg.endswith(';'):
+                    arg = arg[:-1]
+                arg = os.environ.get(arg, '$' + arg)
+            if arg.startswith(noderange):
+                break
+        else:
+            sys.stderr.write(
+                'Shell glob conflict detected, specified target "{0}" '
+                'not in command line, but is a file.  You can use "set -f" in '
+                'bash or change directories such that there is no filename '
+                'that would conflict.'
+                '\n'.format(noderange))
+            sys.exit(1)

confluent_client/confluent/asynctlvdata.py

@@ -0,0 +1,323 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2014 IBM Corporation
+# Copyright 2015 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import array
+import asyncio
+import ctypes
+import ctypes.util
+import confluent.tlv as tlv
+import socket
+from datetime import datetime
+import json
+import os
+import struct
+
+try:
+    unicode
+except NameError:
+    unicode = str
+
+try:
+    range = xrange
+except NameError:
+    pass
+
+
+class iovec(ctypes.Structure):   # from uio.h
+    _fields_ = [('iov_base', ctypes.c_void_p),
+                ('iov_len', ctypes.c_size_t)]
+
+
+iovec_ptr = ctypes.POINTER(iovec)
+
+
+class cmsghdr(ctypes.Structure):  # also from bits/socket.h
+    _fields_ = [('cmsg_len', ctypes.c_size_t),
+                ('cmsg_level', ctypes.c_int),
+                ('cmsg_type', ctypes.c_int)]
+
+    @classmethod
+    def init_data(cls, cmsg_len, cmsg_level, cmsg_type, cmsg_data):
+        Data = ctypes.c_ubyte * ctypes.sizeof(cmsg_data)
+
+        class _flexhdr(ctypes.Structure):
+            _fields_ = cls._fields_ + [('cmsg_data', Data)]
+
+        datab = Data(*bytearray(cmsg_data))
+        return _flexhdr(cmsg_len=cmsg_len, cmsg_level=cmsg_level,
+                        cmsg_type=cmsg_type, cmsg_data=datab)
+
+
+def CMSG_LEN(length):
+    sizeof_cmshdr = ctypes.sizeof(cmsghdr)
+    return ctypes.c_size_t(CMSG_ALIGN(sizeof_cmshdr).value + length)
+
+
+SCM_RIGHTS = 1
+
+
+class msghdr(ctypes.Structure):  # from bits/socket.h
+    _fields_ = [('msg_name', ctypes.c_void_p),
+                ('msg_namelen', ctypes.c_uint),
+                ('msg_iov', ctypes.POINTER(iovec)),
+                ('msg_iovlen', ctypes.c_size_t),
+                ('msg_control', ctypes.c_void_p),
+                ('msg_controllen', ctypes.c_size_t),
+                ('msg_flags', ctypes.c_int)]
+
+
+def CMSG_ALIGN(length):  # bits/socket.h
+    ret = (length + ctypes.sizeof(ctypes.c_size_t) - 1
+           & ~(ctypes.sizeof(ctypes.c_size_t) - 1))
+    return ctypes.c_size_t(ret)
+
+
+def CMSG_SPACE(length):  # bits/socket.h
+    ret = CMSG_ALIGN(length).value + CMSG_ALIGN(ctypes.sizeof(cmsghdr)).value
+    return ctypes.c_size_t(ret)
+
+
+class ClientFile(object):
+    def __init__(self, name, mode, fd):
+        self.fileobject = os.fdopen(fd, mode)
+        self.filename = name
+
+
+
+
+def _sendmsg(loop, fut, sock, msg, fds, rfd):
+    if rfd is not None:
+        loop.remove_reader(rfd)
+    if fut.cancelled():
+        return
+    try:
+        retdata = sock.sendmsg(
+            [msg],
+            [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))])
+    except (BlockingIOError, InterruptedError):
+        fd = sock.fileno()
+        loop.add_reader(fd, _sendmsg, loop, fut, sock, fd)
+    except Exception as exc:
+        fut.set_exception(exc)
+    else:
+        fut.set_result(retdata)
+
+
+def send_fds(sock, msg, fds):
+    cloop = asyncio.get_event_loop()
+    fut = cloop.create_future()
+    _sendmsg(cloop, fut, sock, msg, fds, None)
+    return fut
+
+
+def _recvmsg(loop, fut, sock, msglen, maxfds, rfd):
+    if rfd is not None:
+        loop.remove_reader(rfd)
+    fds = array.array("i")   # Array of ints
+    try:
+        msg, ancdata, flags, addr = sock.recvmsg(
+            msglen, socket.CMSG_LEN(maxfds * fds.itemsize))
+    except (BlockingIOError, InterruptedError):
+        fd = sock.fileno()
+        loop.add_reader(fd, _recvmsg, loop, fut, sock, msglen, maxfds, fd)
+    except Exception as exc:
+        fut.set_exception(exc)
+    else:
+        for cmsg_level, cmsg_type, cmsg_data in ancdata:
+            if (cmsg_level == socket.SOL_SOCKET
+                    and cmsg_type == socket.SCM_RIGHTS):
+                # Append data, ignoring any truncated integers at the end.
+                fds.frombytes(
+                    cmsg_data[
+                        :len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
+        fut.set_result((msg, list(fds)))
+
+
+def recv_fds(sock, msglen, maxfds):
+    cloop = asyncio.get_event_loop()
+    fut = cloop.create_future()
+    _recvmsg(cloop, fut, sock, msglen, maxfds, None)
+    return fut
+
+
+def decodestr(value):
+    ret = None
+    try:
+        ret = value.decode('utf-8')
+    except UnicodeDecodeError:
+        try:
+            ret = value.decode('cp437')
+        except UnicodeDecodeError:
+            ret = value
+    except AttributeError:
+        return value
+    return ret
+
+
+def unicode_dictvalues(dictdata):
+    for key in dictdata:
+        if isinstance(dictdata[key], bytes):
+            dictdata[key] = decodestr(dictdata[key])
+        elif isinstance(dictdata[key], datetime):
+            dictdata[key] = dictdata[key].strftime('%Y-%m-%dT%H:%M:%S')
+        elif isinstance(dictdata[key], list):
+            _unicode_list(dictdata[key])
+        elif isinstance(dictdata[key], dict):
+            unicode_dictvalues(dictdata[key])
+
+
+def _unicode_list(currlist):
+    for i in range(len(currlist)):
+        if isinstance(currlist[i], str):
+            currlist[i] = decodestr(currlist[i])
+        elif isinstance(currlist[i], dict):
+            unicode_dictvalues(currlist[i])
+        elif isinstance(currlist[i], list):
+            _unicode_list(currlist[i])
+
+
+async def sendall(handle, data):
+    if isinstance(handle, tuple):
+        handle[1].write(data)
+        return await handle[1].drain()
+    else:
+        cloop = asyncio.get_event_loop()
+        return await cloop.sock_sendall(handle, data)
+
+async def close(handle):
+    if isinstance(handle, tuple):
+        handle[1].close()
+        await handle[1].wait_closed()
+    else:
+        handle.close()
+
+async def send(handle, data, filehandle=None):
+    cloop = asyncio.get_event_loop()
+    if isinstance(data, unicode):
+        try:
+            data = data.encode('utf-8')
+        except AttributeError:
+            pass
+    if isinstance(data, bytes) or isinstance(data, unicode):
+        # plain text, e.g. console data
+        tl = len(data)
+        if tl == 0:
+            # if you don't have anything to say, don't say anything at all
+            return
+        if tl < 16777216:
+            # type for string is '0', so we don't need
+            # to xor anything in
+            await sendall(handle, struct.pack("!I", tl))
+        else:
+            raise Exception("String data length exceeds protocol")
+        await sendall(handle, data)
+    elif isinstance(data, dict):  # JSON currently only goes to 4 bytes
+        # Some structured message, like what would be seen in http responses
+        unicode_dictvalues(data)  # make everything unicode, assuming UTF-8
+        sdata = json.dumps(data, ensure_ascii=False, separators=(',', ':'))
+        sdata = sdata.encode('utf-8')
+        tl = len(sdata)
+        if tl > 16777215:
+            raise Exception("JSON data exceeds protocol limits")
+        # xor in the type (0b1 << 24)
+        if filehandle is None:
+            tl |= 16777216
+            await sendall(handle, struct.pack("!I", tl))
+            await sendall(handle, sdata)
+        elif isinstance(handle, tuple):
+            raise Exception("Cannot send filehandle over network socket")
+        else:
+            tl |= (2 << 24)
+            await cloop.sock_sendall(handle, struct.pack("!I", tl))
+            await send_fds(handle, b'', [filehandle])
+
+
+async def _grabhdl(handle, size):
+    if isinstance(handle, tuple):
+        return await handle[0].read(size)
+    else:
+        cloop = asyncio.get_event_loop()
+        return await cloop.sock_recv(handle, size)
+
+
+async def recvall(handle, size):
+    rd = await _grabhdl(handle, size)
+    while len(rd) < size:
+        nd = await _grabhdl(handle, size - len(rd))
+        if not nd:
+            raise Exception("Error reading data")
+        rd += nd
+    return rd
+
+
+async def recv(handle):
+    tl = await _grabhdl(handle, 4)
+    if not tl:
+        return None
+    while len(tl) < 4:
+        ndata = await _grabhdl(handle, 4 - len(tl))
+        if not ndata:
+            raise Exception("Error reading data")
+        tl += ndata
+    if len(tl) == 0:
+        return None
+    tl = struct.unpack("!I", tl)[0]
+    if tl & 0b10000000000000000000000000000000:
+        raise Exception("Protocol Violation, reserved bit set")
+    # 4 byte tlv
+    dlen = tl & 16777215  # grab lower 24 bits
+    datatype = (tl & 2130706432) >> 24  # grab 7 bits from near beginning
+    if dlen == 0:
+        return None
+    if datatype == tlv.Types.filehandle:
+        if isinstance(handle, tuple):
+            raise Exception('Filehandle not supported over TLS socket')
+        filehandles = array.array('i')
+        rawbuffer = bytearray(2048)
+        pkttype = ctypes.c_ubyte * 2048
+        data = pkttype.from_buffer(rawbuffer)
+        cmsgsize = CMSG_SPACE(ctypes.sizeof(ctypes.c_int)).value
+        cmsgarr = bytearray(cmsgsize)
+        cmtype = ctypes.c_ubyte * cmsgsize
+        cmsg = cmtype.from_buffer(cmsgarr)
+        cmsg.cmsg_level = socket.SOL_SOCKET
+        cmsg.cmsg_type = SCM_RIGHTS
+        cmsg.cmsg_len = CMSG_LEN(ctypes.sizeof(ctypes.c_int))
+        iov = iovec()
+        iov.iov_base = ctypes.addressof(data)
+        iov.iov_len = 2048
+        msg = msghdr()
+        msg.msg_iov = ctypes.pointer(iov)
+        msg.msg_iovlen = 1
+        msg.msg_control = ctypes.addressof(cmsg)
+        msg.msg_controllen = ctypes.sizeof(cmsg)
+        i = await recv_fds(handle, 2048, 4)
+        data = i[0]
+        filehandles = i[1]
+        data = json.loads(bytes(data))
+        return ClientFile(data['filename'], data['mode'], filehandles[0])
+    else:
+        data = await _grabhdl(handle, dlen)
+        while len(data) < dlen:
+            ndata = await _grabhdl(handle, dlen - len(data))
+            if not ndata:
+                raise Exception("Error reading data")
+            data += ndata
+    if datatype == tlv.Types.text:
+        return data
+    elif datatype == tlv.Types.json:
+        return json.loads(data)

confluent_client/confluent/screensqueeze.py

@@ -16,11 +16,55 @@ import fcntl
 import sys
 import struct
 import termios
+import select
 
-def get_screengeom():
+def get_screengeom(escfallback=False):
     # returns height in cells, width in cells, width in pixels, height in pixels
-    return struct.unpack('hhhh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ,
-                                           b'........'))
+    geom = list(struct.unpack('hhhh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ,
+                                           b'........')))
+    if escfallback and (geom[0] == 0 or geom[1] == 0):
+        cellgeom = get_cell_geometry_using_esc_18t()
+        geom[0], geom[1] = cellgeom
+    if escfallback and (geom[2] == 0 or geom[3] == 0):
+        pixgeom = get_pixel_geometry_using_esc_14t()
+        geom[2], geom[3] = pixgeom
+    return geom
+
+def get_pixel_geometry_using_esc_14t():
+    sys.stdout.write('\x1b[14t')
+    sys.stdout.flush()
+    rlist, _, _ = select.select([sys.stdin], [], [], 1)
+    if not rlist:
+        return 0, 0
+    response = ''
+    while True:
+        c = sys.stdin.read(1)
+        if c == 't':
+            break
+        response += c
+    if not response.startswith('\x1b[4;'):
+        return 0, 0
+    pixgeom = response[4:].split(';')
+    return int(pixgeom[1]), int(pixgeom[0])
+
+def get_cell_geometry_using_esc_18t():
+    sys.stdout.write('\x1b[18t')
+    sys.stdout.flush()
+    rlist, _, _ = select.select([sys.stdin], [], [], 1)
+    if not rlist:
+        return 0, 0
+    response = ''
+    while True:
+        c = sys.stdin.read(1)
+        if c == 't':
+            break
+        response += c
+    if not response.startswith('\x1b[8;'):
+        return 0, 0
+    cellgeom = response[4:].split(';')
+    return int(cellgeom[0]), int(cellgeom[1])
+        
+        
 class ScreenPrinter(object):
 
     def __init__(self, noderange, client, textlen=4):

confluent_client/confluent/tlvdata.py

@@ -19,12 +19,8 @@ import array
 import ctypes
 import ctypes.util
 import confluent.tlv as tlv
-try:
-    import eventlet.green.socket as socket
-    import eventlet.green.select as select
-except ImportError:
-    import socket
-    import select
+import socket
+import select
 from datetime import datetime
 import json
 import os

confluent_client/doc/man/imgutil.ronn

@@ -0,0 +1,139 @@
+# imgutil(1) -- Work with confluent OS cloning and diskless images
+
+## SYNOPSIS
+
+`imgutil` `build` [<options>] <scratchdir>  
+`imgutil` `exec` [<options>] <scratchdir> [<cmd>...]  
+`imgutil` `unpack` <profilename> <scratchdir>  
+`imgutil` `pack` [<options>] <scratchdir> <profilename>  
+`imgutil` `capture` <node> <profilename>
+
+## DESCRIPTION
+
+**imgutil** is a utility for creating, managing, and deploying OS images for diskless boot and system cloning in a Confluent environment. It supports building images from scratch, capturing images from running systems, and packing/unpacking diskless profiles.
+
+## COMMANDS
+
+* `build`:
+  Build a new diskless image from scratch in the specified scratch directory.
+
+* `exec`:
+  Start the specified scratch directory as a container and optionally run a command inside it.
+
+* `unpack`:
+  Unpack a diskless image profile to a scratch directory for modification.
+
+* `pack`:
+  Pack a scratch directory into a diskless profile that can be deployed.
+
+* `capture`:
+  Capture an image for cloning from a running system.
+
+## BUILD OPTIONS
+
+* `-r`, `--addrepos` <repository>:
+  Repositories to add in addition to the main source. May be specified multiple times.
+
+* `-p`, `--packagelist` <file>:
+  Filename of package list to replace default pkglist.
+
+* `-a`, `--addpackagelist` <file>:
+  A list of additional packages to include. May be specified multiple times.
+
+* `-s`, `--source` <directory>:
+  Directory to pull installation from, typically a subdirectory of `/var/lib/confluent/distributions`. By default, the repositories for the build system are used. For Ubuntu, this is not supported; the build system repositories are always used.
+
+* `-y`, `--non-interactive`:
+  Avoid prompting for confirmation.
+
+* `-v`, `--volume` <mount>:
+  Directory to make available in the build environment. `-v /` will cause it to be mounted in image as `/run/external/`. `-v /:/run/root` will override the target to be `/run/root`. Something like `/var/lib/repository:-` will cause it to mount to the identical path inside the image. May be specified multiple times.
+
+* <scratchdir>:
+  Directory to build new diskless root in.
+
+## EXEC OPTIONS
+
+* `-v`, `--volume` <mount>:
+  Directory to make available in the build environment. `-v /` will cause it to be mounted in image as `/run/external/`. `-v /:/run/root` will override the target to be `/run/root`. May be specified multiple times.
+
+* <scratchdir>:
+  Directory of an unpacked diskless root.
+
+* <cmd>:
+  Optional command to run (defaults to a shell).
+
+## UNPACK OPTIONS
+
+* <profilename>:
+  The diskless OS profile to unpack.
+
+* <scratchdir>:
+  Directory to extract diskless root to.
+
+## PACK OPTIONS
+
+* `-b`, `--baseprofile` <profile>:
+  Profile to copy extra info from. For example, to make a new version of an existing profile, reference the previous one as baseprofile.
+
+* `-u`, `--unencrypted`:
+  Pack an unencrypted image rather than encrypting.
+
+* <scratchdir>:
+  Directory containing diskless root.
+
+* <profilename>:
+  The desired diskless OS profile name to pack the root into.
+
+## CAPTURE OPTIONS
+
+* <node>:
+  Node to capture image from.
+
+* <profilename>:
+  Profile name for captured image.
+
+## EXAMPLES
+
+Build a diskless image from a distribution:
+
+    imgutil build -s  alma-9.6-x86_64 /tmp/myimage
+
+Execute a shell in an unpacked image:
+
+    imgutil exec /tmp/myimage
+
+Execute a specific command in an image:
+
+    imgutil exec /tmp/myimage /bin/rpm -qa
+
+Unpack an existing profile for modification:
+
+    imgutil unpack myprofile /tmp/myimage
+
+Pack a modified image into a new profile:
+
+    imgutil pack /tmp/myimage myprofile-v2
+
+Capture an image from a running node:
+
+    imgutil capture node01 production-image
+
+## FILES
+
+* `/var/lib/confluent/public/os/`:
+  Default location for OS profiles.
+
+* `/var/lib/confluent/private/os/`:
+  Location for encrypted image keys and private data.
+
+* `/var/lib/confluent/distributions/`:
+  Default location for installation sources.
+
+## SEE ALSO
+
+osdeploy(8)
+
+## AUTHOR
+
+Written for the Confluent project.

confluent_client/doc/man/nodeattrib.ronn.tmpl

@@ -24,6 +24,8 @@ For a full list of attributes, run `nodeattrib <node> all` against a node.
 If `-c` is specified, this will set the nodeattribute to a null value.
 This is different from setting the value to an empty string.
 
+Arbitrary custom attributes can also be created with the `custom.` prefix.
+
 Attributes may be specified by wildcard, for example `net.*switch` will report
 all attributes that begin with `net.` and end with `switch`.
 

confluent_client/doc/man/nodeconfig.ronn

@@ -49,8 +49,7 @@ actually be in effect until a reboot.
 
 * `-r COMPONENT`, `--restoredefault=COMPONENT`:
   Request that the specified component of the targeted nodes will have its
-  configuration reset to default.  Currently the only component implemented
-  is uefi.
+  configuration reset to default.  The component may be "uefi" or "bmc".
 
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:
   Specify a maximum number of nodes to configure, prompting if over 

confluent_client/doc/man/nodegroupattrib.ronn.tmpl

@@ -11,7 +11,7 @@ nodegroupattrib(8) -- List or change confluent nodegroup attributes
 
 ## DESCRIPTION
 
-`nodegroupattrip` queries the confluent server to get information about nodes.
+`nodegroupattrib` queries the confluent server to get information about nodes.
 In the simplest form, it simply takes the given group and lists the attributes of that group.
 
 Contrasted with nodeattrib(8), settings managed by nodegroupattrib will be added

confluent_client/doc/man/nodemedia.ronn

@@ -32,6 +32,13 @@ BMCs map a virtual USB device to that url.  Content is loaded on demand, and
 as such that URL is referenced potentially once for every IO operation that
 the host platform attempts.
 
+## NOTES
+
+When doing an attach of an https:// url, you may hit an error if you have not enrolled your certificate authority.
+In a general confluent environment, you can usually address it by:
+  `# for cert in /var/lib/confluent/public/site/tls/*.pem; do nodecertutil s1-s4 installbmccacert $cert; done`  
+  
+
 ## OPTIONS
 
 * `-h`, `--help`:

confluent_osdeploy/bluefield/bfb-autoinstall

@@ -0,0 +1,74 @@
+#!/usr/bin/python3
+import glob
+import gzip
+import base64
+import os
+import subprocess
+import sys
+import tempfile
+
+def collect_certificates(tmpdir):
+    certdata = ''
+    for cacert in glob.glob(f'{tmpdir}/*.pem'):
+        with open(cacert, 'r') as f:
+            certdata += f.read()
+    return certdata
+
+def embed_certificates(incfg, certdata):
+    if not certdata:
+        raise Exception('No certificates found to embed')
+    incfg = incfg.replace('%CONFLUENTCERTCOLL%', certdata)
+    return incfg
+
+def embed_identity(incfg, identityjson):
+    incfg = incfg.replace('%IDENTJSON%', identityjson)
+    return incfg
+
+def embed_apiclient(incfg, apiclient):
+    with open(apiclient, 'r') as f:
+        apiclientdata = f.read()
+    compressed = gzip.compress(apiclientdata.encode())
+    encoded = base64.b64encode(compressed).decode()
+    incfg = incfg.replace('%APICLIENTZ64%', encoded)
+    return incfg
+
+def embed_data(tmpdir, outfile):
+    templatefile = f'{tmpdir}/bfb.cfg.template'
+    with open(templatefile, 'r') as f:
+        incfg = f.read()
+    
+    certdata = collect_certificates(tmpdir)
+    incfg = embed_certificates(incfg, certdata)
+    
+    with open(f'{tmpdir}/identity.json', 'r') as f:
+        identityjson = f.read()
+    
+    incfg = embed_identity(incfg, identityjson)
+    
+    incfg = embed_apiclient(incfg, f'{tmpdir}/../apiclient')
+    
+    with open(outfile, 'w') as f:
+        f.write(incfg)
+
+def get_identity_json(node):
+    identity_file = f'/var/lib/confluent/private/site/identity_files/{node}.json'
+    try:
+        with open(identity_file, 'r') as f:
+            return f.read()
+    except FileNotFoundError:
+        return None
+
+if __name__ == '__main__':
+    if len(sys.argv) != 4:
+        print("Usage: bfb-autoinstall <node> <bfbfile> <rshim>")
+        sys.exit(1)
+    
+    node = sys.argv[1]
+    bfbfile = sys.argv[2]
+    rshim = sys.argv[3]
+
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+    currdir = os.getcwd()
+    tempdir = tempfile.mkdtemp(prefix=f'bfb-autoinstall-{node}-')
+    embed_data(f'{currdir}/{node}', f'{tempdir}/bfb.cfg')
+    subprocess.check_call(['bfb-install', '-b', bfbfile, '-c', f'{tempdir}/bfb.cfg', '-r', rshim])

confluent_osdeploy/bluefield/hostscripts/bfb-autoinstall

@@ -0,0 +1,74 @@
+#!/usr/bin/python3
+import glob
+import gzip
+import base64
+import os
+import subprocess
+import sys
+import tempfile
+
+def collect_certificates(tmpdir):
+    certdata = ''
+    for cacert in glob.glob(f'{tmpdir}/*.pem'):
+        with open(cacert, 'r') as f:
+            certdata += f.read()
+    return certdata
+
+def embed_certificates(incfg, certdata):
+    if not certdata:
+        raise Exception('No certificates found to embed')
+    incfg = incfg.replace('%CONFLUENTCERTCOLL%', certdata)
+    return incfg
+
+def embed_identity(incfg, identityjson):
+    incfg = incfg.replace('%IDENTJSON%', identityjson)
+    return incfg
+
+def embed_apiclient(incfg, apiclient):
+    with open(apiclient, 'r') as f:
+        apiclientdata = f.read()
+    compressed = gzip.compress(apiclientdata.encode())
+    encoded = base64.b64encode(compressed).decode()
+    incfg = incfg.replace('%APICLIENTZ64%', encoded)
+    return incfg
+
+def embed_data(tmpdir, outfile):
+    templatefile = f'{tmpdir}/bfb.cfg.template'
+    with open(templatefile, 'r') as f:
+        incfg = f.read()
+    
+    certdata = collect_certificates(tmpdir)
+    incfg = embed_certificates(incfg, certdata)
+    
+    with open(f'{tmpdir}/identity.json', 'r') as f:
+        identityjson = f.read()
+    
+    incfg = embed_identity(incfg, identityjson)
+    
+    incfg = embed_apiclient(incfg, f'{tmpdir}/../apiclient')
+    
+    with open(outfile, 'w') as f:
+        f.write(incfg)
+
+def get_identity_json(node):
+    identity_file = f'/var/lib/confluent/private/site/identity_files/{node}.json'
+    try:
+        with open(identity_file, 'r') as f:
+            return f.read()
+    except FileNotFoundError:
+        return None
+
+if __name__ == '__main__':
+    if len(sys.argv) != 4:
+        print("Usage: bfb-autoinstall <node> <bfbfile> <rshim>")
+        sys.exit(1)
+    
+    node = sys.argv[1]
+    bfbfile = sys.argv[2]
+    rshim = sys.argv[3]
+
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+    currdir = os.getcwd()
+    tempdir = tempfile.mkdtemp(prefix=f'bfb-autoinstall-{node}-')
+    embed_data(f'{currdir}/{node}', f'{tempdir}/bfb.cfg')
+    subprocess.check_call(['bfb-install', '-b', bfbfile, '-c', f'{tempdir}/bfb.cfg', '-r', rshim])

confluent_osdeploy/bluefield/profiles/default/bluefield.cfg.template

@@ -0,0 +1,76 @@
+function bfb_modify_os() {
+     echo 'ubuntu:!' | chpasswd -e
+     mkdir -p /mnt/opt/confluent/bin/
+     cat > /mnt/opt/confluent/bin/confluentbootstrap.sh << 'END_OF_EMBED'
+#!/bin/bash
+     cat > /usr/local/share/ca-certificates/confluent.crt << 'END_OF_CERTS'
+%CONFLUENTCERTCOLL%
+END_OF_CERTS
+     update-ca-certificates
+     mkdir -p /opt/confluent/bin /etc/confluent/
+     cp /usr/local/share/ca-certificates/confluent.crt /etc/confluent/ca.pem
+     cat > /opt/confluent/bin/apiclient.gz.b64 << 'END_OF_CLIENT'
+%APICLIENTZ64%
+END_OF_CLIENT
+     base64 -d /opt/confluent/bin/apiclient.gz.b64 | gunzip > /opt/confluent/bin/apiclient
+     cat > /etc/confluent/ident.json << 'END_OF_IDENT'
+%IDENTJSON%
+END_OF_IDENT
+     python3 /opt/confluent/bin/apiclient -i /etc/confluent/ident.json /confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
+     PROFILE=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+     ROOTPASS=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg | awk '{print $2}'|grep -v null)
+     if [ -n "$ROOTPASS" ]; then
+          echo root:$ROOTPASS | chpasswd -e
+          echo "ubuntu:$ROOTPASS" | chpasswd -e
+     else
+          echo 'ubuntu:!' | chpasswd -e
+     fi
+     cntmp=$(mktemp -d)
+     cd "$cntmp" || { echo "Failed to cd to temporary directory $cntmp"; exit 1; }
+     touch /etc/confluent/confluent.deploycfg
+     python3 /opt/confluent/bin/apiclient /confluent-public/os/$PROFILE/scripts/confignet > confignet
+     python3 confignet
+     cd -
+     rm -rf "$cntmp"
+     python3 /opt/confluent/bin/apiclient /confluent-public/os/$PROFILE/scripts/functions > /etc/confluent/functions
+     bash /etc/confluent/functions run_remote setupssh
+     for cert in /etc/ssh/ssh*-cert.pub; do
+         if [ -s $cert ]; then
+             echo HostCertificate $cert >> /etc/ssh/sshd_config.d/90-confluent.conf
+         fi
+     done
+     mkdir -p /var/log/confluent
+     chmod 700 /var/log/confluent
+     touch /var/log/confluent/confluent-firstboot.log
+     touch /var/log/confluent/confluent-post.log
+     chmod 600 /var/log/confluent/confluent-post.log
+     chmod 600 /var/log/confluent/confluent-firstboot.log
+     exec >> /var/log/confluent/confluent-post.log
+     exec 2>> /var/log/confluent/confluent-post.log
+     bash /etc/confluent/functions run_remote_python syncfileclient
+     bash /etc/confluent/functions run_remote_parts post.d
+     bash /etc/confluent/functions run_remote_config post.d
+     exec >> /var/log/confluent/confluent-firstboot.log
+     exec 2>> /var/log/confluent/confluent-firstboot.log
+     bash /etc/confluent/functions run_remote_parts firstboot.d
+     bash /etc/confluent/functions run_remote_config firstboot.d
+     python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus  -d 'status: staged'
+     python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus  -d 'status: complete'
+     systemctl disable confluentbootstrap
+     rm /etc/systemd/system/confluentbootstrap.service
+END_OF_EMBED
+    chmod +x /mnt/opt/confluent/bin/confluentbootstrap.sh
+    cat > /mnt/etc/systemd/system/confluentbootstrap.service << EOS
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/confluentbootstrap.sh
+
+[Install]
+WantedBy=multi-user.target
+EOS
+    chroot /mnt systemctl enable confluentbootstrap
+}

confluent_osdeploy/bluefield/profiles/default/nodedeploy-bfb

@@ -0,0 +1,125 @@
+#!/usr/bin/python3
+
+import os
+import sys
+import tempfile
+import glob
+import shutil
+import shlex
+import subprocess
+import select
+
+sys.path.append('/opt/lib/confluent/python')
+
+import confluent.sortutil as sortutil
+import confluent.client as client
+
+
+def prep_outdir(node):
+    tmpdir = tempfile.mkdtemp()
+    for certfile in glob.glob('/var/lib/confluent/public/site/tls/*.pem'):
+        basename = os.path.basename(certfile)
+        destfile = os.path.join(tmpdir, basename)
+        shutil.copy2(certfile, destfile)
+    subprocess.check_call(shlex.split(f'confetty set /nodes/{node}/deployment/ident_image=create'))
+    shutil.copy2(f'/var/lib/confluent/private/identity_files/{node}.json', os.path.join(tmpdir, 'identity.json'))
+    return tmpdir
+
+def exec_bfb_install(host, nodetorshim, bfbfile, installprocs, pipedesc, all, poller):
+    remotedir = subprocess.check_output(shlex.split(f'ssh {host} mktemp -d /tmp/bfb.XXXXXX')).decode().strip()
+    bfbbasename = os.path.basename(bfbfile)
+    subprocess.check_call(shlex.split(f'rsync -avz --info=progress2 {bfbfile} {host}:{remotedir}/{bfbbasename}'))
+    subprocess.check_call(shlex.split(f'rsync -avc --info=progress2 /opt/lib/confluent/osdeploy/bluefield/hostscripts/ {host}:{remotedir}/'))
+    for node in nodetorshim:
+        rshim = nodetorshim[node]
+        nodeoutdir = prep_outdir(node)
+        nodeprofile = subprocess.check_output(shlex.split(f'nodeattrib {node} deployment.pendingprofile')).decode().strip().split(':', 2)[2].strip()
+        shutil.copy2(f'/var/lib/confluent/public/os/{nodeprofile}/bfb.cfg.template', os.path.join(nodeoutdir, 'bfb.cfg.template'))
+        subprocess.check_call(shlex.split(f'rsync -avz {nodeoutdir}/ {host}:{remotedir}/{node}/'))
+        shutil.rmtree(nodeoutdir)
+        run_cmdv(node, shlex.split(f'ssh {host} sh /etc/confluent/functions confluentpython {remotedir}/bfb-autoinstall {node} {remotedir}/{bfbbasename} {rshim}'), all, poller, pipedesc)
+
+
+def run_cmdv(node, cmdv, all, poller, pipedesc):
+    try:
+        nopen = subprocess.Popen(
+            cmdv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    except OSError as e:
+        if e.errno == 2:
+            sys.stderr.write('{0}: Unable to find local executable file "{1}"\n'.format(node, cmdv[0]))
+            return
+        raise
+    pipedesc[nopen.stdout.fileno()] = {'node': node, 'popen': nopen,
+                              'type': 'stdout', 'file': nopen.stdout}
+    pipedesc[nopen.stderr.fileno()] = {'node': node, 'popen': nopen,
+                              'type': 'stderr', 'file': nopen.stderr}
+    all.add(nopen.stdout)
+    poller.register(nopen.stdout, select.EPOLLIN)
+    all.add(nopen.stderr)
+    poller.register(nopen.stderr, select.EPOLLIN)
+
+if __name__ == '__main__':
+
+
+    if len(sys.argv) < 3:
+        print(f'Usage: {sys.argv[0]} <host> <bfbfile> <node1:rshim1> [<node2:rshim2> ...]')
+        sys.exit(1)
+
+    host = sys.argv[1]
+    bfbfile = sys.argv[2]
+    nodetorshim = {}
+    for arg in sys.argv[3:]:
+        node, rshim = arg.split(':')
+        nodetorshim[node] = rshim
+
+    installprocs = {}
+    pipedesc = {}
+    all = set()
+    poller = select.epoll()
+
+    exec_bfb_install(host, nodetorshim, bfbfile, installprocs, pipedesc, all, poller)
+    rdy = poller.poll(10)
+    pendingexecs = []
+    exitcode = 0
+    while all:
+        pernodeout = {}
+        for r in rdy:
+            r = r[0]
+            desc = pipedesc[r]
+            r = desc['file']
+            node = desc['node']
+            data = True
+            singlepoller = select.epoll()
+            singlepoller.register(r, select.EPOLLIN)
+            while data and singlepoller.poll(0):
+                data = r.readline()
+                if data:
+                    if desc['type'] == 'stdout':
+                        if node not in pernodeout:
+                            pernodeout[node] = []
+                        pernodeout[node].append(data)
+                    else:
+                        data = client.stringify(data)
+                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        sys.stderr.flush()
+                else:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        poller.unregister(r)
+                        r.close()
+                        if desc['type'] == 'stdout' and pendingexecs:
+                            node, cmdv = pendingexecs.popleft()
+                            run_cmdv(node, cmdv, all, poller, pipedesc)
+            singlepoller.close()
+        for node in sortutil.natural_sort(pernodeout):
+            for line in pernodeout[node]:
+                line = client.stringify(line)
+                sys.stdout.write('{0}: {1}'.format(node, line))
+        sys.stdout.flush()
+        if all:
+            rdy = poller.poll(10)
+
+

confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient

@@ -3,6 +3,7 @@ try:
     import http.client as client
 except ImportError:
     import httplib as client
+import base64
 import ctypes
 import ctypes.util
 import glob
@@ -15,6 +16,13 @@ import sys
 import struct
 import time
 import re
+import hashlib
+try:
+    import json
+    import hmac
+except ImportError:
+    json = None
+    hmac = None
 
 class InvalidApiKey(Exception):
     pass
@@ -39,40 +47,115 @@ c_crypt.restype = ctypes.c_char_p
 
 
 def get_my_addresses():
-    nlhdrsz = struct.calcsize('IHHII')
-    ifaddrsz = struct.calcsize('BBBBI')
-    # RTM_GETADDR = 22
-    # nlmsghdr struct: u32 len, u16 type, u16 flags, u32 seq, u32 pid
-    nlhdr = struct.pack('IHHII', nlhdrsz + ifaddrsz, 22, 0x301, 0, 0)
-    # ifaddrmsg struct: u8 family, u8 prefixlen, u8 flags, u8 scope, u32 index
-    ifaddrmsg = struct.pack('BBBBI', 0, 0, 0, 0, 0)
-    s = socket.socket(socket.AF_NETLINK, socket.SOCK_RAW, socket.NETLINK_ROUTE)
-    s.bind((0, 0))
-    s.sendall(nlhdr + ifaddrmsg)
     addrs = []
-    while True:
-        pdata = s.recv(65536)
-        v = memoryview(pdata)
-        if struct.unpack('H', v[4:6])[0] == 3:  # netlink done message
-            break
-        while len(v):
-            length, typ = struct.unpack('IH', v[:6])
-            if typ == 20:
-                fam, plen, _, scope, ridx = struct.unpack('BBBBI', v[nlhdrsz:nlhdrsz+ifaddrsz])
-                if scope in (253, 0):
-                    rta = v[nlhdrsz+ifaddrsz:length]
-                    while len(rta):
-                        rtalen, rtatyp = struct.unpack('HH', rta[:4])
-                        if rtalen < 4:
-                            break
-                        if rtatyp == 1:
-                            addrs.append((fam, rta[4:rtalen], plen, ridx))
-                        rta = rta[msg_align(rtalen):]
-            v = v[msg_align(length):]
+    for ifa in get_ifaddrs():
+        if ifa[0] == 'ip':
+            addrs.append((ifa[1], ifa[2], ifa[3]))
     return addrs
 
+def get_mac_addresses():
+    macs = []
+    for ifa in get_ifaddrs():
+        if ifa[0] == 'ETHER':
+            macs.append((ifa[1], ifa[2]))
+    return macs
 
-def scan_confluents():
+def get_ifaddrs():
+    class sockaddr(ctypes.Structure):
+        _fields_ = [
+            ('sa_family', ctypes.c_uint16),
+            ('sa_data', ctypes.c_ubyte * 14),
+        ]
+    
+    class sockaddr_in(ctypes.Structure):
+        _fields_ = [
+            ('sin_family', ctypes.c_uint16),
+            ('sin_port', ctypes.c_uint16),
+            ('sin_addr', ctypes.c_ubyte * 4),
+            ('sin_zero', ctypes.c_ubyte * 8),
+        ]
+    
+    class sockaddr_in6(ctypes.Structure):
+        _fields_ = [
+            ('sin6_family', ctypes.c_uint16),
+            ('sin6_port', ctypes.c_uint16),
+            ('sin6_flowinfo', ctypes.c_uint32),
+            ('sin6_addr', ctypes.c_ubyte * 16),
+            ('sin6_scope_id', ctypes.c_uint32),
+        ]
+    
+    class sockaddr_ll(ctypes.Structure):
+        _fields_ = [
+            ('sll_family', ctypes.c_uint16),
+            ('sll_protocol', ctypes.c_uint16),
+            ('sll_ifindex', ctypes.c_int32),
+            ('sll_hatype', ctypes.c_uint16),
+            ('sll_pkttype', ctypes.c_uint8),
+            ('sll_halen', ctypes.c_uint8),
+            ('sll_addr', ctypes.c_ubyte * 8),
+        ]
+    
+    class ifaddrs(ctypes.Structure):
+        pass
+
+    ifaddrs._fields_ = [
+            ('ifa_next', ctypes.POINTER(ifaddrs)),
+            ('ifa_name', ctypes.c_char_p),
+            ('ifa_flags', ctypes.c_uint),
+            ('ifa_addr', ctypes.POINTER(sockaddr)),
+            ('ifa_netmask', ctypes.POINTER(sockaddr)),
+            ('ifa_ifu', ctypes.POINTER(sockaddr)),
+            ('ifa_data', ctypes.c_void_p),
+        ]
+    
+    libc = ctypes.CDLL(ctypes.util.find_library('c'))
+    libc.getifaddrs.argtypes = [ctypes.POINTER(ctypes.POINTER(ifaddrs))]
+    libc.getifaddrs.restype = ctypes.c_int
+    libc.freeifaddrs.argtypes = [ctypes.POINTER(ifaddrs)]
+    libc.freeifaddrs.restype = None
+    ifap = ctypes.POINTER(ifaddrs)()
+    result = libc.getifaddrs(ctypes.pointer(ifap))
+    if result != 0:
+        return []
+    addresses = []
+    ifa = ifap
+    try:
+        while ifa:
+            if ifa.contents.ifa_addr:
+                family = ifa.contents.ifa_addr.contents.sa_family
+                name = ifa.contents.ifa_name.decode('utf-8') if ifa.contents.ifa_name else None
+                if family in (socket.AF_INET, socket.AF_INET6):
+                    # skip loopback and non-multicast interfaces
+                    if ifa.contents.ifa_flags & 8 or not ifa.contents.ifa_flags & 0x1000:
+                        ifa = ifa.contents.ifa_next
+                        continue
+                if family == socket.AF_INET:
+                    addr_ptr = ctypes.cast(ifa.contents.ifa_addr, ctypes.POINTER(sockaddr_in))
+                    addr_bytes = bytes(addr_ptr.contents.sin_addr)
+                    if_index = socket.if_nametoindex(name) if name else 0
+                    addresses.append(('ip', family, addr_bytes, if_index))
+                elif family == socket.AF_INET6:
+                    addr_ptr = ctypes.cast(ifa.contents.ifa_addr, ctypes.POINTER(sockaddr_in6))
+                    addr_bytes = bytes(addr_ptr.contents.sin6_addr)
+                    scope_id = addr_ptr.contents.sin6_scope_id
+                    addresses.append(('ip', family, addr_bytes, scope_id))          
+                elif family == socket.AF_PACKET:
+                    addr_ptr = ctypes.cast(ifa.contents.ifa_addr, ctypes.POINTER(sockaddr_ll))
+                    halen = addr_ptr.contents.sll_halen
+                    if addr_ptr.contents.sll_hatype in (1, 32) and halen > 0:  # ARPHRD_ETHER or ARPHRD_INFINIBAND
+                        if addr_ptr.contents.sll_hatype == 1 and addr_ptr.contents.sll_addr[0] & 2:  # skip locally administered MACs
+                            ifa = ifa.contents.ifa_next
+                            continue
+                        mac_bytes = bytes(addr_ptr.contents.sll_addr[:halen])
+                        macaddr = ':'.join('{:02x}'.format(b) for b in mac_bytes)
+                        addresses.append(('ETHER', name, macaddr))                
+            ifa = ifa.contents.ifa_next
+    finally:
+        libc.freeifaddrs(ifap)
+    
+    return addresses
+
+def scan_confluents(confuuid=None):
     srvs = {}
     s6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
     s6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
@@ -84,21 +167,24 @@ def scan_confluents():
     s4.bind(('0.0.0.0', 1900))
     doneidxs = set([])
     msg = 'M-SEARCH * HTTP/1.1\r\nST: urn:xcat.org:service:confluent:'
-    with open('/etc/confluent/confluent.deploycfg') as dcfg:
-        for line in dcfg.read().split('\n'):
-            if line.startswith('confluent_uuid:'):
-                confluentuuid = line.split(': ')[1]
-                msg += '/confluentuuid=' + confluentuuid
-                break
+    if not confuuid and os.path.exists('/etc/confluent/confluent.deploycfg'):
+        with open('/etc/confluent/confluent.deploycfg') as dcfg:
+            for line in dcfg.read().split('\n'):
+                if line.startswith('confluent_uuid:'):
+                    confluentuuid = line.split(': ')[1]
+                    msg += '/confluentuuid=' + confluentuuid
+                    break
+    if not confuuid and os.path.exists('/confluent_uuid'):
+        with open('/confluent_uuid') as cuuidin:
+            confluentuuid = cuuidin.read().strip()
+            msg += '/confluentuuid=' + confluentuuid
     try:
         with open('/sys/devices/virtual/dmi/id/product_uuid') as uuidin:
             msg += '/uuid=' + uuidin.read().strip()
     except Exception:
         pass
-    for addrf in glob.glob('/sys/class/net/*/address'):
-        with open(addrf) as addrin:
-            hwaddr = addrin.read().strip()
-            msg += '/mac=' + hwaddr
+    for iface, hwaddr in get_mac_addresses():
+        msg += '/mac=' + hwaddr
     msg = msg.encode('utf8')
     for addr in get_my_addresses():
         if addr[0] == socket.AF_INET6:
@@ -126,6 +212,7 @@ def scan_confluents():
     srvlist = []
     if r:
         r = r[0]
+    nodename = None
     while r:
         for s in r:
             (rsp, peer) = s.recvfrom(9000)
@@ -133,6 +220,7 @@ def scan_confluents():
             current = None
             for line in rsp:
                 if line.startswith(b'NODENAME: '):
+                    nodename = line.replace(b'NODENAME: ', b'').strip().decode('utf8')
                     current = {}
                 elif line.startswith(b'DEFAULTNET: 1'):
                     current['isdefault'] = True
@@ -144,20 +232,37 @@ def scan_confluents():
             if currip.startswith('fe80::') and '%' not in currip:
                 currip = '{0}%{1}'.format(currip, peer[-1])
             srvs[currip] = current
-            srvlist.append(currip)
+            if currip not in srvlist:
+                srvlist.append(currip)
         r = select.select((s4, s6), (), (), 2)
         if r:
             r = r[0]
+    if not os.path.exists('/etc/confluent/confluent.info'):
+        with open('/etc/confluent/confluent.info', 'w+') as cinfo:
+            if nodename:
+                cinfo.write('NODENAME: {0}\n'.format(nodename))
+            for srv in srvlist:
+                cinfo.write('MANAGER: {0}\n'.format(srv))
     return srvlist, srvs
 
 
-def get_net_apikey(nodename, mgr):
+def get_net_apikey(nodename, mgr, hmackey=None, confuuid=None):
     alpha = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789./'
     newpass = ''.join([alpha[x >> 2] for x in bytearray(os.urandom(32))])
     salt = '$5$' + ''.join([alpha[x >> 2] for x in bytearray(os.urandom(8))])
     newpass = newpass.encode('utf8')
     salt = salt.encode('utf8')
     crypted = c_crypt(newpass, salt)
+    if hmackey:
+        hmacvalue = hmac.new(hmackey.encode('utf8'), crypted, hashlib.sha256).digest()
+        hmacvalue = base64.b64encode(hmacvalue).decode('utf8')
+        client = HTTPSClient(host=mgr, phmac=hmacvalue, nodename=nodename, confuuid=confuuid)
+        try:
+            status, rsp = client.grab_url_with_status('/confluent-api/self/registerapikey', data=crypted, returnrsp=True)
+            if status == 200:
+                return newpass.decode('utf8')
+        except Exception:
+            pass
     for addrinfo in socket.getaddrinfo(mgr, 13001, 0, socket.SOCK_STREAM):
         try:
             clisock = socket.socket(addrinfo[0], addrinfo[1])
@@ -195,7 +300,7 @@ def get_net_apikey(nodename, mgr):
     return ''
 
 
-def get_apikey(nodename, hosts, errout=None):
+def get_apikey(nodename, hosts, errout=None, hmackey=None, confuuid=None):
     apikey = ""
     if os.path.exists('/etc/confluent/confluent.apikey'):
         apikey = open('/etc/confluent/confluent.apikey').read().strip()
@@ -204,16 +309,16 @@ def get_apikey(nodename, hosts, errout=None):
     while not apikey:
         for host in hosts:
             try:
-                apikey = get_net_apikey(nodename, host)
+                apikey = get_net_apikey(nodename, host, hmackey=hmackey, confuuid=confuuid)
             except OSError:
                 apikey = None
             if apikey:
                 break
         else:
-            srvlist, _ = scan_confluents()
+            srvlist, _ = scan_confluents(confuuid=confuuid)
             for host in srvlist:
                 try:
-                    apikey = get_net_apikey(nodename, host)
+                    apikey = get_net_apikey(nodename, host, hmackey=hmackey, confuuid=confuuid)
                 except OSError:
                     apikey = None
                 if apikey:
@@ -231,35 +336,43 @@ def get_apikey(nodename, hosts, errout=None):
     return apikey
 
 class HTTPSClient(client.HTTPConnection, object):
-    def __init__(self, usejson=False, port=443, host=None, errout=None, phmac=None, checkonly=False):
+    def __init__(self, usejson=False, port=443, host=None, errout=None, phmac=None, checkonly=False, hmackey=None, nodename=None, confuuid=None):
         self.ignorehosts = set([])
         self.phmac = phmac
+        self.hmackey = hmackey
+        self.confuuid = confuuid
         self.errout = None
+        self.stdheaders = {}
+        if nodename:
+            self.stdheaders['CONFLUENT_NODENAME'] = nodename
         if errout:
             self.errout = open(errout, 'w')
             self.errout.flush()
-        self.stdheaders = {}
         mgtiface = None
         if usejson:
             self.stdheaders['ACCEPT'] = 'application/json'
         if host:
             self.hosts = [host]
-            with open('/etc/confluent/confluent.info') as cinfo:
-                info = cinfo.read().split('\n')
-                for line in info:
-                    if line.startswith('NODENAME:'):
-                        node = line.split(' ')[1]
-                        self.stdheaders['CONFLUENT_NODENAME'] = node
+            if not nodename:
+                with open('/etc/confluent/confluent.info') as cinfo:
+                    info = cinfo.read().split('\n')
+                    for line in info:
+                        if line.startswith('NODENAME:'):
+                            nodename = line.split(' ')[1]
+            self.stdheaders['CONFLUENT_NODENAME'] = nodename
         else:
             self.hosts = []
-            info = open('/etc/confluent/confluent.info').read().split('\n')
+            try:
+                info = open('/etc/confluent/confluent.info').read().split('\n')
+            except Exception:
+                info = []
             havedefault = '0'
             plainhost = ''
             for line in info:
                 host = ''
                 if line.startswith('NODENAME:'):
-                    node = line.split(' ')[1]
-                    self.stdheaders['CONFLUENT_NODENAME'] = node
+                    nodename = line.split(' ')[1]
+                    self.stdheaders['CONFLUENT_NODENAME'] = nodename
                 if line.startswith('MANAGER:') and not host:
                     host = line.split(' ')[1]
                     self.hosts.append(host)
@@ -294,15 +407,14 @@ class HTTPSClient(client.HTTPConnection, object):
             if plainhost and not self.hosts:
                 self.hosts.append(plainhost)
         if self.phmac:
-            with open(phmac, 'r') as hmacin:
-                self.stdheaders['CONFLUENT_CRYPTHMAC'] = hmacin.read()
+            self.stdheaders['CONFLUENT_CRYPTHMAC'] = self.phmac
         elif not checkonly:
-            self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(node, self.hosts, errout=self.errout)
+            self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(nodename, self.hosts, errout=self.errout, hmackey=hmackey, confuuid=self.confuuid)
         if mgtiface:
             self.stdheaders['CONFLUENT_MGTIFACE'] = mgtiface
         self.port = port
         self.host = None
-        self.node = node
+        self.node = nodename
         host = self.check_connections()
         client.HTTPConnection.__init__(self, host, port)
         self.connect()
@@ -342,7 +454,7 @@ class HTTPSClient(client.HTTPConnection, object):
                 continue
             break
         if not foundsrv:
-            srvlist, srvs = scan_confluents()
+            srvlist, srvs = scan_confluents(self.confuuid)
             hosts = []
             for srv in srvlist:
                 if srvs[srv].get('isdefault', False):
@@ -416,7 +528,7 @@ class HTTPSClient(client.HTTPConnection, object):
                 with open('/etc/confluent/confluent.apikey', 'w+') as akfile:
                     akfile.write('')
                 self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(
-                    self.node, [self.host], errout=self.errout)
+                    self.node, [self.host], errout=self.errout, hmackey=self.hmackey, confuuid=self.confuuid)
             if rsp.status == 503:  # confluent is down, but the server running confluent is otherwise up
                 authed = False
                 self.ignorehosts.add(self.host)
@@ -545,8 +657,24 @@ if __name__ == '__main__':
         phmac = sys.argv.index('-p')
         sys.argv.pop(phmac)
         phmac = sys.argv.pop(phmac)
+        with open(phmac, 'r') as hmacin:
+            phmac = hmacin.read()
     except ValueError:
         phmac = None
+    try:
+        identfile = sys.argv.index('-i')
+        sys.argv.pop(identfile)
+        identfile = sys.argv.pop(identfile)
+        with open(identfile) as idin:
+            data = idin.read()
+            identinfo = json.loads(data)
+            nodename = identinfo.get('nodename', None)
+            hmackey = identinfo.get('apitoken', None)
+            confuuid = identinfo.get('confluent_uuid', None)
+    except ValueError:
+        hmackey = None
+        nodename = None
+        confuuid = None
     try:
         checkonly = False
         idxit = sys.argv.index('-c')
@@ -558,7 +686,7 @@ if __name__ == '__main__':
         data = open(sys.argv[-1]).read()
     if outbin:
         with open(outbin, 'ab+') as outf:
-            reader = HTTPSClient(usejson=usejson, errout=errout).grab_url(
+            reader = HTTPSClient(usejson=usejson, errout=errout, hmackey=hmackey, nodename=nodename, confuuid=confuuid).grab_url(
                 sys.argv[1], data, returnrsp=True)
             chunk = reader.read(16384)
             while chunk:
@@ -566,7 +694,7 @@ if __name__ == '__main__':
                 chunk = reader.read(16384)
         sys.exit(0)
 
-    mclient = HTTPSClient(usejson, errout=errout, phmac=phmac, checkonly=checkonly)
+    mclient = HTTPSClient(usejson, errout=errout, phmac=phmac, checkonly=checkonly, hmackey=hmackey, nodename=nodename, confuuid=confuuid)
     if waitfor:
         status = 201
         while status != waitfor:

confluent_osdeploy/common/profile/scripts/confignet

@@ -356,7 +356,7 @@ class NetworkManager(object):
             currteam = deats.get('connection.master', None)
             if currteam == team:
                 return
-            for stg in ('ipv4.dhcp-hostname', 'ipv4.dns', 'ipv6.dns', 'ipv6.dhcp-hostname'):
+            for stg in ('ipv4.dhcp-hostname', 'ipv4.dns', 'ipv6.dns', 'ipv6.dhcp-hostname', 'ipv4.dns-search', 'ipv6.dns-search'):
                 if deats.get(stg, None):
                     bondcfg[stg] = deats[stg]
         if member in self.uuidbyname:
@@ -460,6 +460,9 @@ class NetworkManager(object):
 
 
 if __name__ == '__main__':
+    checktarg = None
+    if '-c' in sys.argv:
+        checktarg = sys.argv[sys.argv.index('-c') + 1]
     havefirewall = subprocess.call(['systemctl', 'status', 'firewalld'])
     havefirewall = havefirewall == 0
     if havefirewall:
@@ -485,15 +488,16 @@ if __name__ == '__main__':
             continue
         myname = s.getsockname()
         s.close()
+        curridx = None
         if len(myname) == 4:
             curridx = myname[-1]
         else:
             myname = myname[0]
             myname = socket.inet_pton(socket.AF_INET, myname)
             for addr in myaddrs:
-                if myname == addr[1].tobytes():
+                if myname == addr[1]:
                     curridx = addr[-1]
-        if curridx in doneidxs:
+        if curridx is not None and curridx in doneidxs:
             continue
         for tries in (1, 2, 3):
             try:
@@ -545,7 +549,7 @@ if __name__ == '__main__':
     rm_tmp_llas(tmpllas)
     if os.path.exists('/usr/sbin/netplan'):
         nm = NetplanManager(dc)
-    if os.path.exists('/usr/bin/nmcli'):
+    elif os.path.exists('/usr/bin/nmcli'):
         nm = NetworkManager(devtypes, dc)
     elif os.path.exists('/usr/sbin/wicked'):
         nm = WickedManager()
@@ -567,4 +571,27 @@ if __name__ == '__main__':
     if havefirewall:
         subprocess.check_call(['systemctl', 'start', 'firewalld'])
     await_tentative()
+    maxwait = 10
+    while maxwait:
+        try:
+            tclient = apiclient.HTTPSClient(checkonly=True)
+            tclient.check_connections()
+            break
+        except Exception:
+            maxwait -= 1
+            time.sleep(1)
+    maxwait = 10
+    if checktarg:
+        while maxwait:
+            try:
+                addrinf = socket.getaddrinfo(checktarg, 443)[0]
+                psock = socket.socket(addrinf[0], socket.SOCK_STREAM)
+                psock.settimeout(10)
+                psock.connect(addrinf[4])
+                psock.close()
+                break
+            except Exception:
+                maxwait -= 1
+                time.sleep(1)
+
 

confluent_osdeploy/common/profile/scripts/setupssh

@@ -7,6 +7,7 @@ for pubkey in /etc/ssh/ssh_host*key.pub; do
         continue
     fi
     certfile=${pubkey/.pub/-cert.pub}
+    echo -n > $certfile
     confluentpython $confapiclient /confluent-api/self/sshcert $pubkey -o $certfile
 done
 if [ -d /etc/ssh/sshd_config.d/ -a ! -e /etc/ssh/sshd_config.d/90-confluent.conf ]; then
@@ -16,6 +17,13 @@ if [ -d /etc/ssh/sshd_config.d/ -a ! -e /etc/ssh/sshd_config.d/90-confluent.conf
 	echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/90-confluent.conf
 	echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/90-confluent.conf
 	echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/90-confluent.conf
+elif [ ! -d /etc/ssh/sshd_config.d/ ] && ! grep HostCertificate /etc/ssh/sshd_config > /dev/null; then
+	for cert in /etc/ssh/ssh*-cert.pub; do
+		echo HostCertificate $cert >> /etc/ssh/sshd_config
+	done
+	echo HostbasedAuthentication yes >> /etc/ssh/sshd_config
+	echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config
+	echo IgnoreRhosts no >> /etc/ssh/sshd_config
 fi
 
 TMPDIR=$(mktemp -d)
@@ -24,12 +32,17 @@ confluentpython $confapiclient /confluent-public/site/initramfs.tgz -o initramfs
 tar xf initramfs.tgz
 for ca in ssh/*.ca; do
 	LINE=$(cat $ca)
-        if [ -z "$LINE" ]; then continue; fi
-	cp -af /etc/ssh/ssh_known_hosts /etc/ssh/ssh_known_hosts.new
-	grep -v "$LINE" /etc/ssh/ssh_known_hosts > /etc/ssh/ssh_known_hosts.new
+	if [ -z "$LINE" ]; then continue; fi
+	if [ -f /etc/ssh/ssh_known_hosts ]; then
+		cp -af /etc/ssh/ssh_known_hosts /etc/ssh/ssh_known_hosts.new
+		grep -v "$LINE" /etc/ssh/ssh_known_hosts > /etc/ssh/ssh_known_hosts.new
+	fi
 	echo '@cert-authority *' $LINE >> /etc/ssh/ssh_known_hosts.new
 	mv /etc/ssh/ssh_known_hosts.new /etc/ssh/ssh_known_hosts
 done
+mkdir -p /root/.ssh/
+chmod 700 /root/.ssh/
+touch /root/.ssh/authorized_keys
 for pubkey in ssh/*.*pubkey; do
 	LINE=$(cat $pubkey)
         if [ -z "$LINE" ]; then continue; fi

confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl

@@ -27,7 +27,6 @@ mkdir -p stateless-bin
 cp -a el8bin/* .
 ln -s el8 el9
 ln -s el8 el10
-mv el10/initramfs/usr el10/initramfs/var
 cp -a debian debian13
 mkdir -p debian13/initramfs/usr
 mv debian13/initramfs/lib debian13/initramfs/usr/
@@ -86,6 +85,9 @@ cp -a esxi7 esxi8
 %install
 mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
 #cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
+mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/common
+cp common/initramfs/opt/confluent/bin/apiclient %{buildroot}/opt/confluent/lib/osdeploy/common
+
 for os in rhvh4 el7 el8 el9 el10 debian debian13 genesis suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 esxi6 esxi7 esxi8 coreos; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs/aarch64/
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs/aarch64/

confluent_osdeploy/debian/profiles/default/scripts/functions

@@ -10,6 +10,13 @@ function test_mgr() {
     return 1
 }
 
+function initconfluentscriptstmp() {
+    if [ -z "$confluentscripttmpdir" ]; then
+        mkdir -p /opt/confluent/tmpexec
+        confluentscripttmpdir=$(mktemp -d /opt/confluent/tmpexec/confluentscripts.XXXXXXXXX)
+    fi
+}
+
 function confluentpython() {
     if [ -x /usr/libexec/platform-python ]; then
         /usr/libexec/platform-python $*
@@ -72,7 +79,8 @@ fetch_remote() {
 }
 
 source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -86,7 +94,8 @@ source_remote_parts() {
 }
 
 run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -105,10 +114,7 @@ source_remote() {
     echo
     echo '---------------------------------------------------------------------------'
     echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
-        unsettmpdir=1
-    fi
+    initconfluentscriptstmp
     echo Sourcing from $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -135,9 +141,9 @@ run_remote() {
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
         unsettmpdir=1
     fi
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -170,7 +176,8 @@ run_remote_python() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)

confluent_osdeploy/el10-diskless/profiles/default/scripts/imageboot.sh

@@ -3,8 +3,8 @@ confluent_whost=$confluent_mgr
 if [[ "$confluent_whost" == *:* ]] && [[ "$confluent_whost" != "["* ]]; then
     confluent_whost="[$confluent_mgr]"
 fi
-mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
-if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
+mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay /sysroot
+if [ "untethered" = "$(getarg confluent_imagemethod)" -o "uncompressed" = "$(getarg confluent_imagemethod)" ]; then
     mount -t tmpfs untethered /mnt/remoteimg
     curl https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
 else
@@ -40,20 +40,53 @@ fi
 
 
 #mount -t tmpfs overlay /mnt/overlay
-modprobe zram
-memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
-memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
-echo $memtot > /sys/block/zram0/disksize
-mkfs.xfs /dev/zram0 > /dev/null
-mount -o discard /dev/zram0 /mnt/overlay
-if [ ! -f /tmp/mountparts.sh ]; then
-    mkdir -p /mnt/overlay/upper /mnt/overlay/work
-    mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
+if [ ! "uncompressed" = "$(getarg confluent_imagemethod)" ]; then
+    modprobe zram
+    memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
+    memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
+    echo $memtot > /sys/block/zram0/disksize
+    mkfs.xfs /dev/zram0 > /dev/null
+fi
+TETHERED=0
+if [ "untethered" = "$(getarg confluent_imagemethod)" -o "uncompressed" = "$(getarg confluent_imagemethod)" ]; then
+    if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
+        mount -o discard /dev/zram0 /sysroot
+    else
+        mount -t tmpfs disklessroot /sysroot
+    fi
+    echo -en "Decrypting and extracting root filesystem: 0%\r"
+    srcsz=$(du -sk /mnt/remote | awk '{print $1}')
+    while [ -f /mnt/remoteimg/rootimg.sfs ]; do
+        dstsz=$(du -sk /sysroot | awk '{print $1}')
+        pct=$((dstsz * 100 / srcsz))
+        if [ $pct -gt 99 ]; then
+            pct=99
+        fi
+        echo -en "Decrypting and extracting root filesystem: $pct%\r"
+        sleep 0.25
+    done &
+    cp -ax /mnt/remote/* /sysroot/
+    umount /mnt/remote
+    if [ -e /dev/mapper/cryptimg ]; then
+        dmsetup remove cryptimg
+    fi
+    losetup -d $loopdev
+    rm /mnt/remoteimg/rootimg.sfs
+    umount /mnt/remoteimg
+    wait
+    echo -e "Decrypting and extracting root filesystem: 100%"
 else
-    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
-        mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
-        mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
-    done
+    TETHERED=1
+    mount -o discard /dev/zram0 /mnt/overlay
+    if [ ! -f /tmp/mountparts.sh ]; then
+        mkdir -p /mnt/overlay/upper /mnt/overlay/work
+        mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
+    else
+        for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
+            mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
+            mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
+        done
+    fi
 fi
 mkdir -p /sysroot/etc/ssh
 mkdir -p /sysroot/etc/confluent
@@ -109,7 +142,7 @@ echo '    EnableSSHKeysign yes' >> $sshconf
 echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
 curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_whost/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
 cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
-chmod 640 /sysroot/etc/ssh/*_key
+chmod 600 /sysroot/etc/ssh/*_key
 cp /tls/*.pem /sysroot/etc/pki/ca-trust/source/anchors/
 chroot /sysroot/ update-ca-trust
 curl -sf https://$confluent_whost/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
@@ -129,4 +162,25 @@ ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
 mv /lib/firmware /lib/firmware-ramfs
 ln -s /sysroot/lib/firmware /lib/firmware
 kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
-exec /opt/confluent/bin/start_root
+if [ $TETHERED -eq 1 ]; then
+    (
+        sleep 86400 &
+        ONBOOTPID=$!
+        mkdir -p /run/confluent
+        echo $ONBOOTPID > /run/confluent/onboot_sleep.pid
+        wait $ONBOOTPID
+        dd if=/mnt/remoteimg/rootimg.sfs iflag=nocache count=0 >& /dev/null
+        rm -rf /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs /lib/firmware-ramfs /usr/lib64/libcrypto.so* /usr/lib64/systemd/ /kernel/ /usr/bin/ /usr/sbin/ /usr/libexec/
+    ) &
+    while [ ! -f /run/confluent/onboot_sleep.pid ]; do
+        sleep 0.1
+    done
+else
+    rm -rf /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs /lib/firmware-ramfs /usr/lib64/libcrypto.so* /usr/lib64/systemd/ /kernel/ /usr/bin/ /usr/sbin/ /usr/libexec/
+fi
+if grep debugssh /proc/cmdline >& /dev/null; then
+    exec /opt/confluent/bin/start_root
+else
+    exec /opt/confluent/bin/start_root -s  # share mount namespace, keep kernel callbacks intact
+fi
+

confluent_osdeploy/el7-diskless/profiles/default/scripts/functions

@@ -10,6 +10,13 @@ function test_mgr() {
     return 1
 }
 
+function initconfluentscriptstmp() {
+    if [ -z "$confluentscripttmpdir" ]; then
+        mkdir -p /opt/confluent/tmpexec
+        confluentscripttmpdir=$(mktemp -d /opt/confluent/tmpexec/confluentscripts.XXXXXXXXX)
+    fi
+}
+
 function confluentpython() {
     if [ -x /usr/libexec/platform-python ]; then
         /usr/libexec/platform-python $*
@@ -72,7 +79,8 @@ fetch_remote() {
 }
 
 source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -86,7 +94,8 @@ source_remote_parts() {
 }
 
 run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -105,10 +114,7 @@ source_remote() {
     echo
     echo '---------------------------------------------------------------------------'
     echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
-        unsettmpdir=1
-    fi
+    initconfluentscriptstmp
     echo Sourcing from $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -135,9 +141,9 @@ run_remote() {
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
         unsettmpdir=1
     fi
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -170,7 +176,8 @@ run_remote_python() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)

confluent_osdeploy/el7/profiles/default/scripts/functions

@@ -10,6 +10,13 @@ function test_mgr() {
     return 1
 }
 
+function initconfluentscriptstmp() {
+    if [ -z "$confluentscripttmpdir" ]; then
+        mkdir -p /opt/confluent/tmpexec
+        confluentscripttmpdir=$(mktemp -d /opt/confluent/tmpexec/confluentscripts.XXXXXXXXX)
+    fi
+}
+
 function confluentpython() {
     if [ -x /usr/libexec/platform-python ]; then
         /usr/libexec/platform-python $*
@@ -72,7 +79,8 @@ fetch_remote() {
 }
 
 source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -86,7 +94,8 @@ source_remote_parts() {
 }
 
 run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -105,10 +114,7 @@ source_remote() {
     echo
     echo '---------------------------------------------------------------------------'
     echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
-        unsettmpdir=1
-    fi
+    initconfluentscriptstmp
     echo Sourcing from $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -135,9 +141,9 @@ run_remote() {
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
         unsettmpdir=1
     fi
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -170,7 +176,8 @@ run_remote_python() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)

confluent_osdeploy/el8-diskless/profiles/default/scripts/firstboot.sh

@@ -25,7 +25,8 @@ if [ ! -f /etc/confluent/firstboot.ran ]; then
     touch /etc/confluent/firstboot.ran
 
     cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
-
+    confluentpython /root/confignet
+    rm /root/confignet
     run_remote firstboot.custom
     # Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
     run_remote_parts firstboot.d

confluent_osdeploy/el8-diskless/profiles/default/scripts/functions

@@ -10,6 +10,13 @@ function test_mgr() {
     return 1
 }
 
+function initconfluentscriptstmp() {
+    if [ -z "$confluentscripttmpdir" ]; then
+        mkdir -p /opt/confluent/tmpexec
+        confluentscripttmpdir=$(mktemp -d /opt/confluent/tmpexec/confluentscripts.XXXXXXXXX)
+    fi
+}
+
 function confluentpython() {
     if [ -x /usr/libexec/platform-python ]; then
         /usr/libexec/platform-python $*
@@ -72,7 +79,8 @@ fetch_remote() {
 }
 
 source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -86,7 +94,8 @@ source_remote_parts() {
 }
 
 run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -105,10 +114,7 @@ source_remote() {
     echo
     echo '---------------------------------------------------------------------------'
     echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
-        unsettmpdir=1
-    fi
+    initconfluentscriptstmp
     echo Sourcing from $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -135,9 +141,9 @@ run_remote() {
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
         unsettmpdir=1
     fi
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -170,7 +176,8 @@ run_remote_python() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)

confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py

@@ -56,7 +56,11 @@ def get_image_metadata(imgpath):
             for md in get_multipart_image_meta(img):
                 yield md
         else:
-            raise Exception('Installation from single part image not supported')
+            yield {'mount': '/', 'filesystem': 'xfs', 'minsize': 4294967296, 'initsize': 954128662528, 'flags': 'rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota', 'device': '/dev/mapper/root', 'compressed_size': 27022069760}
+            yield {'mount': '/boot', 'filesystem': 'xfs', 'minsize': 536870912, 'initsize': 1006632960, 'flags': 'rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota', 'device': '/dev/nvme1n1p2', 'compressed_size': 171462656}
+            yield {'mount': '/boot/efi', 'filesystem': 'vfat', 'minsize': 33554432, 'initsize': 627900416, 'flags': 'rw,relatime,fmask=0077,dmask=0077,codepage=437,iocharset=ascii,shortname=winnt,errors=remount-ro', 'device': '/dev/nvme1n1p1', 'compressed_size': 1576960}            
+            #raise Exception('Installation from single part image not supported')            
+
 
 class PartedRunner():
     def __init__(self, disk):
@@ -75,8 +79,17 @@ def fixup(rootdir, vols):
     for vol in vols:
         devbymount[vol['mount']] = vol['targetdisk']
     fstabfile = os.path.join(rootdir, 'etc/fstab')
-    with open(fstabfile) as tfile:
-        fstab = tfile.read().split('\n')
+    if os.path.exists(fstabfile):
+        with open(fstabfile) as tfile:
+            fstab = tfile.read().split('\n')
+    else:
+        # fabricate a reference fstab
+        fstab = [
+            "#ORIGFSTAB#/dev/mapper/root# /                       xfs     defaults        0 0",
+            "#ORIGFSTAB#UUID=aaf9e0f9-aa4d-4d74-9e75-3537620cfe23# /boot                   xfs     defaults        0 0",
+            "#ORIGFSTAB#UUID=C21D-B881#          /boot/efi               vfat    umask=0077,shortname=winnt 0 2",
+            "#ORIGFSTAB#/dev/mapper/swap# none                    swap    defaults        0 0",
+        ]       
     while not fstab[0]:
         fstab = fstab[1:]
     if os.path.exists(os.path.join(rootdir, '.autorelabel')):
@@ -126,8 +139,10 @@ def fixup(rootdir, vols):
         newcfg = ifcfg.split('/')[-1]
         newcfg = os.path.join(rootdir, 'etc/NetworkManager/system-connections/{0}'.format(newcfg))
         shutil.copy2(ifcfg, newcfg)
-    shutil.rmtree(os.path.join(rootdir, 'etc/confluent/'))
-    shutil.copytree('/etc/confluent', os.path.join(rootdir, 'etc/confluent'))
+    rootconfluentdir = os.path.join(rootdir, 'etc/confluent/')
+    if os.path.exists(rootconfluentdir):
+        shutil.rmtree(rootconfluentdir)
+    shutil.copytree('/etc/confluent', rootconfluentdir)
     if policy:
         sys.stdout.write('Applying SELinux labeling...')
         sys.stdout.flush()
@@ -142,14 +157,41 @@ def fixup(rootdir, vols):
     grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub')
     if not os.path.exists(grubsyscfg):
         grubsyscfg = os.path.join(rootdir, 'etc/default/grub')
-    with open(grubsyscfg) as defgrubin:
-        defgrub = defgrubin.read().split('\n')
+    currcmdline = []
+    with open('/proc/cmdline') as cmdlinein:
+        cmdline = cmdlinein.read().strip()
+        for arg in cmdline.split():
+            if arg.startswith('console='):
+                currcmdline.append(arg)
+            elif arg == 'quiet':
+                currcmdline.append(arg)
+    currcmdlinestr = ' '.join(currcmdline)
+    if os.path.exists(grubsyscfg):
+        with open(grubsyscfg) as defgrubin:
+            defgrub = defgrubin.read().split('\n')
+    else:
+        defgrub = [
+        'GRUB_TIMEOUT=5',
+        'GRUB_DISTRIBUTOR="$(sed ' + "'s, release .*$,,g'" + ' /etc/system-release)"',
+        'GRUB_DEFAULT=saved',
+        'GRUB_DISABLE_SUBMENU=true',
+        'GRUB_TERMINAL=""',
+        'GRUB_SERIAL_COMMAND=""',
+        'GRUB_CMDLINE_LINUX="{} crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M rd.lvm.lv=vg/root rd.lvm.lv=vg/swap"'.format(currcmdlinestr),
+        'GRUB_DISABLE_RECOVERY="true"',
+        'GRUB_ENABLE_BLSCFG=true',
+        ]
+    if not os.path.exists(os.path.join(rootdir, "etc/kernel/cmdline")):
+        with open(os.path.join(rootdir, "etc/kernel/cmdline"), "w") as cmdlineout:
+            cmdlineout.write("{} root=/dev/mapper/localstorage-root rd.lvm.lv=localstorage/root".format(currcmdlinestr))
     with open(grubsyscfg, 'w') as defgrubout:
         for gline in defgrub:
             gline = gline.split()
             newline = []
             for ent in gline:
                 if ent.startswith('resume=') or ent.startswith('rd.lvm.lv'):
+                    if ent.endswith('"'):
+                        newline.append('"')
                     continue
                 newline.append(ent)
             defgrubout.write(' '.join(newline) + '\n')
@@ -159,6 +201,12 @@ def fixup(rootdir, vols):
         grubcfg = grubcfg[:-1]
     if len(grubcfg) == 1:
         grubcfg = grubcfg[0]
+    elif not grubcfg:
+        grubcfg = '/boot/grub2/grub.cfg'
+        paths = glob.glob(os.path.join(rootdir, 'boot/efi/EFI/*'))
+        for path in paths:
+            with open(os.path.join(path, 'grub.cfg'), 'w') as stubgrubout:
+                stubgrubout.write("search --no-floppy --root-dev-only --fs-uuid --set=dev " + bootuuid + "\nset prefix=($dev)/grub2\nexport $prefix\nconfigfile $prefix/grub.cfg\n")
     else:
         for gcfg in grubcfg:
             rgcfg = os.path.join(rootdir, gcfg[1:])  # gcfg has a leading / to get rid of
@@ -214,10 +262,18 @@ def fixup(rootdir, vols):
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
+    try:
+        os.makedirs(os.path.join(rootdir, 'opt/confluent/bin'))
+    except Exception:
+        pass
+    shutil.copy2('/opt/confluent/bin/apiclient', os.path.join(rootdir, 'opt/confluent/bin/apiclient'))
     #other network interfaces
 
 
 def had_swap():
+    if not os.path.exists('/etc/fstab'):
+        # diskless source, assume swap
+        return True
     with open('/etc/fstab') as tabfile:
         tabs = tabfile.read().split('\n')
         for tab in tabs:
@@ -362,6 +418,8 @@ def install_to_disk(imgpath):
             subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ'])
             source = vol['mount'].replace('/', '_')
             source = '/run/imginst/sources/' + source
+            if not os.path.exists(source):
+                source = '/run/imginst/sources/_' + vol['mount']            
             blankfsstat = os.statvfs('/run/imginst/targ')
             blankused = (blankfsstat.f_blocks - blankfsstat.f_bfree) * blankfsstat.f_bsize
             sys.stdout.write('\nWriting {0}: '.format(vol['mount']))
@@ -419,8 +477,14 @@ def install_to_disk(imgpath):
 
 
 
-
-            subprocess.check_call(['umount', '/run/imginst/targ'])
+            while True:
+                try:
+                    subprocess.check_call(['umount', '/run/imginst/targ'])
+                except subprocess.CalledProcessError:
+                    print("Failed to unmount /run/imginst/targ, retrying")
+                    time.sleep(1)
+                else:
+                    break
         for vol in allvols:
             subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
         fixup('/run/imginst/targ', allvols)

confluent_osdeploy/el8-diskless/profiles/default/scripts/imageboot.sh

@@ -4,10 +4,12 @@ if [[ "$confluent_whost" == *:* ]] && [[ "$confluent_whost" != "["* ]]; then
     confluent_whost="[$confluent_mgr]"
 fi
 mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
+TETHERED=0
 if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
     mount -t tmpfs untethered /mnt/remoteimg
     curl https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
 else
+    TETHERED=1
     confluent_urls="$confluent_urls https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs"
     /opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg
 fi
@@ -130,4 +132,17 @@ ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
 mv /lib/firmware /lib/firmware-ramfs
 ln -s /sysroot/lib/firmware /lib/firmware
 kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
+if [ $TETHERED -eq 1 ]; then
+    (
+        sleep 86400 &
+        ONBOOTPID=$!
+        mkdir -p /run/confluent
+        echo $ONBOOTPID > /run/confluent/onboot_sleep.pid
+        wait $ONBOOTPID
+        dd if=/mnt/remoteimg/rootimg.sfs iflag=nocache count=0 >& /dev/null
+    ) &
+    while [ ! -f /run/confluent/onboot_sleep.pid ]; do
+        sleep 0.1
+    done
+fi
 exec /opt/confluent/bin/start_root

confluent_osdeploy/el8-diskless/profiles/default/scripts/installimage

@@ -5,6 +5,7 @@
 # and existing mounts of image (to take advantage of caching)
 mount -o bind /sys /sysroot/sys
 mount -o bind /dev /sysroot/dev
+mount -o bind /dev/pts /sysroot/dev/pts
 mount -o bind /proc /sysroot/proc
 mount -o bind /run /sysroot/run
 
@@ -21,8 +22,14 @@ else
     done
 fi
 cd /sysroot/run
+cp /run/sshd.pid /tmp/dbgssh.pid
+chroot /sysroot/ bash -c "/usr/sbin/sshd"
 chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_python getinstalldisk"
 chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_parts pre.d"
+for nameserver in $(sed -n '/^nameservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //'); do
+    echo "nameserver $nameserver" >> /sysroot/etc/resolv.conf
+done
+#chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_python confignet"
 if [ ! -f /sysroot/tmp/installdisk ]; then
     echo 'Unable to find a suitable installation target device, ssh to port 2222 to investigate'
     while [ ! -f /sysroot/tmp/installdisk ]; do
@@ -39,7 +46,10 @@ chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python imag
 echo "Port 22" >> /etc/ssh/sshd_config
 echo 'Match LocalPort 22' >> /etc/ssh/sshd_config
 echo '    ChrootDirectory /sysroot/run/imginst/targ' >> /etc/ssh/sshd_config
-kill -HUP $(cat /run/sshd.pid)
+kill $(cat /sysroot/var/run/sshd.pid)
+kill -HUP $(cat /tmp/dbgssh.pid)
+cp /sysroot/etc/pki/ca-trust/source/anchors/* /sysroot/run/imginst/targ/etc/pki/ca-trust/source/anchors/
+chroot /sysroot/run/imginst/targ update-ca-trust
 
 chroot /sysroot/run/imginst/targ bash -c "source /etc/confluent/functions; run_remote post.sh"
 chroot /sysroot bash -c "umount \$(tac /proc/mounts|awk '{print \$2}'|grep ^/run/imginst/targ)"

confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh

@@ -59,7 +59,7 @@ rpm --import /etc/pki/rpm-gpg/*
 
 run_remote_python add_local_repositories
 run_remote_python syncfileclient
-run_remote_python confignet
+run_remote_python confignet -c $confluent_mgr
 
 run_remote onboot.custom
 # onboot scripts may be placed into onboot.d, e.g. onboot.d/01-firstaction.sh, onboot.d/02-secondaction.sh
@@ -68,5 +68,14 @@ run_remote_parts onboot.d
 # Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/
 run_remote_config onboot.d
 
+if [ -f /run/confluent/onboot_sleep.pid ]; then
+    loopdev=$(losetup -j /mnt/remoteimg/rootimg.sfs|cut -d: -f 1)
+    if [ -n "$loopdev" ]; then
+        losetup "$loopdev" --direct-io=on
+    fi
+    sleeppid=$(cat /run/confluent/onboot_sleep.pid)
+    kill "$sleeppid"
+    rm -f /run/confluent/onboot_sleep.pid
+fi
 #curl -X POST -d 'status: booted' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
 kill $logshowpid

confluent_osdeploy/el8-diskless/profiles/default/scripts/post.sh

@@ -2,13 +2,17 @@
 
 # This script is executed 'chrooted' into a cloned disk target before rebooting
 #
-
+if [ -f /etc/dracut.conf.d/diskless.conf ]; then
+    rm /etc/dracut.conf.d/diskless.conf
+fi
+for kver in /lib/modules/*; do kver=$(basename $kver); kernel-install add $kver /boot/vmlinuz-$kver; done
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
+run_remote setupssh
 mkdir -p /var/log/confluent
 chmod 700 /var/log/confluent
 exec >> /var/log/confluent/confluent-post.log
@@ -33,6 +37,8 @@ run_remote_parts post.d
 # Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/
 run_remote_config post.d
 
+cd /root/
+fetch_remote confignet
 curl -sf -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
 
 kill $logshowpid

confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient

@@ -277,7 +277,10 @@ def synchronize():
                             try:
                                 uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid
                             except KeyError:
-                                uid = opts[fname][opt]['id']
+                                try:
+                                    uid = opts[fname][opt]['id']
+                                except KeyError:
+                                    raise Exception(f"Unable to map owner of {fname}") 
                         elif opt == 'group':
                             try:
                                 gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid

confluent_osdeploy/el8/profiles/default/initprofile.sh

@@ -3,7 +3,7 @@ sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/; s/oraclelinux/Oracle
 if grep Fedora $2/profile.yaml > /dev/null; then
 	sed -i 's/@^minimal-environment/#/' $2/packagelist
 fi
-if grep ^label: $2/profile.yaml | grep 10 > /dev/null; then
+if grep ^label: $2/profile.yaml | grep ' 10' > /dev/null; then
 	echo 'echo openssh-keysign >> /tmp/addonpackages' > $2/scripts/pre.d/enablekeysign
 	chmod 644 $2/scripts/pre.d/enablekeysign
 fi

confluent_osdeploy/el8/profiles/default/scripts/functions

@@ -10,6 +10,13 @@ function test_mgr() {
     return 1
 }
 
+function initconfluentscriptstmp() {
+    if [ -z "$confluentscripttmpdir" ]; then
+        mkdir -p /opt/confluent/tmpexec
+        confluentscripttmpdir=$(mktemp -d /opt/confluent/tmpexec/confluentscripts.XXXXXXXXX)
+    fi
+}
+
 function confluentpython() {
     if [ -x /usr/libexec/platform-python ]; then
         /usr/libexec/platform-python $*
@@ -72,7 +79,8 @@ fetch_remote() {
 }
 
 source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -86,7 +94,8 @@ source_remote_parts() {
 }
 
 run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -105,10 +114,7 @@ source_remote() {
     echo
     echo '---------------------------------------------------------------------------'
     echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
-        unsettmpdir=1
-    fi
+    initconfluentscriptstmp
     echo Sourcing from $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -135,9 +141,9 @@ run_remote() {
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
         unsettmpdir=1
     fi
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -170,7 +176,8 @@ run_remote_python() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)

confluent_osdeploy/el9-diskless/profiles/default/scripts/firstboot.sh

@@ -31,8 +31,10 @@ done
 if [ ! -f /etc/confluent/firstboot.ran ]; then
     touch /etc/confluent/firstboot.ran
 
-    cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
 
+    cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+    confluentpython /root/confignet
+    rm /root/confignet
     run_remote firstboot.custom
     # Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
     run_remote_parts firstboot.d

confluent_osdeploy/el9-diskless/profiles/default/scripts/functions

@@ -10,6 +10,13 @@ function test_mgr() {
     return 1
 }
 
+function initconfluentscriptstmp() {
+    if [ -z "$confluentscripttmpdir" ]; then
+        mkdir -p /opt/confluent/tmpexec
+        confluentscripttmpdir=$(mktemp -d /opt/confluent/tmpexec/confluentscripts.XXXXXXXXX)
+    fi
+}
+
 function confluentpython() {
     if [ -x /usr/libexec/platform-python ]; then
         /usr/libexec/platform-python $*
@@ -72,7 +79,8 @@ fetch_remote() {
 }
 
 source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -86,7 +94,8 @@ source_remote_parts() {
 }
 
 run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -105,10 +114,7 @@ source_remote() {
     echo
     echo '---------------------------------------------------------------------------'
     echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
-        unsettmpdir=1
-    fi
+    initconfluentscriptstmp
     echo Sourcing from $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -135,9 +141,9 @@ run_remote() {
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
         unsettmpdir=1
     fi
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -170,7 +176,8 @@ run_remote_python() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)

confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py

@@ -70,9 +70,9 @@ def get_image_metadata(imgpath):
                 yield md
         else:
             # plausible filesystem structure to apply to a nominally "diskless" image
-            yield {'mount': '/', 'filesystem': 'xfs', 'minsize': 39513563136, 'initsize': 954128662528, 'flags': 'rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota', 'device': '/dev/mapper/root', 'compressed_size': 27022069760}
-            yield {'mount': '/boot', 'filesystem': 'xfs', 'minsize': 232316928, 'initsize': 1006632960, 'flags': 'rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota', 'device': '/dev/nvme1n1p2', 'compressed_size': 171462656}
-            yield {'mount': '/boot/efi', 'filesystem': 'vfat', 'minsize': 7835648, 'initsize': 627900416, 'flags': 'rw,relatime,fmask=0077,dmask=0077,codepage=437,iocharset=ascii,shortname=winnt,errors=remount-ro', 'device': '/dev/nvme1n1p1', 'compressed_size': 1576960}
+            yield {'mount': '/', 'filesystem': 'xfs', 'minsize': 4294967296, 'initsize': 954128662528, 'flags': 'rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota', 'device': '/dev/mapper/root', 'compressed_size': 27022069760}
+            yield {'mount': '/boot', 'filesystem': 'xfs', 'minsize': 536870912, 'initsize': 1006632960, 'flags': 'rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota', 'device': '/dev/nvme1n1p2', 'compressed_size': 171462656}
+            yield {'mount': '/boot/efi', 'filesystem': 'vfat', 'minsize': 33554432, 'initsize': 627900416, 'flags': 'rw,relatime,fmask=0077,dmask=0077,codepage=437,iocharset=ascii,shortname=winnt,errors=remount-ro', 'device': '/dev/nvme1n1p1', 'compressed_size': 1576960}
             #raise Exception('Installation from single part image not supported')
 
 class PartedRunner():
@@ -170,6 +170,15 @@ def fixup(rootdir, vols):
     grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub')
     if not os.path.exists(grubsyscfg):
         grubsyscfg = os.path.join(rootdir, 'etc/default/grub')
+    currcmdline = []
+    with open('/proc/cmdline') as cmdlinein:
+        cmdline = cmdlinein.read().strip()
+        for arg in cmdline.split():
+            if arg.startswith('console='):
+                currcmdline.append(arg)
+            elif arg == 'quiet':
+                currcmdline.append(arg)
+    currcmdlinestr = ' '.join(currcmdline)
     kcmdline = os.path.join(rootdir, 'etc/kernel/cmdline')
     if os.path.exists(kcmdline):
         with open(kcmdline) as kcmdlinein:
@@ -181,8 +190,10 @@ def fixup(rootdir, vols):
             elif ent.startswith('root='):
                 newkcmdlineent.append('root={}'.format(newrootdev))
             elif ent.startswith('rd.lvm.lv='):
-                ent = convert_lv(ent)
-                if ent:
+                nent = convert_lv(ent)
+                if nent:
+                    newkcmdlineent.append(ent)
+                else:
                     newkcmdlineent.append(ent)
             else:
                 newkcmdlineent.append(ent)
@@ -204,8 +215,10 @@ def fixup(rootdir, vols):
                     elif cfgpart.startswith('resume='):
                         newcfgparts.append('resume={}'.format(newswapdev))
                     elif cfgpart.startswith('rd.lvm.lv='):
-                        cfgpart = convert_lv(cfgpart)
-                        if cfgpart:
+                        ncfgpart = convert_lv(cfgpart)
+                        if ncfgpart:
+                            newcfgparts.append(ncfgpart)
+                        else:
                             newcfgparts.append(cfgpart)
                     else:
                         newcfgparts.append(cfgpart)
@@ -221,13 +234,13 @@ def fixup(rootdir, vols):
         'GRUB_DISABLE_SUBMENU=true',
         'GRUB_TERMINAL=""',
         'GRUB_SERIAL_COMMAND=""',
-        'GRUB_CMDLINE_LINUX="crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M rd.lvm.lv=vg/root rd.lvm.lv=vg/swap"',
+        'GRUB_CMDLINE_LINUX="{}crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M rd.lvm.lv=vg/root rd.lvm.lv=vg/swap"'.format(currcmdlinestr),
         'GRUB_DISABLE_RECOVERY="true"',
         'GRUB_ENABLE_BLSCFG=true',
         ]
     if not os.path.exists(os.path.join(rootdir, "etc/kernel/cmdline")):
         with open(os.path.join(rootdir, "etc/kernel/cmdline"), "w") as cmdlineout:
-            cmdlineout.write("root=/dev/mapper/localstorage-root rd.lvm.lv=localstorage/root")
+            cmdlineout.write("{} root=/dev/mapper/localstorage-root rd.lvm.lv=localstorage/root".format(currcmdlinestr))
     with open(grubsyscfg, 'w') as defgrubout:
         for gline in defgrub:
             gline = gline.split()
@@ -238,11 +251,11 @@ def fixup(rootdir, vols):
                 elif ent.startswith('root='):
                     newline.append('root={}'.format(newrootdev))
                 elif ent.startswith('rd.lvm.lv='):
-                    ent = convert_lv(ent)
-                    if ent:
+                    nent = convert_lv(ent)
+                    if nent:
+                        newline.append(nent)
+                    else:
                         newline.append(ent)
-                    elif '""' in ent:
-                        newline.append('""')
                 else:
                     newline.append(ent)
             defgrubout.write(' '.join(newline) + '\n')
@@ -305,8 +318,8 @@ def fixup(rootdir, vols):
     for vol in vols:
         if vol['mount'] == '/boot/efi':
             targdev = vol['targetdisk']
-            partnum = re.search('(\d+)$', targdev).group(1)
-            targblock = re.search('(.*)\d+$', targdev).group(1)
+            partnum = re.search(r'(\d+)$', targdev).group(1)
+            targblock = re.search(r'(.*)\d+$', targdev).group(1)
     if targblock:
         if targblock.endswith('p') and 'nvme' in targblock:
             targblock = targblock[:-1]
@@ -390,7 +403,7 @@ def install_to_disk(imgpath):
             deflvmsize += fs['initsize']
             minlvmsize += fs['minsize']
         else:
-            plainvols[int(re.search('(\d+)$', fs['device'])[0])] = fs
+            plainvols[int(re.search(r'(\d+)$', fs['device'])[0])] = fs
         if fs['initsize'] > biggestsize:
             biggestfs = fs
             biggestsize = fs['initsize']
@@ -454,7 +467,7 @@ def install_to_disk(imgpath):
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', vgname, lvmpart])
         vgroupmap = {}
-        if yaml and vgmap:
+        if yaml and vgmap and os.path.exists('/tmp/volumegroupmap.yml'):
             with open('/tmp/volumegroupmap.yml') as mapin:
                 vgroupmap = yaml.safe_load(mapin)
         donedisks = {}
@@ -588,7 +601,13 @@ def install_to_disk(imgpath):
 
 
 
-            subprocess.check_call(['umount', '/run/imginst/targ'])
+            while True:
+                try:
+                    subprocess.check_call(['umount', '/run/imginst/targ'])
+                    break
+                except subprocess.CalledProcessError:
+                    print("Failed to unmount /run/imginst/targ, retrying")
+                    time.sleep(1)
         for vol in allvols:
             subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
         fixup('/run/imginst/targ', allvols)

confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh

@@ -3,8 +3,8 @@ confluent_whost=$confluent_mgr
 if [[ "$confluent_whost" == *:* ]] && [[ "$confluent_whost" != "["* ]]; then
     confluent_whost="[$confluent_mgr]"
 fi
-mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
-if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
+mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay /sysroot
+if [ "untethered" = "$(getarg confluent_imagemethod)" -o "uncompressed" = "$(getarg confluent_imagemethod)" ]; then
     mount -t tmpfs untethered /mnt/remoteimg
     curl https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
 else
@@ -40,20 +40,53 @@ fi
 
 
 #mount -t tmpfs overlay /mnt/overlay
-modprobe zram
-memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
-memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
-echo $memtot > /sys/block/zram0/disksize
-mkfs.xfs /dev/zram0 > /dev/null
-mount -o discard /dev/zram0 /mnt/overlay
-if [ ! -f /tmp/mountparts.sh ]; then
-    mkdir -p /mnt/overlay/upper /mnt/overlay/work
-    mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
+if [ ! "uncompressed" = "$(getarg confluent_imagemethod)" ]; then
+    modprobe zram
+    memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
+    memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
+    echo $memtot > /sys/block/zram0/disksize
+    mkfs.xfs /dev/zram0 > /dev/null
+fi
+TETHERED=0
+if [ "untethered" = "$(getarg confluent_imagemethod)" -o "uncompressed" = "$(getarg confluent_imagemethod)" ]; then
+    if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
+        mount -o discard /dev/zram0 /sysroot
+    else
+        mount -t tmpfs disklessroot /sysroot
+    fi
+    echo -en "Decrypting and extracting root filesystem: 0%\r"
+    srcsz=$(du -sk /mnt/remote | awk '{print $1}')
+    while [ -f /mnt/remoteimg/rootimg.sfs ]; do
+        dstsz=$(du -sk /sysroot | awk '{print $1}')
+        pct=$((dstsz * 100 / srcsz))
+        if [ $pct -gt 99 ]; then
+            pct=99
+        fi
+        echo -en "Decrypting and extracting root filesystem: $pct%\r"
+        sleep 0.25
+    done &
+    cp -ax /mnt/remote/* /sysroot/
+    umount /mnt/remote
+    if [ -e /dev/mapper/cryptimg ]; then
+        dmsetup remove cryptimg
+    fi
+    losetup -d $loopdev
+    rm /mnt/remoteimg/rootimg.sfs
+    umount /mnt/remoteimg
+    wait
+    echo -e "Decrypting and extracting root filesystem: 100%"
 else
-    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
-        mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
-        mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
-    done
+    TETHERED=1
+    mount -o discard /dev/zram0 /mnt/overlay
+    if [ ! -f /tmp/mountparts.sh ]; then
+        mkdir -p /mnt/overlay/upper /mnt/overlay/work
+        mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
+    else
+        for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
+            mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
+            mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
+        done
+    fi
 fi
 mkdir -p /sysroot/etc/ssh
 mkdir -p /sysroot/etc/confluent
@@ -109,8 +142,10 @@ echo '    EnableSSHKeysign yes' >> $sshconf
 echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
 curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_whost/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
 cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
-chmod 640 /sysroot/etc/ssh/*_key
-chroot /sysroot chgrp ssh_keys /etc/ssh/*_key
+if grep ^ssh_keys: /etc/group > /dev/null; then
+    chmod 640 /sysroot/etc/ssh/*_key
+    chroot /sysroot chgrp ssh_keys /etc/ssh/*_key
+fi
 cp /tls/*.pem /sysroot/etc/pki/ca-trust/source/anchors/
 chroot /sysroot/ update-ca-trust
 curl -sf https://$confluent_whost/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
@@ -129,10 +164,37 @@ mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
 mv /lib/firmware /lib/firmware-ramfs
 ln -s /sysroot/lib/firmware /lib/firmware
+rm -f /sysroot/etc/dracut.conf.d/diskless.conf # remove diskless dracut from runtime, to make kdump happier
 kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
+if grep debugssh /proc/cmdline >& /dev/null; then
+    debugssh=1
+else
+    debugssh=0
+fi
+if [ $TETHERED -eq 1 ]; then
+    # In tethered mode, the double-caching is useful to get through tricky part of
+    # onboot with confignet. After that, it's excessive cache usage.
+    # Give the onboot script a hook to have us come in and enable directio to the
+    # squashfs and drop the cache of the rootimg so far
+    (
+        sleep 86400 &
+        ONBOOTPID=$!
+        mkdir -p /run/confluent
+        echo $ONBOOTPID > /run/confluent/onboot_sleep.pid
+        wait $ONBOOTPID
+        dd if=/mnt/remoteimg/rootimg.sfs iflag=nocache count=0 >& /dev/null
+        if [ $debugssh -eq 0 ]; then
+            rm -rf /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs /lib/firmware-ramfs /usr/lib64/libcrypto.so* /usr/lib64/systemd/ /kernel/ /usr/bin/ /usr/sbin/ /usr/libexec/
+        fi
+    ) &
+    while [ ! -f /run/confluent/onboot_sleep.pid ]; do
+        sleep 0.1
+    done
+elif [ $debugssh -eq 0 ]; then
+    rm -rf /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs /lib/firmware-ramfs /usr/lib64/libcrypto.so* /usr/lib64/systemd/ /kernel/ /usr/bin/ /usr/sbin/ /usr/libexec/
+fi
 if grep debugssh /proc/cmdline >& /dev/null; then
     exec /opt/confluent/bin/start_root
 else
-    rm -rf /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs /lib/firmware-ramfs /usr/lib64/libcrypto.so* /usr/lib64/systemd/ /kernel/ /usr/bin/ /usr/sbin/ /usr/libexec/
     exec /opt/confluent/bin/start_root -s  # share mount namespace, keep kernel callbacks intact
 fi

confluent_osdeploy/el9-diskless/profiles/default/scripts/installimage

@@ -5,8 +5,12 @@
 # and existing mounts of image (to take advantage of caching)
 mount -o bind /sys /sysroot/sys
 mount -o bind /dev /sysroot/dev
+mount -o bind /dev/pts /sysroot/dev/pts
 mount -o bind /proc /sysroot/proc
 mount -o bind /run /sysroot/run
+mount -t efivarfs none /sysroot/sys/firmware/efi/efivars
+
+
 
 
 if [ ! -f /tmp/mountparts.sh ]; then
@@ -21,8 +25,16 @@ else
     done
 fi
 cd /sysroot/run
+[ -f /run/sshd.pid ] && 
+cp /run/sshd.pid /tmp/dbgssh.pid
+chmod 0600 /sysroot/etc/ssh/ssh*key
+chroot /sysroot/ bash -c "/usr/sbin/sshd"
 chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_python getinstalldisk"
 chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_parts pre.d"
+for nameserver in $(sed -n '/^nameservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //'); do
+    echo "nameserver $nameserver" >> /sysroot/etc/resolv.conf
+done
+
 if [ ! -f /sysroot/tmp/installdisk ]; then
     echo 'Unable to find a suitable installation target device, ssh to port 2222 to investigate'
     while [ ! -f /sysroot/tmp/installdisk ]; do
@@ -40,7 +52,8 @@ chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python imag
 echo "Port 22" >> /etc/ssh/sshd_config
 echo 'Match LocalPort 22' >> /etc/ssh/sshd_config
 echo '    ChrootDirectory /sysroot/run/imginst/targ' >> /etc/ssh/sshd_config
-kill -HUP $(cat /run/sshd.pid)
+kill $(cat /sysroot/var/run/sshd.pid)
+[ -f /tmp/dbgssh.pid ] && kill -HUP $(cat /tmp/dbgssh.pid)
 cp /sysroot/etc/pki/ca-trust/source/anchors/* /sysroot/run/imginst/targ/etc/pki/ca-trust/source/anchors/
 chroot /sysroot/run/imginst/targ update-ca-trust
 

confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh

@@ -53,7 +53,7 @@ rpm --import /etc/pki/rpm-gpg/*
 
 run_remote_python add_local_repositories
 run_remote_python syncfileclient
-run_remote_python confignet
+run_remote_python confignet -c $confluent_mgr
 
 run_remote onboot.custom
 # onboot scripts may be placed into onboot.d, e.g. onboot.d/01-firstaction.sh, onboot.d/02-secondaction.sh
@@ -62,5 +62,15 @@ run_remote_parts onboot.d
 # Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/
 run_remote_config onboot.d
 
+if [ -f /run/confluent/onboot_sleep.pid ]; then
+    loopdev=$(losetup -j /mnt/remoteimg/rootimg.sfs|cut -d: -f 1)
+    if [ -n "$loopdev" ]; then
+        losetup "$loopdev" --direct-io=on
+    fi
+    sleeppid=$(cat /run/confluent/onboot_sleep.pid)
+    kill "$sleeppid"
+    rm -f /run/confluent/onboot_sleep.pid
+fi
+
 #curl -X POST -d 'status: booted' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
 kill $logshowpid

confluent_osdeploy/el9-diskless/profiles/default/scripts/post.sh

@@ -43,7 +43,8 @@ run_remote_parts post.d
 
 # Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/
 run_remote_config post.d
-
+cd /root/
+fetch_remote confignet
 # rebuild initrd, pick up new drivers if needed
 dracut -f /boot/initramfs-$(uname -r).img $(uname -r)
 

confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel

@@ -62,8 +62,8 @@ if [ -e /tmp/confluentident/cnflnt.yml ]; then
         fi
         v4nm=$(grep ipv4_netmask: $tcfg)
         v4nm=${v4nm#ipv4_netmask: }
-        localcli network ip interface ipv4 set -i vmk0 -I $v4addr -N $v4nm -g $v4gw -t static
-        localcli network ip route ipv4 add -n default -g $v4gw
+        while ! localcli network ip interface ipv4 set -i vmk0 -I $v4addr -N $v4nm -g $v4gw -t static; do echo "Retrying..."; sleep 5; done
+        while ! localcli network ip route ipv4 add -n default -g $v4gw; do sleep 1; done
     fi
     hmackeyfile=$(mktemp)
     echo -n $(grep ^apitoken: /tmp/confluentident/cnflnt.yml|awk '{print $2}') > $hmackeyfile
@@ -73,6 +73,20 @@ if [ -e /tmp/confluentident/cnflnt.yml ]; then
     hmacfile=$(mktemp)
     ln -s /opt/confluent/bin/clortho /opt/confluent/bin/genpasshmac
     /opt/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    echo -n 'Checking connectivity to server: '
+    maxwait=30
+    while ! /opt/confluent/bin/apiclient -c >& /dev/null; do
+        echo -n '.'
+        sleep 1
+        maxwait=$((maxwait - 1))
+        if [ $maxwait -le 0 ]; then
+            echo "Unable to contact deployment server, verify network connectivity"
+            echo "A debug session has been made available on Alt-F1"
+            sleep 30
+            maxwait=30
+	fi
+    done
+    echo
     echo -n 'Registering new API key with deployment server: '
     /opt/confluent/bin/apiclient -p $hmacfile /confluent-api/self/registerapikey $passcrypt
     echo

confluent_osdeploy/esxi7/profiles/hypervisor/kickstart

@@ -1,6 +1,5 @@
 accepteula
-clearpart --firstdisk --overwritevmfs
-install --firstdisk --overwritevmfs
+%include /tmp/storagecfg
 %include /tmp/ksnet
 %include /tmp/rootpw
 reboot

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/getinstalldisk

@@ -0,0 +1,148 @@
+#!/usr/bin/python3
+import subprocess
+import os
+
+class SilentException(Exception):
+    pass
+
+class DiskInfo(object):
+    def __init__(self, devname, devinfo):
+        self.name = devname
+        self.path = '/dev/' + devname
+        self.wwn = None
+        self.model = devinfo.get('model', 'Unknown')
+        self.driver = devinfo.get('adapter_driver', 'Unknown')
+        self.size = devinfo.get('size', 0)  # in MiB
+        if not devinfo.get('is_local', False):
+            raise SilentException("Not local")
+        if devinfo.get('is_removable', False):
+            raise SilentException("Removable")
+        if devinfo.get('is_usb', False):
+            raise SilentException("USB device")
+        if devinfo.get('type', '').lower() in ('cd-rom',):
+            raise SilentException("CD-ROM device")
+        if self.size < 2048:
+            raise SilentException("Too small")
+
+        
+
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+            return 0
+        if self.driver == 'vmw_ahci':
+            return 2
+        if self.driver == 'nvme_pcie':
+            return 3
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+def list_disks():
+    current_dev = None
+    disks = {}
+    devlist = subprocess.check_output(['localcli', 'storage', 'core', 'device', 'list'])
+    if not isinstance(devlist, str):
+        devlist = devlist.decode('utf8')
+    devbyadp = {}
+    for line in devlist.split('\n'):
+        if not line.strip():
+            continue
+        if not line.startswith(' '):
+            current_dev = line.rsplit(':', 1)[0]
+            if current_dev not in disks:
+                disks[current_dev] = {}
+        elif current_dev:
+            if '   Model:' in line:
+                disks[current_dev]['model'] = ' '.join(line.split()[1:])
+            elif '   Driver:' in line:
+                disks[current_dev]['driver'] = ' '.join(line.split()[1:])
+            elif '   Is Local:' in line:
+                disks[current_dev]['is_local'] = ' '.join(line.split()[2:]).lower() == 'true'
+            elif '   Is Removable:' in line:
+                disks[current_dev]['is_removable'] = ' '.join(line.split()[2:]).lower() == 'true'
+            elif '   Size:' in line:  # in MiB
+                disks[current_dev]['size'] = int(line.split()[1])
+            elif '   Is SSD:' in line:
+                disks[current_dev]['is_ssd'] = ' '.join(line.split()[2:]).lower() == 'true'
+            elif '   Is USB:' in line:
+                disks[current_dev]['is_usb'] = ' '.join(line.split()[2:]).lower() == 'true'
+            elif '   Is Removable:' in line:
+                disks[current_dev]['is_removable'] = ' '.join(line.split()[2:]).lower() == 'true'
+            elif 'Device Type:' in line:
+                disks[current_dev]['type'] = ' '.join(line.split()[2:])
+    for dev in disks:
+        pathlist = subprocess.check_output(['localcli', 'storage', 'core', 'path', 'list', '--device', dev])
+        if not isinstance(pathlist, str):
+            pathlist = pathlist.decode('utf8')
+        for line in pathlist.split('\n'):
+            if not line.strip():
+                continue
+            if not line.startswith(' '):
+                continue
+            if '   Adapter Identifier:' in line:
+                adpname = ' '.join(line.split()[2:])
+                disks[dev]['adapter_id'] = adpname
+            elif '   Adapter:' in line:
+                adp = ' '.join(line.split()[1:])
+                disks[dev]['adapter'] = adp
+                devbyadp.setdefault(adp, []).append(dev)
+    adapterlist = subprocess.check_output(['localcli', 'storage', 'core', 'adapter', 'list'])
+    if not isinstance(adapterlist, str):
+        adapterlist = adapterlist.decode('utf8')
+    driverbyadp = {}
+    linenum = 0
+    for line in adapterlist.split('\n'):
+        linenum += 1
+        if not line.strip():
+            continue
+        if linenum < 3:
+            continue
+        parts = line.split()
+        if len(parts) < 2:
+            continue
+        adp = parts[0]
+        driver = parts[1]
+        driverbyadp[adp] = driver
+    for adp in devbyadp:
+        driver = driverbyadp.get(adp, 'Unknown')
+        for dev in devbyadp[adp]:
+            disks[dev]['adapter_driver'] = driver
+    return disks
+        
+def main():
+    disks = []
+    try:
+        alldisks = list_disks()
+    except Exception as e:
+        print("Error listing disks: {0}".format(str(e)))
+        alldisks = {}
+    for disk in alldisks:
+        try:
+            disks.append(DiskInfo(disk, alldisks[disk]))
+        except SilentException:
+            pass
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: [x.priority, x.size])]
+    if nd:
+        with open('/tmp/storagecfg', 'w') as sc:
+            sc.write(f'clearpart --all --drives={nd[0]} --overwritevmfs\n')
+            sc.write(f'install --drive={nd[0]} --overwritevmfs\n')
+    else:
+        with open('/tmp/storagecfg', 'w') as sc:
+            sc.write(f'clearpart --firstdisk --overwritevmfs\n')
+            sc.write(f'install --firstdisk --overwritevmfs\n')
+
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/makeksnet

@@ -45,7 +45,10 @@ try:
         cfg['ipv4_gateway'] = ncfg['ipv4_gateway']
 except Exception:
     pass
-netline = 'network --hostname={0} --bootproto={1}'.format(nodename, cfg['ipv4_method'])
+if cfg['ipv4_method'] == 'static':
+    netline = 'network --hostname={0} --bootproto={1}'.format(nodename, cfg['ipv4_method'])
+else:
+    netline = 'network --bootproto=dhcp'
 if vmnic:
     netline += ' --device={0}'.format(vmnic)
 if cfg['ipv4_method'] == 'static':

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/modinstall

@@ -1,9 +1,12 @@
 #!/bin/sh
 mv /etc/confluent/confluent.deploycfg /etc/confluent/confluent.newdeploycfg
 /opt/confluent/bin/apiclient /confluent-public/os/$profile/scripts/makeksnet >> /tmp/makeksnet
+/opt/confluent/bin/apiclient /confluent-public/os/$profile/scripts/getinstalldisk >> /tmp/getinstalldisk
 mv /etc/confluent/confluent.newdeploycfg /etc/confluent/confluent.deploycfg
 chmod +x /tmp/makeksnet
 /tmp/makeksnet > /tmp/ksnet
+localcli system hostname set --host $node
+python3 /tmp/getinstalldisk
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg|sed -e 's/^rootpassword: //')
 echo rootpw --iscrypted $rootpw > /tmp/rootpw
 export BOOT_CMDLINE=ks=/etc/confluent/ks.cfg

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -56,7 +56,7 @@ cat /tls/*.pem > /etc/pki/tls/certs/ca-bundle.crt
 TRIES=0
 touch /etc/confluent/confluent.info
 TRIES=5
-echo -n "Waitiing for disks..."
+echo -n "Waiting for disks..."
 while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
     sleep 1
     TRIES=$((TRIES - 1))

confluent_osdeploy/suse15-diskless/profiles/default/scripts/functions

@@ -10,6 +10,13 @@ function test_mgr() {
     return 1
 }
 
+function initconfluentscriptstmp() {
+    if [ -z "$confluentscripttmpdir" ]; then
+        mkdir -p /opt/confluent/tmpexec
+        confluentscripttmpdir=$(mktemp -d /opt/confluent/tmpexec/confluentscripts.XXXXXXXXX)
+    fi
+}
+
 function confluentpython() {
     if [ -x /usr/libexec/platform-python ]; then
         /usr/libexec/platform-python $*
@@ -72,7 +79,8 @@ fetch_remote() {
 }
 
 source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -86,7 +94,8 @@ source_remote_parts() {
 }
 
 run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -105,10 +114,7 @@ source_remote() {
     echo
     echo '---------------------------------------------------------------------------'
     echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
-        unsettmpdir=1
-    fi
+    initconfluentscriptstmp
     echo Sourcing from $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -135,9 +141,9 @@ run_remote() {
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
         unsettmpdir=1
     fi
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -170,7 +176,8 @@ run_remote_python() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)

confluent_osdeploy/suse15/profiles/hpc/scripts/functions

@@ -10,6 +10,13 @@ function test_mgr() {
     return 1
 }
 
+function initconfluentscriptstmp() {
+    if [ -z "$confluentscripttmpdir" ]; then
+        mkdir -p /opt/confluent/tmpexec
+        confluentscripttmpdir=$(mktemp -d /opt/confluent/tmpexec/confluentscripts.XXXXXXXXX)
+    fi
+}
+
 function confluentpython() {
     if [ -x /usr/libexec/platform-python ]; then
         /usr/libexec/platform-python $*
@@ -72,7 +79,8 @@ fetch_remote() {
 }
 
 source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -86,7 +94,8 @@ source_remote_parts() {
 }
 
 run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -105,10 +114,7 @@ source_remote() {
     echo
     echo '---------------------------------------------------------------------------'
     echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
-        unsettmpdir=1
-    fi
+    initconfluentscriptstmp
     echo Sourcing from $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -135,9 +141,9 @@ run_remote() {
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
         unsettmpdir=1
     fi
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -170,7 +176,8 @@ run_remote_python() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)

confluent_osdeploy/suse15/profiles/server/scripts/functions

@@ -10,6 +10,13 @@ function test_mgr() {
     return 1
 }
 
+function initconfluentscriptstmp() {
+    if [ -z "$confluentscripttmpdir" ]; then
+        mkdir -p /opt/confluent/tmpexec
+        confluentscripttmpdir=$(mktemp -d /opt/confluent/tmpexec/confluentscripts.XXXXXXXXX)
+    fi
+}
+
 function confluentpython() {
     if [ -x /usr/libexec/platform-python ]; then
         /usr/libexec/platform-python $*
@@ -72,7 +79,8 @@ fetch_remote() {
 }
 
 source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -86,7 +94,8 @@ source_remote_parts() {
 }
 
 run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -105,10 +114,7 @@ source_remote() {
     echo
     echo '---------------------------------------------------------------------------'
     echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
-        unsettmpdir=1
-    fi
+    initconfluentscriptstmp
     echo Sourcing from $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -135,9 +141,9 @@ run_remote() {
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
         unsettmpdir=1
     fi
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -170,7 +176,8 @@ run_remote_python() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/functions

@@ -10,6 +10,13 @@ function test_mgr() {
     return 1
 }
 
+function initconfluentscriptstmp() {
+    if [ -z "$confluentscripttmpdir" ]; then
+        mkdir -p /opt/confluent/tmpexec
+        confluentscripttmpdir=$(mktemp -d /opt/confluent/tmpexec/confluentscripts.XXXXXXXXX)
+    fi
+}
+
 function confluentpython() {
     if [ -x /usr/libexec/platform-python ]; then
         /usr/libexec/platform-python $*
@@ -72,7 +79,8 @@ fetch_remote() {
 }
 
 source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -86,7 +94,8 @@ source_remote_parts() {
 }
 
 run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -105,10 +114,7 @@ source_remote() {
     echo
     echo '---------------------------------------------------------------------------'
     echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
-        unsettmpdir=1
-    fi
+    initconfluentscriptstmp
     echo Sourcing from $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -135,9 +141,9 @@ run_remote() {
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
         unsettmpdir=1
     fi
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -170,7 +176,8 @@ run_remote_python() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)

confluent_osdeploy/ubuntu20.04/initramfs/scripts/init-premount/confluent

@@ -52,13 +52,40 @@ while ! grep NODENAME /custom-installation/confluent/confluent.info; do
             ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
             echo $NIC > /tmp/autodetectnic
         else
-            configure_networking
-            for dsrv in $deploysrvs; do
-                if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
-                    deploysrvs=$dsrv
-                    break
+            rmmod cdc_ether 2> /dev/null
+            while [ ! -f /run/confirmednic ]; do
+                ALLNETDEVS=$(ip a|grep LOWER_UP|grep MULTICAST|awk '{print $2}'|sed -e s/://)
+                rm -rf /run/net* /run/dhcpcd /var/lib/dhcpcd
+                for dev in $(ip a|grep MULTICAST|awk '{print $2}'|sed -e s/://); do
+                   ip a flush $dev
+                   echo 1 > /proc/sys/net/ipv6/conf/$dev/addr_gen_mode
+                   echo 0 > /proc/sys/net/ipv6/conf/$dev/addr_gen_mode
+                done
+                unset DEVICE DEVICE6 IP IP6 dev
+                [ -z "$1" ] || DEVICE=$1
+                shift
+                configure_networking
+                echo $DEVICE > /tmp/autodetectnic
+                for dsrv in $deploysrvs; do
+                    (if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                        echo $dsrv > /run/confirmednic
+                        break
+                    fi) &
+                    chkpid=$!
+                    ( sleep 10 && kill $chkpid ) &
+                    timeoutpid=$!
+                    wait $chkpid
+                    kill $timeoutpid 2> /dev/null
+                    unset chkpid timeoutpid
+                done
+                if [ ! -f /run/confirmednic ]; then
+                    echo "No connectivity to deployment servers, retrying..."
+                    [ -z "$1" ] && set -- $ALLNETDEVS
                 fi
             done
+            deploysrvs=$(cat /run/confirmednic)
+            rm /run/confirmednic
+
         fi
         MGR=$deploysrvs
         NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/functions

@@ -10,6 +10,13 @@ function test_mgr() {
     return 1
 }
 
+function initconfluentscriptstmp() {
+    if [ -z "$confluentscripttmpdir" ]; then
+        mkdir -p /opt/confluent/tmpexec
+        confluentscripttmpdir=$(mktemp -d /opt/confluent/tmpexec/confluentscripts.XXXXXXXXX)
+    fi
+}
+
 function confluentpython() {
     if [ -x /usr/libexec/platform-python ]; then
         /usr/libexec/platform-python $*
@@ -72,7 +79,8 @@ fetch_remote() {
 }
 
 source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -86,7 +94,8 @@ source_remote_parts() {
 }
 
 run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -105,10 +114,7 @@ source_remote() {
     echo
     echo '---------------------------------------------------------------------------'
     echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
-        unsettmpdir=1
-    fi
+    initconfluentscriptstmp
     echo Sourcing from $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -135,9 +141,9 @@ run_remote() {
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
         unsettmpdir=1
     fi
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -170,7 +176,8 @@ run_remote_python() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)

confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent

@@ -53,13 +53,40 @@ while ! grep NODENAME /custom-installation/confluent/confluent.info; do
             ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
             echo $NIC > /tmp/autodetectnic
         else
-            configure_networking
-            for dsrv in $deploysrvs; do
-                if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
-                    deploysrvs=$dsrv
-                    break
+	    rmmod cdc_ether 2> /dev/null
+            while [ ! -f /run/confirmednic ]; do
+                ALLNETDEVS=$(ip a|grep LOWER_UP|grep MULTICAST|awk '{print $2}'|sed -e s/://)
+
+                rm -rf /run/net* /run/dhcpcd /var/lib/dhcpcd
+                for dev in $(ip a|grep MULTICAST|awk '{print $2}'|sed -e s/://); do
+                   ip a flush $dev
+                   echo 1 > /proc/sys/net/ipv6/conf/$dev/addr_gen_mode
+                   echo 0 > /proc/sys/net/ipv6/conf/$dev/addr_gen_mode
+                done
+                unset DEVICE DEVICE6 IP IP6 dev
+        		[ -z "$1" ] || DEVICE=$1
+		        shift
+                configure_networking
+                echo $DEVICE > /tmp/autodetectnic
+                for dsrv in $deploysrvs; do
+                    (if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                        echo $dsrv > /run/confirmednic
+                        break
+                    fi) &
+		            chkpid=$!
+		            ( sleep 10 && kill $chkpid ) &
+		            timeoutpid=$!
+		            wait $chkpid
+		            kill $timeoutpid 2> /dev/null
+		            unset chkpid timeoutpid
+                done
+                if [ ! -f /run/confirmednic ]; then
+                    echo "No connectivity to deployment servers, retrying..."
+		            [ -z "$1" ] && set -- $ALLNETDEVS
                 fi
             done
+            deploysrvs=$(cat /run/confirmednic)
+            rm /run/confirmednic
         fi
         MGR=$deploysrvs
         NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')

confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh

@@ -1,16 +1,16 @@
 #!/bin/bash
 set -e
 sed -i 's/label: ubuntu/label: Ubuntu/' $2/profile.yaml
-if [ -e $1/casper/hwe-vmlinuz ]; then
-    ln -s $1/casper/hwe-vmlinuz $2/boot/kernel
-else
+#if [ -e $1/casper/hwe-vmlinuz ]; then
+#    ln -s $1/casper/hwe-vmlinuz $2/boot/kernel
+#else
     ln -s $1/casper/vmlinuz $2/boot/kernel
-fi
-if [ -e $1/casper/hwe-initrd ]; then
-    ln -s $1/casper/hwe-initrd $2/boot/initramfs/distribution
-else
+#fi
+#if [ -e $1/casper/hwe-initrd ]; then
+#    ln -s $1/casper/hwe-initrd $2/boot/initramfs/distribution
+#else
     ln -s $1/casper/initrd $2/boot/initramfs/distribution
-fi
+#fi
 mkdir -p $2/boot/efi/boot
 if [ -d $1/EFI/boot/ ]; then
     ln -s $1/EFI/boot/* $2/boot/efi/boot

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/functions

@@ -10,6 +10,13 @@ function test_mgr() {
     return 1
 }
 
+function initconfluentscriptstmp() {
+    if [ -z "$confluentscripttmpdir" ]; then
+        mkdir -p /opt/confluent/tmpexec
+        confluentscripttmpdir=$(mktemp -d /opt/confluent/tmpexec/confluentscripts.XXXXXXXXX)
+    fi
+}
+
 function confluentpython() {
     if [ -x /usr/libexec/platform-python ]; then
         /usr/libexec/platform-python $*
@@ -72,7 +79,8 @@ fetch_remote() {
 }
 
 source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -86,7 +94,8 @@ source_remote_parts() {
 }
 
 run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     apiclient=/opt/confluent/bin/apiclient
     if [ -f /etc/confluent/apiclient ]; then
         apiclient=/etc/confluent/apiclient
@@ -105,10 +114,7 @@ source_remote() {
     echo
     echo '---------------------------------------------------------------------------'
     echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
-        unsettmpdir=1
-    fi
+    initconfluentscriptstmp
     echo Sourcing from $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -135,9 +141,9 @@ run_remote() {
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
         unsettmpdir=1
     fi
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     fetch_remote $1
@@ -170,7 +176,8 @@ run_remote_python() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    unset confluentscripttmpdir
+    initconfluentscriptstmp
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)

confluent_osdeploy/utils/autocons.c

@@ -7,6 +7,7 @@
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
+#include <dirent.h>
 
 #define COM1 0x3f8
 #define COM2 0x2f8
@@ -19,6 +20,137 @@
 #define SPEED57600 6
 #define SPEED115200 7
 
+typedef struct {
+    char devnode[32];
+    speed_t speed;
+    int valid;
+} serial_port_t;
+
+serial_port_t process_spcr() {
+    serial_port_t result = {0};
+    char buff[128];
+    int fd;
+    uint64_t address;
+    int currspeed;
+    
+    result.valid = 0;
+    
+    fd = open("/sys/firmware/acpi/tables/SPCR", O_RDONLY);
+    if (fd < 0) {
+        return result;
+    }
+    
+    if (read(fd, buff, 80) < 80) {
+        close(fd);
+        return result;
+    }
+    close(fd);
+    
+    if (buff[8] != 2) return result; // revision 2
+    if (buff[36] != 0) return result; // 16550 only
+    if (buff[40] != 1) return result; // IO only
+    
+    address = *(uint64_t *)(buff + 44);
+    currspeed = buff[58];
+    
+    if (address == COM1) {
+        strncpy(result.devnode, "/dev/ttyS0", sizeof(result.devnode));
+    } else if (address == COM2) {
+        strncpy(result.devnode, "/dev/ttyS1", sizeof(result.devnode));
+    } else if (address == COM3) {
+        strncpy(result.devnode, "/dev/ttyS2", sizeof(result.devnode));
+    } else if (address == COM4) {
+        strncpy(result.devnode, "/dev/ttyS3", sizeof(result.devnode));
+    } else {
+        return result;
+    }
+    
+    if (currspeed == SPEED9600) {
+        result.speed = B9600;
+    } else if (currspeed == SPEED19200) {
+        result.speed = B19200;
+    } else if (currspeed == SPEED57600) {
+        result.speed = B57600;
+    } else if (currspeed == SPEED115200) {
+        result.speed = B115200;
+    } else {
+        return result;
+    }
+    
+    result.valid = 1;
+    return result;
+}
+
+serial_port_t identify_by_sys_vendor() {
+    serial_port_t result = {0};
+    char buff[128];
+    FILE *f;
+    
+    f = fopen("/sys/devices/virtual/dmi/id/sys_vendor", "r");
+    if (f) {
+        if (fgets(buff, sizeof(buff), f)) {
+            if (strstr(buff, "Supermicro")) {
+                strncpy(result.devnode, "/dev/ttyS1", sizeof(result.devnode));
+                result.speed = B115200;
+                result.valid = 1;
+            }
+        }
+        fclose(f);
+    }
+    return result;
+}
+
+serial_port_t search_serial_ports() {
+    serial_port_t result = {0};
+    DIR *dir;
+    struct dirent *entry;
+    int fd;
+    int status;
+    int numfound= 0;
+    int numpossible = 0;
+
+    dir = opendir("/dev");
+    if (!dir) {
+        return result;
+    }
+    
+    while ((entry = readdir(dir)) != NULL) {
+        if (strncmp(entry->d_name, "ttyS", 4) != 0) {
+            continue;
+        }
+        
+        char devpath[64];
+        snprintf(devpath, sizeof(devpath), "/dev/%s", entry->d_name);
+        
+        fd = open(devpath, O_RDWR | O_NOCTTY | O_NONBLOCK);
+        if (fd < 0) {
+            continue;
+        }
+        
+        if (ioctl(fd, TIOCMGET, &status) == 0) {
+            numpossible++;
+            if (numfound < 1) {
+                strncpy(result.devnode, devpath, sizeof(result.devnode));
+                result.speed = B115200;
+            }
+            if (status & TIOCM_CAR) {
+                strncpy(result.devnode, devpath, sizeof(result.devnode));
+                numfound++;
+                result.speed = B115200;
+                
+            }
+        }
+        
+        close(fd);
+    }
+    
+    closedir(dir);
+    if (numfound == 1 || numpossible == 1) {
+        result.valid = 1;
+    }
+    return result;
+}
+
 int main(int argc, char* argv[]) {
     struct termios tty;
     struct termios tty2;
@@ -36,46 +168,38 @@ int main(int argc, char* argv[]) {
     char* offset;
     uint64_t address;
     bufflen = 0;
-    tmpi = open("/sys/firmware/acpi/tables/SPCR", O_RDONLY);
-    if (tmpi < 0) {
-        exit(0);
-    }
-    if (read(tmpi, buff, 80) < 80) {
-        exit(0);
-    }
-    close(tmpi);
-    if (buff[8] != 2) exit(0); //revision 2
-    if (buff[36] != 0) exit(0); //16550 only
-    if (buff[40] != 1) exit(0); //IO only
-    address = *(uint64_t *)(buff + 44);
-    currspeed = buff[58];
-    offset = buff + 10;
-    if (address == COM1) {
-        strncpy(buff, "/dev/ttyS0", 128);
-    } else if (address == COM2) {
-        strncpy(buff, "/dev/ttyS1", 128);
-    } else if (address == COM3) {
-        strncpy(buff, "/dev/ttyS2", 128);
-    } else if (address == COM4) {
-        strncpy(buff, "/dev/ttyS3", 128);
-    } else {
+    #ifndef __x86_64__
+        // Only x86 needs autoconsole, other platforms have reasonable default serial console
+        exit(0);
+    #endif
+    serial_port_t spcr = process_spcr();
+    if (!spcr.valid) {
+        spcr = search_serial_ports();
+    }
+    if (!spcr.valid) {
+        spcr = identify_by_sys_vendor();
+    }
+    if (!spcr.valid) {
         exit(0);
     }
+    strncpy(buff, spcr.devnode, sizeof(buff));
+    offset = strchr(buff, 0);
+    currspeed = spcr.speed;
     ttyf = open(buff, O_RDWR | O_NOCTTY);
     if (ttyf < 0) {
         fprintf(stderr, "Unable to open tty\n");
         exit(1);
     }
-    if (currspeed == SPEED9600) {
+    if (currspeed == B9600) {
         cspeed = B9600;
         strncpy(offset, ",9600", 6);
-    } else if (currspeed == SPEED19200) {
+    } else if (currspeed == B19200) {
         cspeed = B19200;
         strncpy(offset, ",19200", 7);
-    } else if (currspeed == SPEED57600) {
+    } else if (currspeed == B57600) {
         cspeed = B57600;
         strncpy(offset, ",57600", 7);
-    } else if (currspeed == SPEED115200) {
+    } else if (currspeed == B115200) {
         cspeed = B115200;
         strncpy(offset, ",115200", 8);
     } else {

confluent_osdeploy/utils/confusebox/apiclient.go

@@ -2,20 +2,23 @@ package main
 
 import (
 	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
 	"fmt"
 	"io"
-	"os"
+	"net"
 	"net/http"
-	"crypto/x509"
-	"crypto/tls"
+	"os"
 	"strings"
-	"errors"
+	"time"
 )
+
 type ApiClient struct {
-	server string
+	server    string
 	urlserver string
-	apikey string
-	nodename string
+	apikey    string
+	nodename  string
 	webclient *http.Client
 }
 
@@ -24,7 +27,7 @@ func NewApiClient(cafile string, keyfile string, nodename string, server string)
 	if err != nil {
 		return nil, err
 	}
-	cacerts  := x509.NewCertPool()
+	cacerts := x509.NewCertPool()
 	cacerts.AppendCertsFromPEM(currcacerts)
 	apikey := []byte("")
 	if keyfile != "" {
@@ -32,7 +35,7 @@ func NewApiClient(cafile string, keyfile string, nodename string, server string)
 		if err != nil {
 			return nil, err
 		}
-		if apikey[len(apikey) - 1] == 0xa {
+		if apikey[len(apikey)-1] == 0xa {
 			apikey = apikey[:len(apikey)-1]
 		}
 	}
@@ -40,7 +43,9 @@ func NewApiClient(cafile string, keyfile string, nodename string, server string)
 		cinfo, err := os.ReadFile("/etc/confluent/confliuent.info")
 		if err != nil {
 			nodename, err = os.Hostname()
-			if err != nil { return nil, err }
+			if err != nil {
+				return nil, err
+			}
 		}
 		cinfolines := bytes.Split(cinfo, []byte("\n"))
 		if bytes.Contains(cinfolines[0], []byte("NODENAME")) {
@@ -48,6 +53,20 @@ func NewApiClient(cafile string, keyfile string, nodename string, server string)
 			nodename = string(cnodebytes[0])
 		}
 	}
+	// Test connectivity with up to 3 retries
+	var conn net.Conn
+	for i := 0; i < 3; i++ {
+		conn, err = net.Dial("tcp", net.JoinHostPort(server, "443"))
+		if err == nil {
+			conn.Close()
+			break
+		}
+		time.Sleep(5 * time.Second)
+		fmt.Print("Connection attempt failed, retrying...\n")
+		if i == 2 {
+			return nil, fmt.Errorf("failed to connect after 3 attempts: %v", err)
+		}
+	}
 	urlserver := server
 	if strings.Contains(server, ":") {
 		if strings.Contains(server, "%") && !strings.Contains(server, "%25") {
@@ -58,10 +77,11 @@ func NewApiClient(cafile string, keyfile string, nodename string, server string)
 			server = server[:strings.Index(server, "%")]
 		}
 	}
+
 	webclient := &http.Client{
 		Transport: &http.Transport{
 			TLSClientConfig: &tls.Config{
-				RootCAs: cacerts,
+				RootCAs:    cacerts,
 				ServerName: server,
 			},
 		},
@@ -70,34 +90,42 @@ func NewApiClient(cafile string, keyfile string, nodename string, server string)
 	return &vc, nil
 }
 
-func (apiclient *ApiClient) RegisterKey(crypted string, hmac string) (error) {
+func (apiclient *ApiClient) RegisterKey(crypted string, hmac string) error {
 	cryptbytes := []byte(crypted)
 	cryptbuffer := bytes.NewBuffer(cryptbytes)
 	_, err := apiclient.request("/confluent-api/self/registerapikey", "", cryptbuffer, "", hmac)
 	return err
 }
 
-func (apiclient *ApiClient) Fetch(url string, outputfile string, mime string, body io.Reader) (error)  {
+func (apiclient *ApiClient) Fetch(url string, outputfile string, mime string, body io.Reader) error {
 	outp, err := os.Create(outputfile)
-	if err != nil { return err }
+	if err != nil {
+		return err
+	}
 	defer outp.Close()
 	rsp, err := apiclient.request(url, mime, body, "", "")
-	if err != nil { return err }
+	if err != nil {
+		return err
+	}
 	_, err = io.Copy(outp, rsp)
 	return err
 }
 
-func (apiclient *ApiClient) GrabText(url string, mime string, body io.Reader) (string, error){
+func (apiclient *ApiClient) GrabText(url string, mime string, body io.Reader) (string, error) {
 	rsp, err := apiclient.request(url, mime, body, "", "")
-	if err != nil { return "", err }
+	if err != nil {
+		return "", err
+	}
 	rspdata, err := io.ReadAll(rsp)
-	if err != nil { return "", err }
+	if err != nil {
+		return "", err
+	}
 	rsptxt := string(rspdata)
 	return rsptxt, nil
 }
 
 func (apiclient *ApiClient) request(url string, mime string, body io.Reader, method string, hmac string) (io.ReadCloser, error) {
-	if ! strings.Contains(url, "https://") {
+	if !strings.Contains(url, "https://") {
 		url = fmt.Sprintf("https://%s%s", apiclient.urlserver, url)
 	}
 	if method == "" {
@@ -114,8 +142,12 @@ func (apiclient *ApiClient) request(url string, mime string, body io.Reader, met
 	} else {
 		rq, err = http.NewRequest(method, url, body)
 	}
-	if err != nil { return nil, err }
-	if (mime != "") { rq.Header.Set("Accept", mime) }
+	if err != nil {
+		return nil, err
+	}
+	if mime != "" {
+		rq.Header.Set("Accept", mime)
+	}
 	rq.Header.Set("CONFLUENT_NODENAME", apiclient.nodename)
 	if len(hmac) > 0 {
 		rq.Header.Set("CONFLUENT_CRYPTHMAC", hmac)
@@ -124,11 +156,12 @@ func (apiclient *ApiClient) request(url string, mime string, body io.Reader, met
 		rq.Header.Set("CONFLUENT_APIKEY", apiclient.apikey)
 	}
 	rsp, err := apiclient.webclient.Do(rq)
-	if err != nil { return nil, err }
+	if err != nil {
+		return nil, err
+	}
 	if rsp.StatusCode >= 300 {
 		err = errors.New(rsp.Status)
 		return nil, err
 	}
 	return rsp.Body, err
 }
-

confluent_server/bin/collective

@@ -1,6 +1,7 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 
 import argparse
+import asyncio
 import errno
 import os
 import pwd
@@ -14,9 +15,9 @@ if path.startswith('/opt'):
     # if installed into system path, do not muck with things
     sys.path.append(path)
 
-import confluent.client as client
+import confluent.asynclient as client
 import confluent.sortutil as sortutil
-import confluent.tlvdata as tlvdata
+import confluent.asynctlvdata as tlvdata
 
 try:
     input = raw_input
@@ -52,40 +53,47 @@ def make_certificate():
     os.umask(umask)
 
 
-def show_invitation(name, nonvoting=False):
+async def show_invitation(name, nonvoting=False):
     if not os.path.exists('/etc/confluent/srvcert.pem'):
         make_certificate()
-    s = client.Command().connection
+    clicmd = client.Command()
+    await clicmd.ensure_connected()
+    s = clicmd.connection
     role = 'nonvoting' if nonvoting else None
-    tlvdata.send(s, {'collective': {'operation': 'invite', 'name': name, 'role': role}})
-    invite = tlvdata.recv(s)['collective']
+    await tlvdata.send(s, {'collective': {'operation': 'invite', 'name': name, 'role': role}})
+    invite = await tlvdata.recv(s)
+    invite = invite['collective']
     if 'error' in invite:
         sys.stderr.write(invite['error'] + '\n')
         return
     print('{0}'.format(invite['invitation']))
 
 
-def join_collective(server, invitation):
+async def join_collective(server, invitation):
     if not os.path.exists('/etc/confluent/srvcert.pem'):
         make_certificate()
-    s = client.Command().connection
+    clicmd = client.Command()
+    await clicmd.ensure_connected()
+    s = clicmd.connection
     while not invitation:
         invitation = input('Paste the invitation here: ')
-    tlvdata.send(s, {'collective': {'operation': 'join',
+    await tlvdata.send(s, {'collective': {'operation': 'join',
                                     'invitation': invitation,
                                     'server': server}})
-    res = tlvdata.recv(s)
+    res = await tlvdata.recv(s)
     res = res.get('collective',
                   {'status': 'Unknown response: ' + repr(res)})
     print(res.get('status', res.get('error', repr(res))))
     if 'error' in res:
         sys.exit(1)
 
-def delete_member(name):
-    s = client.Command().connection
-    tlvdata.send(s, {'collective': {'operation': 'delete',
+async def delete_member(name):
+    clicmd = client.Command()
+    await clicmd.ensure_connected()
+    s = clicmd.connection
+    await tlvdata.send(s, {'collective': {'operation': 'delete',
                                     'member': name}})
-    res = tlvdata.recv(s)
+    res = await tlvdata.recv(s)
     res = res.get('collective',
                   {'status': 'Unknown response: ' + repr(res)})
     print(res.get('status', res.get('error', repr(res))))
@@ -93,10 +101,12 @@ def delete_member(name):
         sys.exit(1)
 
 
-def show_collective():
-    s = client.Command().connection
-    tlvdata.send(s, {'collective': {'operation': 'show'}})
-    res = tlvdata.recv(s)
+async def show_collective():
+    clicmd = client.Command()
+    await clicmd.ensure_connected()
+    s = clicmd.connection
+    await tlvdata.send(s, {'collective': {'operation': 'show'}})
+    res = await tlvdata.recv(s)
     if 'error' in res:
         print(res['error'])
         return
@@ -121,7 +131,7 @@ def show_collective():
     else:
         print('Run collective show on leader for more data')
 
-def main():
+async def main():
     a = argparse.ArgumentParser(description='Confluent server utility')
     sp = a.add_subparsers(dest='command')
     gc = sp.add_parser('gencert', help='Generate Confluent Certificates for '
@@ -143,13 +153,13 @@ def main():
     if cmdset.command == 'gencert':
         make_certificate()
     elif cmdset.command == 'invite':
-        show_invitation(cmdset.name, cmdset.n)
+        await show_invitation(cmdset.name, cmdset.n)
     elif cmdset.command == 'join':
-        join_collective(cmdset.server, cmdset.i)
+        await join_collective(cmdset.server, cmdset.i)
     elif cmdset.command == 'show':
-        show_collective()
+        await show_collective()
     elif cmdset.command == 'delete':
-        delete_member(cmdset.name)
+        await delete_member(cmdset.name)
 
 if __name__ == '__main__':
-    main()
+    asyncio.run(main())

confluent_server/bin/confluent

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
@@ -17,6 +17,9 @@
 
 import sys
 import os
+
+
+
 path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
@@ -30,9 +33,15 @@ import confluent.main
 #p.enable()
 #try:
 import multiprocessing
-if __name__ == '__main__':
-    multiprocessing.freeze_support()
+
+def main():
     confluent.main.run(sys.argv)
+if __name__ == '__main__':
+    #multiprocessing.freeze_support()
+    #asyncio.get_event_loop().run_until_complete(main())
+    main()
+    #gt = spawn_for_awaitable(confluent.main.run(sys.argv))
+    #gt.wait()
 #except:
 #   pass
 #p.disable()

confluent_server/bin/confluent_selfcheck

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 import argparse
+import asyncio
 import os
 import socket
 import glob
@@ -16,16 +17,23 @@ import confluent.certutil as certutil
 import confluent.client as client
 import confluent.config.configmanager as configmanager
 import confluent.netutil as netutil
-import eventlet.green.subprocess as subprocess
 import tempfile
 import shutil
-import eventlet.green.socket as socket
-import eventlet
-import greenlet
 import pwd
 import signal
 import confluent.collective.manager as collective
 import confluent.noderange as noderange
+import subprocess
+
+def check_sysctl_tuning():
+    with open('/proc/sys/net/ipv4/tcp_sack', 'r') as f:
+        value = f.read().strip()
+        if value == '1':
+            print('OK')
+            return
+        else:
+            emprint('TCP SACK is disabled, network operations to BMCs may be particularly impacted, including firmware updates and virtual media')
+
 
 def check_neigh_overflow():
     dmesgout = subprocess.check_output(['dmesg'])
@@ -66,7 +74,7 @@ def webserver_listening():
         return False
 
 
-def certificates_missing_ips(conn):
+async def certificates_missing_ips(conn):
     # check if the tls can verify by the right CAs, then further
     # check if all ip addresses are in the certificate offered
     ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
@@ -76,9 +84,8 @@ def certificates_missing_ips(conn):
     sock = ctx.wrap_socket(conn)
     crt = sock.getpeercert()
     sans = crt.get('subjectAltName', [])
-    ips = certutil.get_ip_addresses()
     missing_ips = []
-    for ip in ips:
+    async for ip in certutil.get_ip_addresses():
         for san in sans:
             field, val = san
             if val[-1] == '\n':
@@ -113,7 +120,6 @@ def web_api_works():
 
 def nics_missing_ipv6():
     # check for ability to create AF_INET6, for kernel disabled ipv6
-    a = socket.socket(socket.AF_INET6)
     ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
     missingnics = []
     for line in ipaddrs:
@@ -162,15 +168,69 @@ def uuid_matches():
     dbuuid = configmanager.get_global('confluent_uuid')
     return dbuuid == fsuuid
 
-def lookup_node(node):
+async def lookup_node(node):
     try:
-        return socket.getaddrinfo(node, 0)
-    except greenlet.GreenletExit:
-        return None
+        cloop = asyncio.get_event_loop()
+        return await cloop.getaddrinfo(node, 0)
     except Exception:
         return None
+    
+async def check_ssh_to_node(targsships):
+    sshutil.ready_keys = {}
+    sshutil.agent_pid = None
+    cuser = pwd.getpwnam('confluent')
+    os.setgid(cuser.pw_gid)
+    os.setuid(cuser.pw_uid)
+    await sshutil.prep_ssh_key('/etc/confluent/ssh/automation')
+    for targ in targsships:
+        srun = subprocess.run(
+            ['ssh', '-Tn', '-o', 'BatchMode=yes', '-l', 'root',
+            '-o', 'StrictHostKeyChecking=yes', targ, 'true'],
+            stdin=subprocess.DEVNULL, stderr=subprocess.PIPE)
+        if srun.returncode == 0:
+            print(f'Confluent automation access to {targ} seems OK')
+        else:
+            if b'Host key verification failed' in srun.stderr:
+                emprint(f'Confluent ssh unable to verify host key for {targ}, check /etc/ssh/ssh_known_hosts. (Example resolution: osdeploy initialize -k)')
+            elif b'ermission denied' in srun.stderr:
+                emprint(f'Confluent user unable to ssh in to {targ}, check /root/.ssh/authorized_keys on the target system versus /etc/confluent/ssh/automation.pub (Example resolution: osdeploy initialize -a)')
+            else:
+                emprint('Unknown error attempting confluent automation ssh:')
+                sys.stderr.buffer.write(srun.stderr)
+    if sshutil.agent_pid:
+        os.kill(int(sshutil.agent_pid), signal.SIGTERM)
+    sys.exit(0)
 
-if __name__ == '__main__':
+async def check_confluent_ssh():
+    sshutil.ready_keys = {}
+    sshutil.agent_pid = None
+    cuser = pwd.getpwnam('confluent')
+    os.setgid(cuser.pw_gid)
+    os.setuid(cuser.pw_uid)
+    fprint('Checking SSH Certificate authority: ')
+    try:
+        await sshutil.prep_ssh_key('/etc/confluent/ssh/ca')
+        print('OK')
+    except Exception as e:
+        if type(e).__name__ == 'CalledProcessError' and 'UNPROTECTED' in e.stderr.decode():
+            emprint('Permissions incorrect on /etc/confluent/ssh/ca (Example resolution: chmod 600 /etc/confluent/ssh/ca)')
+        else:
+            emprint('Failed to load SSH authority key, deployed servers will not have host certificates for known_hosts and users may be unable to ssh between nodes without a password (Example resolution: osdeploy initialize -s)')
+    fprint('Checking confluent SSH automation key: ')
+    try:
+        await sshutil.prep_ssh_key('/etc/confluent/ssh/automation')
+        print('OK')
+    except Exception as e:
+        if type(e).__name__ == 'CalledProcessError' and 'UNPROTECTED' in e.stderr.decode():
+            emprint('Permissions incorrect on /etc/confluent/ssh/automation (Example resolution: chmod 600 /etc/confluent/ssh/automation)')
+        else:
+            emprint('Failed to load confluent automation key, syncfiles and profile ansible plays will not work (Example resolution: osdeploy initialize -a)')
+    if sshutil.agent_pid:
+        os.kill(int(sshutil.agent_pid), signal.SIGTERM)
+    sys.exit(0)
+
+
+async def main():
     ap = argparse.ArgumentParser(description='Run configuration checks for a system running confluent service')
     ap.add_argument('-n', '--node', help='A node name to run node specific checks against')
     ap.add_argument('-a', '--automation', help='Do checks against a deployed node for automation and syncfiles function', action='store_true')
@@ -193,7 +253,7 @@ if __name__ == '__main__':
         if conn:
             print('Running')
             fprint('Web Certificate: ')
-            cert = certificates_missing_ips(conn)
+            cert = await certificates_missing_ips(conn)
             if cert:
                 cert = ', '.join(cert)
                 emprint('Addresses missing from certificate: {0} (Example resolution: osdeploy initialize -t)'.format(cert))
@@ -216,6 +276,8 @@ if __name__ == '__main__':
             emprint('ARP/Neighbor table problem detected, evaluate and increase net.ipv*.neigh.default.gc_thresh*')
         else:
             print('OK')
+        fprint('Checking sysctl tunables: ')
+        check_sysctl_tuning()
         fprint('TFTP Status: ')
         if tftp_works():
             print('OK')
@@ -245,35 +307,8 @@ if __name__ == '__main__':
                 emprint('No matching public key found for root user (Example resolution: osdeploy initialize -u)')
         else:
             emprint('No trusted ssh keys for root user, passwordless SSH from managers to nodes may not work (Example resolution: osdeploy initialize -u)')
-        if sshutil.sshver() > 7.6:
-            child = os.fork()
-            if child > 0:
-                pid, extcode = os.waitpid(child, 0)
-            else:
-                sshutil.ready_keys = {}
-                sshutil.agent_pid = None
-                cuser = pwd.getpwnam('confluent')
-                os.setgid(cuser.pw_gid)
-                os.setuid(cuser.pw_uid)
-                fprint('Checking SSH Certificate authority: ')
-                try:
-                    sshutil.prep_ssh_key('/etc/confluent/ssh/ca')
-                    print('OK')
-                except Exception as e:
-                    if type(e).__name__ == 'CalledProcessError' and 'UNPROTECTED' in e.stderr.decode():
-                        emprint('Permissions incorrect on /etc/confluent/ssh/ca (Example resolution: chmod 600 /etc/confluent/ssh/ca)')
-                    else:
-                        emprint('Failed to load SSH authority key, deployed servers will not have host certificates for known_hosts and users may be unable to ssh between nodes without a password (Example resolution: osdeploy initialize -s)')
-                fprint('Checking confluent SSH automation key: ')
-                try:
-                    sshutil.prep_ssh_key('/etc/confluent/ssh/automation')
-                    print('OK')
-                except Exception as e:
-                    if type(e).__name__ == 'CalledProcessError' and 'UNPROTECTED' in e.stderr.decode():
-                        emprint('Permissions incorrect on /etc/confluent/ssh/automation (Example resolution: chmod 600 /etc/confluent/ssh/automation)')
-                    else:
-                        emprint('Failed to load confluent automation key, syncfiles and profile ansible plays will not work (Example resolution: osdeploy initialize -a)')
-                sys.exit(0)
+        if await sshutil.sshver() > 7.6:
+            subprocess.run([sys.executable, __file__, '--check-ssh'])
         fprint('Checking for blocked insecure boot: ')
         if insecure_boot_attempts():
             emprint('Some nodes are attempting network boot using PXE or HTTP boot, but the node is not configured to allow this (Example resolution: nodegroupattrib everything deployment.useinsecureprotocols=firmware)')
@@ -357,7 +392,7 @@ if __name__ == '__main__':
         for nic in glob.glob("/sys/class/net/*/ifindex"):
             idx = int(open(nic, "r").read())
             nicname = nic.split('/')[-2]
-            ncfg = netutil.get_nic_config(cfg, args.node, ifidx=idx)
+            ncfg = await netutil.get_nic_config(cfg, args.node, ifidx=idx)
             if ncfg['ipv4_address']:
                 targsships.append(ncfg['ipv4_address'])
             if ncfg['ipv4_address'] or ncfg['ipv4_method'] == 'dhcp':
@@ -378,53 +413,30 @@ if __name__ == '__main__':
         if allok:
             print(f'No issues detected with attributes of {args.node}')
         fprint("Checking name resolution: ")
-        lk = eventlet.spawn(lookup_node, args.node)
-        eventlet.sleep(0.1)
-        tries = 5
-        while not lk.dead and tries > 0:
-            eventlet.sleep(1)
-            tries -= 1
         deaddns = False
-        if not tries:
+        try:
+            result = await asyncio.wait_for(lookup_node(args.node), timeout=5)
+        except asyncio.exceptions.TimeoutError:
             emprint('Name resolution takes too long, check state of /etc/resolv.conf and indicated nameservers, this can produce failure to netboot or failure to commence installation')
-            lk.kill()
             deaddns = True
-        result = lk.wait()
         if not result and not deaddns:
             emprint('Name resolution failed for node, it is normally a good idea for the node name to resolve to an IP')
         if result:
             print("OK")
         if args.automation:
             print(f'Checking confluent automation access to {args.node}...')
-            child = os.fork()
-            if child > 0:
-                pid, extcode = os.waitpid(child, 0)
-            else:
-                sshutil.ready_keys = {}
-                sshutil.agent_pid = None
-                cuser = pwd.getpwnam('confluent')
-                os.setgid(cuser.pw_gid)
-                os.setuid(cuser.pw_uid)
-                sshutil.prep_ssh_key('/etc/confluent/ssh/automation')
-                for targ in targsships:
-                    srun = subprocess.run(
-                        ['ssh', '-Tn', '-o', 'BatchMode=yes', '-l', 'root',
-                        '-o', 'StrictHostKeyChecking=yes', targ, 'true'],
-                        stdin=subprocess.DEVNULL, stderr=subprocess.PIPE)
-                    if srun.returncode == 0:
-                        print(f'Confluent automation access to {targ} seems OK')
-                    else:
-                        if b'Host key verification failed' in srun.stderr:
-                            emprint(f'Confluent ssh unable to verify host key for {targ}, check /etc/ssh/ssh_known_hosts. (Example resolution: osdeploy initialize -k)')
-                        elif b'ermission denied' in srun.stderr:
-                            emprint(f'Confluent user unable to ssh in to {targ}, check /root/.ssh/authorized_keys on the target system versus /etc/confluent/ssh/automation.pub (Example resolution: osdeploy initialize -a)')
-                        else:
-                            emprint('Unknown error attempting confluent automation ssh:')
-                            sys.stderr.buffer.write(srun.stderr)
-                os.kill(int(sshutil.agent_pid), signal.SIGTERM)
+            subprocess.run([sys.executable, __file__, '--check-ssh'] + targsships)
     else:
         print("Skipping node checks, no node specified (Example: confluent_selfcheck -n n1)")
 # possible checks:
 # arping on the node, check for dupes/against nodeinventory?
 # arping -D for mgt own ip addresses? check for dupes, also check for bleed through from one nic to another
 # iterate through profiles, use mtools to extract site initramfs, check if outdated
+if __name__ == '__main__':
+    if len(sys.argv) >= 2 and sys.argv[1] == '--check-ssh':
+        if len(sys.argv) >= 3:
+            asyncio.run(check_ssh_to_node(sys.argv[2:]))
+        else:
+            asyncio.run(check_confluent_ssh())
+    else:
+        asyncio.run(main())

confluent_server/bin/confluentdbutil

@@ -16,6 +16,7 @@
 # limitations under the License.
 
 
+import asyncio
 import getpass
 import optparse
 import sys
@@ -79,12 +80,13 @@ if args[0] in ('restore', 'merge'):
         # Use the format parameter based on the --yaml option
         format = 'yaml' if options.yaml else 'json'
         
-        cfm.restore_db_from_directory(
+        dp = cfm.restore_db_from_directory(
             dumpdir, password,
             merge="skip" if args[0] == 'merge' else False, 
             skipped=skipped,
             format=format)
             
+        asyncio.get_event_loop().run_until_complete(dp)
         if skipped['nodes']:
             skippedn = ','.join(skipped['nodes'])
             print('The following nodes were skipped during merge: '
@@ -93,7 +95,6 @@ if args[0] in ('restore', 'merge'):
             skippedn = ','.join(skipped['nodegroups'])
             print('The following node groups were skipped during merge: '
                   '{}'.format(skippedn))
-
         cfm.statelessmode = False
         cfm.ConfigManager.wait_for_sync(True)
         if owner != 0:
@@ -126,8 +127,8 @@ elif args[0] == 'dump':
     
     # Use the format parameter based on the --yaml option
     format = 'yaml' if options.yaml else 'json'
-    cfm.dump_db_to_directory(dumpdir, password, options.redact,
-                             options.skipkeys, format=format)
-
+    dp = cfm.dump_db_to_directory(dumpdir, password, options.redact,
+                                  options.skipkeys, format=format)
+    asyncio.get_event_loop().run_until_complete(dp)
 
 

confluent_server/bin/osdeploy

@@ -3,6 +3,7 @@
 __author__ = 'jjohnson2,bfinley'
 
 import argparse
+import asyncio
 import glob
 import os
 import os.path
@@ -16,10 +17,9 @@ path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
     sys.path.append(path)
 import confluent.collective.manager as collective
-import eventlet.green.subprocess as subprocess
 import confluent.selfservice as selfservice
 import confluent.util as util
-import confluent.client as client
+import confluent.asynclient as client
 import confluent.sshutil as sshutil
 import confluent.certutil as certutil
 import confluent.netutil as netutil
@@ -36,7 +36,7 @@ def emprint(txt):
         print(txt)
 
 fnamechars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789.^'
-def main(args):
+async def main(args):
     ap = argparse.ArgumentParser(description='Manage OS deployment resources')
     sp = ap.add_subparsers(dest='command')
     wiz = sp.add_parser('initialize', help='Do OS deployment preparation')
@@ -66,15 +66,15 @@ def main(args):
     if cmdset.command == 'list':
         return oslist()
     if cmdset.command == 'import':
-        return osimport(cmdset.imagefile, custname=cmdset.n)
+        return await osimport(cmdset.imagefile, custname=cmdset.n)
     if cmdset.command == 'importcheck':
         return osimport(cmdset.imagefile, checkonly=True)
     if cmdset.command == 'initialize':
-        return initialize(cmdset)
+        return await initialize(cmdset)
     if cmdset.command == 'updateboot':
-        return updateboot(cmdset.profile)
+        return await updateboot(cmdset.profile)
     if cmdset.command == 'rebase':
-        return rebase(cmdset.profile)
+        return await rebase(cmdset.profile)
     ap.print_help()
 
 def symlinkp(src, trg):
@@ -160,7 +160,7 @@ def init_confluent_myname():
             os._exit(0)
 
 
-def local_node_trust_setup():
+async def local_node_trust_setup():
     init_confluent_myname()
     allnodes, domain = selfservice.get_cluster_list()
     myname = collective.get_myname()
@@ -173,7 +173,7 @@ def local_node_trust_setup():
     myprincipals.add(myshortname)
     if domain:
         myprincipals.add('{0}.{1}'.format(myshortname, domain))
-    for addr in netutil.get_my_addresses():
+    for addr in await netutil.get_my_addresses():
         addr = socket.inet_ntop(addr[0], addr[1])
         myprincipals.add(addr)
     for pubkey in glob.glob('/etc/ssh/ssh_host_*_key.pub'):
@@ -186,7 +186,12 @@ def local_node_trust_setup():
         with open(certfile, 'w') as certout:
             certout.write(cert)
         if restorecon:
-            subprocess.check_call(['/usr/sbin/restorecon', certfile])
+            rcproc = await asyncio.create_subprocess_exec(
+                '/usr/sbin/restorecon', certfile)
+            rc = await rcproc.wait()
+            if rc != 0:
+                raise Exception("Failure to restorecon")
+            #subprocess.check_call(['/usr/sbin/restorecon', certfile])
     with open('/etc/ssh/sshd_config', 'r') as sshconf:
         currconfig = sshconf.read().split('\n')
     for conline in currconfig:
@@ -204,12 +209,17 @@ def local_node_trust_setup():
         for node in util.natural_sort(allnodes):
             equivout.write(node + '\n')
     if restorecon:
-        subprocess.check_call(
-            ['/usr/sbin/restorecon',
-            '/etc/ssh/shosts.equiv', '/root/.shosts'])
+        rcproc = await asyncio.create_subprocess_exec(
+            '/usr/sbin/restorecon', '/etc/ssh/shosts.equiv', '/root/.shosts')
+        rc = await rcproc.wait()
+        if rc != 0:
+            raise Exception('Unable to restorecon')
+        #subprocess.check_call(
+        #    ['/usr/sbin/restorecon',
+        #    '/etc/ssh/shosts.equiv', '/root/.shosts'])
 
 
-def install_tftp_content():
+async def install_tftp_content():
     tftplocation = None
     candidates = ('/tftpboot', '/var/lib/tftpboot', '/srv/tftpboot', '/srv/tftp')
     for cand in candidates:
@@ -224,7 +234,11 @@ def install_tftp_content():
             emprint('/tftpboot is detected as tftp directory, will not try to automatically enable tftp, as it is presumed to be externally managed')
         else:
             try:
-                subprocess.check_call(['systemctl', 'enable', 'tftp.socket', '--now'])
+                tfproc = await asyncio.create_subprocess_exec('systemctl', 'enable', 'tftp.socket', '--now')
+                rc = await tfproc.wait()
+                if rc != 0:
+                    raise Exception('{0}'.format(rc))
+                #subprocess.check_call(['systemctl', 'enable', 'tftp.socket', '--now'])
                 print('TFTP service is enabled and running')
             except Exception:
                 emprint('Unable to automatically enable and start tftp.socket, tftp server may already be running outside of systemd control')
@@ -251,7 +265,7 @@ def install_tftp_content():
 
 
 
-def initialize(cmdset):
+async def initialize(cmdset):
     if os.getuid() != 0:
         sys.stderr.write('This command must run as root user\n')
         sys.exit(1)
@@ -300,20 +314,28 @@ def initialize(cmdset):
                             'passphrase protected ssh key easier.\n')
             sys.exit(1)
         init_confluent_myname()
-        sshutil.initialize_root_key(False)
+        await sshutil.initialize_root_key(False)
     if cmdset.t:
         didsomething = True
         init_confluent_myname()
-        certutil.create_certificate()
+        await certutil.create_certificate()
         if os.path.exists('/usr/lib/systemd/system/httpd.service'):
             try:
-                subprocess.check_call(['systemctl', 'try-restart', 'httpd'])
+                hrproc = await asyncio.create_subprocess_exec('systemctl', 'try-restart', 'httpd')
+                rc = await hrproc.wait()
+                if rc != 0:
+                    raise Exception('Failed restarting HTTP')
+                #subprocess.check_call(['systemctl', 'try-restart', 'httpd'])
                 print('HTTP server has been restarted if it was running')
             except Exception:
                 emprint('New HTTPS certificates generated, restart the web server manually')
         elif os.path.exists('/usr/lib/systemd/system/apache2.service'):
             try:
-                subprocess.check_call(['systemctl', 'try-restart', 'apache2'])
+                hrproc = await asyncio.create_subprocess_exec('systemctl', 'try-restart', 'apache2')
+                rc = await hrproc.wait()
+                if rc != 0:
+                    raise Exception('Failed restarting HTTP')
+                # subprocess.check_call(['systemctl', 'try-restart', 'apache2'])
                 print('HTTP server has been restarted if it was running')
             except Exception:
                 emprint('New HTTPS certificates generated, restart the web server manually')
@@ -323,20 +345,20 @@ def initialize(cmdset):
         didsomething = True
         init_confluent_myname()
         try:
-            sshutil.initialize_ca()
+            await sshutil.initialize_ca()
         except sshutil.AlreadyExists:
             emprint('Skipping generation of SSH CA, already present and would likely be more problematic to regenerate than to reuse (if absolutely sure you want to discard old CA, then delete /etc/confluent/ssh/ca* and restart confluent)')
     if cmdset.a:
         didsomething = True
         init_confluent_myname()
         try:
-            sshutil.initialize_root_key(True, True)
+            await sshutil.initialize_root_key(True, True)
         except sshutil.AlreadyExists:
             emprint('Skipping generation of new automation key, already present and regeneration usually causes more problems. (If absolutely certain, delete /etc/confluent/ssh/automation* and restart confluent)')
     if cmdset.p:
-        install_tftp_content()
+        await install_tftp_content()
     if cmdset.l:
-        local_node_trust_setup()
+        await local_node_trust_setup()
     if cmdset.k:
         cas = set([])
         cakeys = set([])
@@ -369,7 +391,7 @@ def initialize(cmdset):
             sys.exit(rc)
     if not didsomething and (cmdset.k or cmdset.l or cmdset.g or cmdset.p):
         if cmdset.g:
-            updateboot('genesis-x86_64')
+            await updateboot('genesis-x86_64')
         sys.exit(0)
     if not didsomething:
         sys.stderr.write('Nothing was done, use initialize -i for '
@@ -384,7 +406,7 @@ def initialize(cmdset):
     totar = []
     if not os.path.exists('confluent_uuid'):
         c = client.Command()
-        for rsp in c.read('/uuid'):
+        async for rsp in c.read('/uuid'):
             uuid = rsp.get('uuid', {}).get('value', None)
             if uuid:
                 oum = os.umask(0o11)
@@ -410,15 +432,20 @@ def initialize(cmdset):
             for fname in files:
                 topack.append(os.path.join(currd, fname))
     with open(tmpname, 'wb') as initramfs:
-        packit = subprocess.Popen(['cpio', '-H', 'newc', '-o'],
-                                  stdout=initramfs, stdin=subprocess.PIPE)
+        packit = await asyncio.subprocess.create_subprocess_exec(
+            'cpio', '-H', 'newc', '-o',
+            stdin=asyncio.subprocess.PIPE, stdout=initramfs)
+        #packit = subprocess.Popen(['cpio', '-H', 'newc', '-o'],
+        #                          stdout=initramfs, stdin=subprocess.PIPE)
         for packfile in topack:
             if not isinstance(packfile, bytes):
                 packfile = packfile.encode('utf8')
             packit.stdin.write(packfile)
             packit.stdin.write(b'\n')
+        await packit.stdin.drain()
         packit.stdin.close()
-        res = packit.wait()
+        await packit.stdin.wait_closed()
+        res = await packit.wait()
         if res:
             sys.stderr.write('Error occurred while packing site initramfs')
             sys.exit(1)
@@ -434,11 +461,13 @@ def initialize(cmdset):
     finally:
         os.umask(oum)
     if cmdset.g:
-        updateboot('genesis-x86_64')
+        await updateboot('genesis-x86_64')
     if totar:
         tmptarname = tmpname.replace('cpio', 'tgz')
         tarcmd = ['tar', '-czf', tmptarname] + totar
-        subprocess.check_call(tarcmd)
+        tarproc = await asyncio.subprocess.create_subprocess_exec(*tarcmd)
+        await tarproc.wait()
+        #subprocess.check_call(tarcmd)
         os.rename(tmptarname, '/var/lib/confluent/public/site/initramfs.tgz')
         oum = os.umask(0o22)
         try:
@@ -449,17 +478,19 @@ def initialize(cmdset):
     print('Site initramfs content packed successfully')
 
     if not os.path.exists('/etc/confluent/srvcert.pem'):
-        subprocess.check_call(['collective', 'gencert'])
+        gcproc = await asyncio.subprocess.create_subprocess_exec('collective', 'gencert')
+        await gcproc.wait()
+        #subprocess.check_call(['collective', 'gencert'])
     # TODO: check selinux and segetbool for httpd_can_network_connect
     # httpd available and enabled?
 
 
-def updateboot(profilename):
+async def updateboot(profilename):
     if not os.path.exists('/var/lib/confluent/public/site/initramfs.cpio'):
         emprint('Must generate site content first (TLS (-t) and/or SSH (-s))')
         return 1
     c = client.Command()
-    for rsp in c.update('/deployment/profiles/{0}'.format(profilename),
+    async for rsp in c.update('/deployment/profiles/{0}'.format(profilename),
             {'updateboot': 1}):
         if 'updated' in rsp:
             print('Updated: {0}'.format(rsp['updated']))
@@ -467,9 +498,9 @@ def updateboot(profilename):
             print(repr(rsp))
 
 
-def rebase(profilename):
+async def rebase(profilename):
     c = client.Command()
-    for rsp in c.update('/deployment/profiles/{0}'.format(profilename), {'rebase': 1}):
+    async for rsp in c.update('/deployment/profiles/{0}'.format(profilename), {'rebase': 1}):
         if 'updated' in rsp:
             print('Updated: {0}'.format(rsp['updated']))
         elif 'customized' in rsp:
@@ -501,7 +532,7 @@ def oslist():
     print("")
 
 
-def osimport(imagefile, checkonly=False, custname=None):
+async def osimport(imagefile, checkonly=False, custname=None):
     c = client.Command()
     imagefile = os.path.abspath(imagefile)
     if c.unixdomain:
@@ -518,7 +549,7 @@ def osimport(imagefile, checkonly=False, custname=None):
     apiargs = {'filename': imagefile}
     if custname:
         apiargs['custname'] = custname
-    for rsp in c.create(apipath, apiargs):
+    async for rsp in c.create(apipath, apiargs):
         if 'target' in rsp:
             importing = True
             shortname = rsp['name']
@@ -543,7 +574,7 @@ def osimport(imagefile, checkonly=False, custname=None):
             print(repr(rsp))
     try:
         while importing:
-            for rsp in c.read('/deployment/importing/{0}'.format(shortname)):
+            async for rsp in c.read('/deployment/importing/{0}'.format(shortname)):
                 if 'progress' in rsp:
                     sys.stdout.write('{0}: {1:.2f}%    \r'.format(rsp['phase'],
                                                             rsp['progress']))
@@ -563,7 +594,8 @@ def osimport(imagefile, checkonly=False, custname=None):
                 time.sleep(0.5)
     finally:
         if shortname:
-            list(c.delete('/deployment/importing/{0}'.format(shortname)))
+            async for x in c.delete('/deployment/importing/{0}'.format(shortname)):
+                pass
 
 if __name__ == '__main__':
-    main(sys.argv)
+    asyncio.run(main(sys.argv))

confluent_server/builddeb

@@ -38,9 +38,9 @@ if [ "$OPKGNAME" = "confluent-server" ]; then
     if grep wheezy /etc/os-release; then
         sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex, python-dateutil, python-pyopenssl, python-msgpack/' debian/control
     elif grep jammy /etc/os-release; then
-	    sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi(>=1.5.71), python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil/' debian/control
+	    sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-aiohttp, python3-pyparsing, python3-pyghmi(>=1.5.71), python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil/' debian/control
     else
-	    sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi(>=1.5.71), python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil, python3-pyasyncore/' debian/control
+	    sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-aiohttp, python3-pyparsing, python3-pyghmi(>=1.5.71), python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil, python3-pyasyncore/' debian/control
     fi
     if grep wheezy /etc/os-release; then
         echo 'confluent_client python-confluent-client' >> debian/pydist-overrides

confluent_server/confluent/alerts.py

@@ -34,7 +34,7 @@ import confluent.exceptions as exc
 import confluent.lookuptools as lookuptools
 import confluent.core
 
-def decode_alert(varbinds, configmanager):
+async def decode_alert(varbinds, configmanager):
     """Decode an SNMP alert for a server
 
     Given the agentaddr, OID for the trap, and a dict of varbinds,
@@ -49,11 +49,11 @@ def decode_alert(varbinds, configmanager):
         agentaddr = varbinds['.1.3.6.1.6.3.18.1.3.0']
     except KeyError:
         agentaddr = varbinds['1.3.6.1.6.3.18.1.3.0']
-    node = lookuptools.node_by_manager(agentaddr)
+    node = await lookuptools.node_by_manager(agentaddr)
     if node is None:
         raise exc.InvalidArgumentException(
             'Unable to find a node with specified manager')
-    return confluent.core.handle_path(
+    return await confluent.core.handle_path(
         '/nodes/{0}/events/hardware/decode'.format(node), 'update',
         configmanager, varbinds, autostrip=False)
 

confluent_server/confluent/asynchttp.py

@@ -14,21 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Overall, the result of this shall be:
-# - Web clients can create the same out-of-order responsiveness as socket
-#   clients (but with more complexity on their end)
-# - Web clients can share single request among console sessions
-# - Web clients can get async notify of things like node add/remove, events
-
-# This provides an async strategy to http clients.  The design is that a http
-# session may have an 'async' resource.  In such a case, any requests are
-# queued and immediately the response is given accepting the queued request.
-# A request flags itself as queue-compatible through an HTTP header indicating
-# the identifier of the async thread.  As responses happen to the queued
-# request, data is dispatched to the first registered poller for data on
-# the session.  This way, a client may elect to provide multiple pollers
-# to mitigate general choppiness of http network pattern.  It may not be
-# worth it, but it's possible.
+# This handles ownership of asynchronous behavior driving sessions
+# with websockets.  There was a long-polling HTTP mechanism but that is removed
+# Now it's possible to have asynchronous requests multiplexed over a single websockets
+# with none of the "choppiness" inherent to multiple long-polling requests
 
 # Additionally support console session multiplexing, to mitigate needed
 # connection count.
@@ -39,16 +28,16 @@
 # Much like console sessions, these will be reaped if a client spends too
 # far away.
 
+import asyncio
 import collections
 import confluent.exceptions as exc
 import confluent.messages as messages
 import confluent.util as util
-import eventlet
-import greenlet
+import confluent.core as core
+import confluent.log as log
 import time
 
 _asyncsessions = {}
-_cleanthread = None
 _consolesessions = None
 
 
@@ -74,25 +63,14 @@ class AsyncTermRelation(object):
 
 class AsyncSession(object):
 
-    def __init__(self, wshandler=None):
+    def __init__(self, wshandler):
         self.asyncid = _assign_asyncid(self)
-        self.responses = collections.deque()
         self.wshandler = wshandler
-        self._evt = None
         self.termrelations = []
         self.consoles = set([])
-        if not wshandler:
-            self.reaper = eventlet.spawn_after(15, self.destroy)
 
-    def add(self, requestid, rsp):
-        if self.wshandler:
-            self.wshandler(messages.AsyncMessage((requestid, rsp)))
-        if self.responses is None:
-            return
-        self.responses.append((requestid, rsp))
-        if self._evt:
-            self._evt.send()
-            self._evt = None
+    async def add(self, requestid, rsp):
+        await self.wshandler(messages.AsyncMessage((requestid, rsp)))
 
     def set_term_relation(self, env):
         # need a term relation to keep track of what data belongs
@@ -107,64 +85,39 @@ class AsyncSession(object):
         self.consoles.add(sessionid)
 
     def destroy(self):
-        if self._evt:
-            self._evt.send()
-            self._evt = None
         for console in self.consoles:
             _consolesessions[console]['session'].destroy()
         self.consoles = set([])
-        self.responses = None
         del _asyncsessions[self.asyncid]
 
-    def run_handler(self, handler, requestid):
+    async def run_handler(self, handler, requestid):
         try:
-            for rsp in handler:
-                self.add(requestid, rsp)
-            self.add(requestid, messages.AsyncCompletion())
+            async for rsp in core.iterate_responses(handler):
+                await self.add(requestid, rsp)
+            await self.add(requestid, messages.AsyncCompletion())
         except Exception as e:
-            self.add(requestid, e)
+            print(repr(e))
+            log.logtrace()
+            await self.add(requestid, e)
 
-    def get_responses(self, timeout=25):
-        self.reaper.cancel()
-        self.reaper = eventlet.spawn_after(timeout + 15, self.destroy)
-        nextexpiry = time.time() + 90
-        for csess in list(self.consoles):
-            try:
-                _consolesessions[csess]['expiry'] = nextexpiry
-            except KeyError:  # session has been closed elsewhere
-                self.consoles.discard(csess)
-        if self._evt:
-            # TODO(jjohnson2): This precludes the goal of 'double barreled'
-            # access....  revisit if this could matter
-            raise Exception('get_responses is not re-entrant')
-        if not self.responses:  # wait to accumulate some
-            self._evt = eventlet.event.Event()
-            with eventlet.Timeout(timeout, False):
-                self._evt.wait()
-            self._evt = None
-        while self.responses:
-            yield self.responses.popleft()
-
-
-def run_handler(hdlr, env):
-    asyncsessid = env['HTTP_CONFLUENTASYNCID']
+async def run_handler(hdlr, req):
+    asyncsessid = req.headers['ConfluentAsyncId']
     try:
         asyncsession = _asyncsessions[asyncsessid]['asyncsession']
-        requestid = env['HTTP_CONFLUENTREQUESTID']
+        requestid = req.headers['ConfluentRequestId']
     except KeyError:
         raise exc.InvalidArgumentException(
                 'Invalid Session ID or missing request id')
-    eventlet.spawn_n(asyncsession.run_handler, hdlr, requestid)
+    cloop = asyncio.get_event_loop()
+    cloop.create_task(asyncsession.run_handler(hdlr, requestid))
     return requestid
 
 
 def get_async(env, querydict):
-    global _cleanthread
     return _asyncsessions[env['HTTP_CONFLUENTASYNCID']]['asyncsession']
 
 
 def handle_async(env, querydict, threadset, wshandler=None):
-    global _cleanthread
     # This may be one of two things, a request for a new async stream
     # or a request for next data from async stream
     # httpapi otherwise handles requests an injecting them to queue
@@ -174,25 +127,7 @@ def handle_async(env, querydict, threadset, wshandler=None):
         if wshandler:
             yield currsess
             return
-        yield messages.AsyncSession(currsess.asyncid)
-        return
-    if  querydict['asyncid'] not in _asyncsessions:
-        raise exc.InvalidArgumentException(
-                'Invalid or expired async id')
-    mythreadid = greenlet.getcurrent()
-    threadset.add(mythreadid)
-    loggedout = None
-    currsess = None
-    try:
-        currsess = _asyncsessions[querydict['asyncid']]['asyncsession']
-        for rsp in currsess.get_responses():
-            yield messages.AsyncMessage(rsp)
-    except greenlet.GreenletExit as ge:
-        loggedout = ge
-    threadset.discard(mythreadid)
-    if loggedout is not None:
-        currsess.destroy()
-        raise exc.LoggedOut()
+    raise Exception("Long polling asynchttp is discontinued")
 
 
 def set_console_sessions(consolesessions):

confluent_server/confluent/auth.py

@@ -19,13 +19,13 @@
 # the PBKDF2 transform is skipped unless a user has been idle for sufficient
 # time
 
+import asyncio
 import confluent.config.configmanager as configmanager
-import eventlet
-import eventlet.tpool
 try:
     import Cryptodome.Protocol.KDF as KDF
 except ImportError:
     import Crypto.Protocol.KDF as KDF
+from concurrent.futures import ProcessPoolExecutor
 from fnmatch import fnmatch
 import hashlib
 import hmac
@@ -33,6 +33,7 @@ import msgpack
 import multiprocessing
 import os
 import pwd
+import confluent.tasks as tasks
 import confluent.userutil as userutil
 import confluent.util as util
 pam = None
@@ -238,7 +239,7 @@ def authorize(name, element, tenant=False, operation='create',
     return False
 
 
-def check_user_passphrase(name, passphrase, operation=None, element=None, tenant=False):
+async def check_user_passphrase(name, passphrase, operation=None, element=None, tenant=False):
     """Check a a login name and passphrase for authenticity and authorization
 
     The function combines authentication and authorization into one function.
@@ -268,7 +269,7 @@ def check_user_passphrase(name, passphrase, operation=None, element=None, tenant
         # by a user, which might be malicious
         # would normally make an event and wait
         # but here there's no need for that
-        eventlet.sleep(0.5)
+        await asyncio.sleep(0.5)
     cfm = configmanager.ConfigManager(tenant, username=user)
     ucfg = cfm.get_user(user)
     if ucfg is None:
@@ -280,7 +281,7 @@ def check_user_passphrase(name, passphrase, operation=None, element=None, tenant
         except KeyError:
             pass
     if ucfg is None:
-        eventlet.sleep(0.05)
+        await asyncio.sleep(0.05)
         return None
     bpassphrase = None
     if isinstance(passphrase, dict) and len(passphrase) == 1:
@@ -304,22 +305,16 @@ def check_user_passphrase(name, passphrase, operation=None, element=None, tenant
         # throw it at the worker pool when implemented
         # maybe a distinct worker pool, wondering about starving out non-auth stuff
         salt, crypt = ucfg['cryptpass']
-        # execute inside tpool to get greenthreads to give it a special thread
-        # world
-        # TODO(jbjohnso): util function to generically offload a call
-        # such a beast could be passed into pyghmi as a way for pyghmi to
-        # magically get offload of the crypto functions without having
-        # to explicitly get into the eventlet tpool game
         global authworkers
         global authcleaner
         if authworkers is None:
-            authworkers = multiprocessing.Pool(processes=1)
+            authworkers = ProcessPoolExecutor(max_workers=1) # multiprocessing.Pool(processes=1)
         else:
             authcleaner.cancel()
-        authcleaner = eventlet.spawn_after(30, _clean_authworkers)
-        crypted = eventlet.tpool.execute(_do_pbkdf, passphrase, salt)
+        authcleaner = tasks.spawn_task_after(30, _clean_authworkers)
+        crypted = await _do_pbkdf(passphrase, salt)
         del _passchecking[(user, tenant)]
-        eventlet.sleep(
+        await asyncio.sleep(
             0.05)  # either way, we want to stall so that client can't
         # determine failure because there is a delay, valid response will
         # delay as well
@@ -332,52 +327,56 @@ def check_user_passphrase(name, passphrase, operation=None, element=None, tenant
             pwe = pwd.getpwnam(user)
         except KeyError:
             #pam won't work if the user doesn't exist, don't go further
-            eventlet.sleep(0.05)  # stall even on test for existence of a username
+            await asyncio.sleep(0.05)  # stall even on test for existence of a username
             return None
-        if os.getuid() != 0:
-            # confluent is running with reduced privilege, however, pam_unix refuses
-            # to let a non-0 user check anothers password.
-            # We will fork and the child will assume elevated privilege to
-            # get unix_chkpwd helper to enable checking /etc/shadow
-            getprompt, sendprompt = os.pipe()
-            getprompt, sendprompt = os.fdopen(getprompt, 'rb', 0), os.fdopen(sendprompt, 'wb', 0)
-            pid = os.fork()
-            if not pid:
-                usergood = False
-                try:
-                    getprompt.close()
-                    # we change to the uid we are trying to authenticate as, because
-                    # pam_unix uses unix_chkpwd which reque
-                    os.setuid(pwe.pw_uid)
-                    pa = pam.pam()
-                    usergood = pa.authenticate(user, passphrase, service=_pamservice)
-                    if (not usergood and len(pa.prompts) > 1 and
-                            (not isinstance(passphrase, dict) or
-                            (set(passphrase) - pa.prompts))):
-                        sendprompt.write(msgpack.packb(list(pa.prompts)))
-                        sendprompt.close()
-                        os._exit(2)
-                finally:
-                    os._exit(0 if usergood else 1)
-            sendprompt.close()
-            usergood = os.waitpid(pid, 0)[1]
-            if (usergood >> 8) == 2:
-                prompts = getprompt.read()
-                if (prompts):
-                    raise PromptsNeeded(msgpack.unpackb(prompts))
-            usergood = usergood == 0
-            getprompt.close()
-        else:
-            # We are running as root, we don't need to fork in order to authenticate the
-            # user
-            usergood = pam.authenticate(user, passphrase, service=_pamservice)
+        usergood = await asyncio.get_event_loop().run_in_executor(authworkers, pam_check, pwe, user, passphrase)
         if usergood:
             if bpassphrase:
                 _passcache[(user, tenant)] = hashlib.sha256(bpassphrase).digest()
             return authorize(user, element, tenant, operation, skipuserobj=False)
-    eventlet.sleep(0.05)  # stall even on test for existence of a username
+    await asyncio.sleep(0.05)  # stall even on test for existence of a username
     return None
 
+def pam_check(pwe, user, passphrase):
+    if os.getuid() != 0:
+        # confluent is running with reduced privilege, however, pam_unix refuses
+        # to let a non-0 user check anothers password.
+        # We will fork and the child will assume elevated privilege to
+        # get unix_chkpwd helper to enable checking /etc/shadow
+        getprompt, sendprompt = os.pipe()
+        getprompt, sendprompt = os.fdopen(getprompt, 'rb', 0), os.fdopen(sendprompt, 'wb', 0)
+        pid = os.fork()  # we are forking with asyncio, but we are not using async in the child so it should be fine.
+        if not pid:
+            usergood = False
+            try:
+                getprompt.close()
+                # we change to the uid we are trying to authenticate as, because
+                # pam_unix uses unix_chkpwd which reque
+                os.setuid(pwe.pw_uid)
+                pa = pam.pam()
+                usergood = pa.authenticate(user, passphrase, service=_pamservice)
+                if (not usergood and len(pa.prompts) > 1 and
+                        (not isinstance(passphrase, dict) or
+                        (set(passphrase) - pa.prompts))):
+                    sendprompt.write(msgpack.packb(list(pa.prompts)))
+                    sendprompt.close()
+                    os._exit(2)
+            finally:
+                os._exit(0 if usergood else 1)
+        sendprompt.close()
+        usergood = os.waitpid(pid, 0)[1]
+        if (usergood >> 8) == 2:
+            prompts = getprompt.read()
+            if (prompts):
+                raise PromptsNeeded(msgpack.unpackb(prompts))
+        usergood = usergood == 0
+        getprompt.close()
+    else:
+        # We are running as root, we don't need to fork in order to authenticate the
+        # user
+        usergood = pam.authenticate(user, passphrase, service=_pamservice)
+    return usergood
+
 def _apply_pbkdf(passphrase, salt):
     return KDF.PBKDF2(passphrase, salt, 32, 10000,
                       lambda p, s: hmac.new(p, s, hashlib.sha256).digest())
@@ -390,9 +389,10 @@ def _clean_authworkers():
     authcleaner = None
 
 
-def _do_pbkdf(passphrase, salt):
+async def _do_pbkdf(passphrase, salt):
     # we must get it over to the authworkers pool or else get blocked in
     # compute.  However, we do want to wait for result, so we have
     # one of the exceedingly rare sort of circumstances where 'apply'
     # actually makes sense
-    return authworkers.apply(_apply_pbkdf, [passphrase, salt])
+    res = await asyncio.get_event_loop().run_in_executor(authworkers, _apply_pbkdf, passphrase, salt)
+    return res

confluent_server/confluent/certutil.py

@@ -1,11 +1,21 @@
 import os
+if __name__ == '__main__':
+    import sys
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.realpath(os.path.join(path, '..'))
+    if path.startswith('/opt'):
+        sys.path.append(path)
 import confluent.collective.manager as collective
 import confluent.util as util
 from os.path import exists
+import datetime
 import shutil
 import socket
-import eventlet.green.subprocess as subprocess
 import tempfile
+try:
+    import cryptography.x509 as x509
+except ImportError:
+    x509 = None
 
 def mkdirp(targ):
     try:
@@ -31,8 +41,8 @@ def normalize_uid():
         raise Exception('Need to run as root or owner of /etc/confluent')
     return curruid
 
-def get_ip_addresses():
-    lines, _ = util.run(['ip', 'addr'])
+async def get_ip_addresses():
+    lines, _ = await util.check_output('ip', 'addr')
     if not isinstance(lines, str):
         lines = lines.decode('utf8')
     for line in lines.split('\n'):
@@ -143,11 +153,11 @@ def get_certificate_paths():
         tlsmateriallocation.setdefault('bundles', []).append(ngbundlepath)
     return tlsmateriallocation
 
-def assure_tls_ca():
+async def assure_tls_ca():
     keyout, certout = ('/etc/confluent/tls/cakey.pem', '/etc/confluent/tls/cacert.pem')
     if not os.path.exists(certout):
         #create_simple_ca(keyout, certout)
-        create_full_ca(certout)
+        await create_full_ca(certout)
     fname = '/var/lib/confluent/public/site/tls/{0}.pem'.format(
         collective.get_myname())
     ouid = normalize_uid()
@@ -159,8 +169,8 @@ def assure_tls_ca():
             raise
     try:
         shutil.copy2('/etc/confluent/tls/cacert.pem', fname)
-        hv, _ = util.run(
-            ['openssl', 'x509', '-in', '/etc/confluent/tls/cacert.pem', '-hash', '-noout'])
+        hv, _ = await util.check_output(
+            'openssl', 'x509', '-in', '/etc/confluent/tls/cacert.pem', '-hash', '-noout')
         if not isinstance(hv, str):
             hv = hv.decode('utf8')
         hv = hv.strip()
@@ -178,6 +188,17 @@ def assure_tls_ca():
         os.symlink(certname, hashname)
     finally:
         os.seteuid(ouid)
+    return certout
+
+#def is_self_signed(pem):
+#    cert = ssl.PEM_cert_to_DER_cert(pem)
+#    return cert.get('subjectAltName', []) == cert.get('issuer', [])
+# x509 certificate issuer subject comparison..
+#>>> b.issuer
+#<Name(C=US,ST=NC,L=RTP,O=Lenovo,CN=XCC-7D9D-J102MM2T)>
+#>>> b.subject
+#<Name(C=US,ST=NC,L=RTP,O=Lenovo,CN=XCC-7D9D-J102MM2T)>
+
 
 def substitute_cfg(setting, key, val, newval, cfgfile, line):
     if key.strip() == setting:
@@ -185,7 +206,7 @@ def substitute_cfg(setting, key, val, newval, cfgfile, line):
         return True
     return False
 
-def create_full_ca(certout):
+async def create_full_ca(certout):
     mkdirp('/etc/confluent/tls/ca/private')
     keyout = '/etc/confluent/tls/ca/private/cakey.pem'
     csrout = '/etc/confluent/tls/ca/ca.csr'
@@ -222,24 +243,24 @@ def create_full_ca(certout):
                         cfgfile.write(line.strip() + '\n')
                     continue
                 cfgfile.write(line.strip() + '\n')
-            cfgfile.write('\n[CACert]\nbasicConstraints = CA:true\n\n[ca_confluent]\n')
-    subprocess.check_call(
-        ['openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
-        keyout])
-    subprocess.check_call(
-        ['openssl', 'req', '-new', '-key', keyout, '-out', csrout, '-subj', subj])
-    subprocess.check_call(
-        ['openssl', 'ca', '-config', newcfg, '-batch', '-selfsign',
+            cfgfile.write('\n[CACert]\nbasicConstraints = critical,CA:true\nkeyUsage = critical,keyCertSign,cRLSign\n[ca_confluent]\n')
+    await util.check_call(
+        'openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
+        keyout)
+    await util.check_call(
+        'openssl', 'req', '-new', '-key', keyout, '-out', csrout, '-subj', subj)
+    await util.check_call(
+        'openssl', 'ca', '-config', newcfg, '-batch', '-selfsign',
         '-extensions', 'CACert', '-extfile', newcfg, 
-        '-notext', '-startdate',
+        '-notext', '-md', 'sha384', '-startdate',
          '19700101010101Z', '-enddate', '21000101010101Z', '-keyfile',
-         keyout, '-out', '/etc/confluent/tls/ca/cacert.pem', '-in', csrout]
+         keyout, '-out', '/etc/confluent/tls/ca/cacert.pem', '-in', csrout
     )
     shutil.copy2('/etc/confluent/tls/ca/cacert.pem', certout)
 #openssl ca -config openssl.cnf -selfsign -keyfile cakey.pem -startdate 20150214120000Z -enddate 20160214120000Z
 #20160107071311Z -enddate 20170106071311Z
 
-def create_simple_ca(keyout, certout):
+async def create_simple_ca(keyout, certout):
     try:
         os.makedirs('/etc/confluent/tls')
     except OSError as e:
@@ -249,92 +270,136 @@ def create_simple_ca(keyout, certout):
     tmphdl, tmpconfig = tempfile.mkstemp()
     os.close(tmphdl)
     shutil.copy2(sslcfg, tmpconfig)
-    subprocess.check_call(
-            ['openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
-            keyout])
+    await util.check_call(
+            'openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
+            keyout)
     try:
         subj = '/CN=Confluent TLS Certificate authority ({0})'.format(socket.gethostname())
         if len(subj) > 68:
             subj = subj[:68]
         with open(tmpconfig, 'a') as cfgfile:
-            cfgfile.write('\n[CACert]\nbasicConstraints = CA:true\n')
-        subprocess.check_call([
+            cfgfile.write('\n[CACert]\nbasicConstraints = critical,CA:true\n')
+        await util.check_call(
                 'openssl', 'req', '-new', '-x509', '-key', keyout, '-days',
                 '27300', '-out', certout, '-subj', subj,
                 '-extensions', 'CACert', '-config', tmpconfig
-            ])
+            )
     finally:
         os.remove(tmpconfig)
 
-def create_certificate(keyout=None, certout=None, csrout=None):
-    if not keyout:
+async def create_certificate(keyout=None, certout=None, csrfile=None, subj=None, san=None, backdate=True, days=None):
+    now_utc = datetime.datetime.now(datetime.timezone.utc)
+    if backdate:
+        # To deal with wildly off clocks, we backdate certificates.
+        startdate = '20000101010101Z'
+    else:
+        # apply a mild backdate anyway, even if these are supposed to be for more accurate clocks
+        startdate = (now_utc - datetime.timedelta(hours=24)).strftime('%Y%m%d%H%M%SZ')
+    if days is None:
+        enddate = '21000101010101Z'
+    else:
+        enddate = (now_utc + datetime.timedelta(days=days)).strftime('%Y%m%d%H%M%SZ')
+    tlsmateriallocation = {}
+    if not certout:
         tlsmateriallocation = get_certificate_paths()
         keyout = tlsmateriallocation.get('keys', [None])[0]
         certout = tlsmateriallocation.get('certs', [None])[0]
         if not certout:
             certout = tlsmateriallocation.get('bundles', [None])[0]
-    if not keyout or not certout:
+    if (not keyout and not csrfile) or not certout:
         raise Exception('Unable to locate TLS certificate path automatically')
-    assure_tls_ca()
-    shortname = socket.gethostname().split('.')[0]
-    longname = shortname # socket.getfqdn()
-    if not csrout:
-        subprocess.check_call(
-            ['openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
-             keyout])
-    ipaddrs = list(get_ip_addresses())
-    san = ['IP:{0}'.format(x) for x in ipaddrs]
-    # It is incorrect to put IP addresses as DNS type.  However
-    # there exists non-compliant clients that fail with them as IP
-    # san.extend(['DNS:{0}'.format(x) for x in ipaddrs])
-    dnsnames = set(ipaddrs)
-    dnsnames.add(shortname)
-    for currip in ipaddrs:
-        dnsnames.add(socket.getnameinfo((currip, 0), 0)[0])
-    for currname in dnsnames:
-        san.append('DNS:{0}'.format(currname))
-    #san.append('DNS:{0}'.format(longname))
-    san = ','.join(san)
+    cacertname = await assure_tls_ca()
+    if not subj:
+        shortname = socket.gethostname().split('.')[0]
+        longname = shortname # socket.getfqdn()
+        subj = '/CN={0}'.format(longname)
+    elif '/CN=' not in subj:
+        subj = '/CN={0}'.format(subj)
+    if not csrfile:
+        await util.check_call(
+            'openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
+             keyout)
+    permitdomains = []
+    if x509:
+        # check if this CA has name constraints, and avoid violating them
+        with open(cacertname, 'rb') as f:
+            cer = x509.load_pem_x509_certificate(f.read())
+        for extension in cer.extensions:
+            if extension.oid == x509.ExtensionOID.NAME_CONSTRAINTS:
+                nc = extension.value
+                for pname in nc.permitted_subtrees:
+                    permitdomains.append(pname.value)
+    if not san:
+        ipaddrs = []
+        async for ip in get_ip_addresses():
+            ipaddrs.append(ip)
+        if not permitdomains:
+            san = ['IP:{0}'.format(x) for x in ipaddrs]
+            # It is incorrect to put IP addresses as DNS type.  However
+            # there exists non-compliant clients that fail with them as IP
+            # san.extend(['DNS:{0}'.format(x) for x in ipaddrs])
+            dnsnames = set(ipaddrs)
+            dnsnames.add(shortname)
+            dnsnames.add(longname)
+        else:
+            # nameconstraints preclude IP and shortname
+            san = []
+            dnsnames = set()
+            for suffix in permitdomains:
+                if longname.endswith(suffix):
+                    dnsnames.add(longname)
+                    break
+        for currip in ipaddrs:
+            currname = socket.getnameinfo((currip, 0), 0)[0]
+            for suffix in permitdomains:
+                if currname.endswith(suffix):
+                    dnsnames.add(currname)
+                    break
+            if not permitdomains:
+                dnsnames.add(currname)
+        for currname in dnsnames:
+            san.append('DNS:{0}'.format(currname))
+        #san.append('DNS:{0}'.format(longname))
+        san = ','.join(san)
     sslcfg = get_openssl_conf_location()
     tmphdl, tmpconfig = tempfile.mkstemp()
     os.close(tmphdl)
     tmphdl, extconfig = tempfile.mkstemp()
     os.close(tmphdl)
     needcsr = False
-    if csrout is None:
+    if csrfile is None:
         needcsr = True
-        tmphdl, csrout = tempfile.mkstemp()
+        tmphdl, csrfile = tempfile.mkstemp()
         os.close(tmphdl)
     shutil.copy2(sslcfg, tmpconfig)
     try:
+        with open(extconfig, 'a') as cfgfile:
+            cfgfile.write('\nbasicConstraints=critical,CA:false\nkeyUsage=critical,digitalSignature\nextendedKeyUsage=serverAuth,clientAuth\nsubjectAltName={0}'.format(san))
         if needcsr:
             with open(tmpconfig, 'a') as cfgfile:
                 cfgfile.write('\n[SAN]\nsubjectAltName={0}'.format(san))
-            with open(extconfig, 'a') as cfgfile:
-                cfgfile.write('\nbasicConstraints=CA:false\nsubjectAltName={0}'.format(san))
-            subprocess.check_call([
-                'openssl', 'req', '-new', '-key', keyout, '-out', csrout, '-subj',
-                '/CN={0}'.format(longname),
-               '-extensions', 'SAN', '-config', tmpconfig
-            ])
-        else:
-            # when used manually, allow the csr SAN to stand
-            # may add explicit subj/SAN argument, in which case we would skip copy
-            with open(tmpconfig, 'a') as cfgfile:
-                cfgfile.write('\ncopy_extensions=copy\n')
-            with open(extconfig, 'a') as cfgfile:
-                cfgfile.write('\nbasicConstraints=CA:false\n')
+            await util.check_call(
+                'openssl', 'req', '-new', '-key', keyout, '-out', csrfile, '-subj',
+                subj, '-extensions', 'SAN', '-config', tmpconfig
+            )
+        #else:
+        #    # when used manually, allow the csr SAN to stand
+        #    # may add explicit subj/SAN argument, in which case we would skip copy
+        #    #with open(tmpconfig, 'a') as cfgfile:
+        #    #    cfgfile.write('\ncopy_extensions=copy\n')
+        #    with open(extconfig, 'a') as cfgfile:
+        #        cfgfile.write('\nbasicConstraints=CA:false\n')
         if os.path.exists('/etc/confluent/tls/cakey.pem'):
             # simple style CA in effect, make a random serial number and
             # hope for the best, and accept inability to backdate the cert
             serialnum = '0x' + ''.join(['{:02x}'.format(x) for x in bytearray(os.urandom(20))])
-            subprocess.check_call([
-                'openssl', 'x509', '-req', '-in', csrout,
+            await util.check_call(
+                'openssl', 'x509', '-req', '-in', csrfile,
                 '-CA', '/etc/confluent/tls/cacert.pem',
                 '-CAkey', '/etc/confluent/tls/cakey.pem',
                 '-set_serial', serialnum, '-out', certout, '-days', '27300',
                 '-extfile', extconfig
-            ])
+            )
         else:
             # we moved to a 'proper' CA, mainly for access to backdating
             # start of certs for finicky system clocks
@@ -348,13 +413,12 @@ def create_certificate(keyout=None, certout=None, csrout=None):
                 shutil.copy2(cacfgfile, tmpcafile)
                 os.close(tmphdl)
                 cacfgfile = tmpcafile
-            # with realcalock:  # if we put it in server, we must lock it
-            subprocess.check_call([
-                'openssl', 'ca', '-config', cacfgfile,
-                '-in', csrout, '-out', certout, '-batch', '-notext',
-                '-startdate', '19700101010101Z', '-enddate', '21000101010101Z',
-                '-extfile', extconfig
-            ])
+            await util.check_call(
+                'openssl', 'ca', '-config', cacfgfile, '-rand_serial',
+                '-in', csrfile, '-out', certout, '-batch', '-notext',
+                '-startdate', startdate, '-enddate', enddate, '-md', 'sha384',
+                '-extfile', extconfig, '-subj', subj
+            )
         for keycopy in tlsmateriallocation.get('keys', []):
             if keycopy != keyout:
                 shutil.copy2(keyout, keycopy)
@@ -381,18 +445,43 @@ def create_certificate(keyout=None, certout=None, csrout=None):
     finally:
         os.remove(tmpconfig)
         if needcsr:
-            os.remove(csrout)
-        print(extconfig)  # os.remove(extconfig)
+            os.remove(csrfile)
+        os.remove(extconfig)
 
 
 if __name__ == '__main__':
     import sys
+    import ipaddress
     outdir = os.getcwd()
     keyout = os.path.join(outdir, 'key.pem')
-    certout = os.path.join(outdir, sys.argv[2] + 'cert.pem')
+    certout = os.path.join(outdir, 'cert.pem')
     csrout = None
+    subj, san = (None, None)
+    try:
+        bindex = sys.argv.index('-b')
+        bmcnode = sys.argv.pop(bindex + 1)  # Remove bmcnode argument
+        sys.argv.pop(bindex)      # Remove -b flag
+        import confluent.config.configmanager as cfm
+        c = cfm.ConfigManager(None)
+        subj, san = util.get_bmc_subject_san(c, bmcnode)
+    except ValueError:
+        bindex = None
+    if subj is None:
+        try:
+            sans = set()
+            sindex = sys.argv.index('-s')
+            subj = sys.argv.pop(sindex + 1)  # Remove subject argument
+            sys.argv.pop(sindex)      # Remove -s flag
+            try:
+                ipaddress.ip_address(subj)
+                sans.add('IP:{0}'.format(subj))
+            except ValueError:
+                sans.add('DNS:{0}'.format(subj))
+            san = ','.join(sans) if sans else None
+        except ValueError:
+            pass
     try:
         csrout = sys.argv[1]
     except IndexError:
         csrout = None
-    create_certificate(keyout, certout, csrout)
+    create_certificate(keyout, certout, csrout, subj, san, backdate=False, days=3650)