Fix mistake in spec file

Add support for a confluent=<host> kernel argument in init-premount: configure networking, flush interfaces, autodetect the primary NIC (saved to /tmp/autodetectnic), verify TLS connectivity to the provided server, call the whoami endpoint over TLS to obtain the node name, and write results to /custom-installation/confluent/confluent.info (with fallback to copernicus on failure).

Also update casper-bottom logic to handle IPv4 manager addresses: for IPv6 the manager is still bracketed and scoped interface resolved as before; for IPv4 the script now uses the previously detected NIC (/tmp/autodetectnic) or falls back to an `ip route get <mgr>` lookup to determine DEVICE. This ensures routed IPv4 deployments work correctly.

build/arm/el10/Dockerfile

@@ -0,0 +1,8 @@
+FROM almalinux:10
+RUN ["yum", "-y","update"]
+RUN ["yum", "-y","install","gcc","make","rpm-build","python3-devel","python3-setuptools","createrepo","python3", "perl", "perl-DBI", "perl-JSON", "perl-XML-LibXML", "pinentry-tty", "rpm-sign", "git", "golang"]
+ADD rpmmacro /root/.rpmmacros
+ADD buildpackages.sh /bin/
+#VOLUME ["/rpms", "/srpms"]
+CMD ["/bin/bash","/bin/buildpackages.sh"]
+

build/arm/el10/buildpackages.sh

@@ -0,0 +1,6 @@
+for package in /srpms/*; do
+        rpmbuild --rebuild $package
+done
+find ~/rpmbuild/RPMS -type f -exec cp {} /rpms/ \;
+
+

build/arm/el10/rpmmacro

@@ -0,0 +1,3 @@
+%_gpg_digest_algo sha256
+%_gpg_name Lenovo Scalable Infrastructure
+

build/arm/el8/Dockerfile

@@ -0,0 +1,8 @@
+FROM almalinux:8
+RUN ["yum", "-y","update"]
+RUN ["yum", "-y","install","gcc","make","rpm-build","python3-devel","python3-setuptools","createrepo","python3", "perl", "perl-DBI", "perl-JSON", "perl-Net-DNS", "perl-DB_File", "perl-XML-LibXML", "rpm-sign", "git", "fuse-devel","libcurl-devel"]
+ADD rpmmacro /root/.rpmmacros
+ADD buildpackages.sh /bin/
+#VOLUME ["/rpms", "/srpms"]
+CMD ["/bin/bash","/bin/buildpackages.sh"]
+

build/arm/el8/buildpackages.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+for package in /srpms/*; do
+        rpmbuild --rebuild $package
+done
+find ~/rpmbuild/RPMS -type f -exec cp {} /rpms/ \;
+

build/arm/el8/rpmmacro

@@ -0,0 +1,2 @@
+%_gpg_digest_algo sha256
+%_gpg_name Lenovo Scalable Infrastructure

build/arm/el9/Dockerfile

@@ -0,0 +1,10 @@
+FROM almalinux:9
+RUN ["yum", "-y","update"]
+RUN ["yum", "-y","install","gcc","make","rpm-build","python3-devel","python3-setuptools","createrepo","python3", "perl", "perl-DBI", "perl-JSON", "perl-Net-DNS", "perl-DB_File", "perl-XML-LibXML", "pinentry-tty", "rpm-sign", "epel-release", "git"]
+RUN ["crb", "enable"]
+RUN ["yum", "-y","install","fuse-devel","libcurl-devel"]
+ADD rpmmacro /root/.rpmmacros
+ADD buildpackages.sh /bin/
+#VOLUME ["/rpms", "/srpms"]
+CMD ["/bin/bash","/bin/buildpackages.sh"]
+

build/arm/el9/buildpackages.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+for package in /srpms/*; do
+        rpmbuild --rebuild $package
+done
+find ~/rpmbuild/RPMS -type f -exec cp {} /rpms/ \;
+

build/arm/el9/rpmmacro

@@ -0,0 +1,2 @@
+%_gpg_digest_algo sha256
+%_gpg_name Lenovo Scalable Infrastructure

build/arm/noble/Dockerfile

@@ -0,0 +1,12 @@
+FROM ubuntu:noble
+ADD stdeb.patch /tmp/
+ADD buildapt.sh /bin/
+ADD distributions.tmpl /bin/
+RUN ["apt-get", "update"]
+RUN ["apt-get", "install", "-y", "reprepro", "python3-stdeb", "gnupg-agent", "devscripts", "debhelper", "libsoap-lite-perl", "libdbi-perl", "quilt", "git", "python3-pyparsing", "python3-dnspython", "python3-eventlet", "python3-netifaces", "python3-paramiko", "dh-python", "libjson-perl", "ronn", "alien", "gcc", "make"]
+RUN ["mkdir", "-p", "/sources/git/"]
+RUN ["mkdir", "-p", "/debs/"]
+RUN ["mkdir", "-p", "/apt/"]
+RUN ["bash", "-c", "patch -p1 < /tmp/stdeb.patch"]
+CMD ["/bin/bash", "/bin/buildapt.sh"]
+

build/arm/noble/buildapt.sh

@@ -0,0 +1,21 @@
+#cp -a /sources/git /tmp
+for builder in $(find /sources/git -name builddeb); do
+    cd $(dirname $builder)
+    ./builddeb /debs/
+done
+cp /prebuilt/* /debs/
+cp /osd/*.deb /debs/
+mkdir -p /apt/conf/
+CODENAME=$(grep VERSION_CODENAME= /etc/os-release | sed -e 's/.*=//')
+if [ -z "$CODENAME" ]; then
+    CODENAME=$(grep VERSION= /etc/os-release | sed -e 's/.*(//' -e 's/).*//')
+fi
+if ! grep $CODENAME /apt/conf/distributions; then
+    sed -e s/#CODENAME#/$CODENAME/ /bin/distributions.tmpl >> /apt/conf/distributions
+fi
+cd /apt/
+reprepro includedeb $CODENAME /debs/*.deb
+for dsc in /debs/*.dsc; do
+    reprepro includedsc $CODENAME $dsc
+done
+

build/arm/noble/distributions.tmpl

@@ -0,0 +1,7 @@
+Origin: Lenovo HPC Packages
+Label: Lenovo HPC Packages
+Codename: #CODENAME#
+Architectures: amd64 source
+Components: main
+Description: Lenovo HPC Packages
+

build/arm/noble/stdeb.patch

@@ -0,0 +1,34 @@
+diff -urN t/usr/lib/python3/dist-packages/stdeb/cli_runner.py t.patch/usr/lib/python3/dist-packages/stdeb/cli_runner.py
+--- t/usr/lib/python3/dist-packages/stdeb/cli_runner.py 2024-06-11 18:30:13.930328999 +0000
++++ t.patch/usr/lib/python3/dist-packages/stdeb/cli_runner.py   2024-06-11 18:32:05.392731405 +0000
+@@ -8,7 +8,7 @@
+     from ConfigParser import SafeConfigParser  # noqa: F401
+ except ImportError:
+     # python 3.x
+-    from configparser import SafeConfigParser  # noqa: F401
++    from configparser import ConfigParser  # noqa: F401
+ from distutils.util import strtobool
+ from distutils.fancy_getopt import FancyGetopt, translate_longopt
+ from stdeb.util import stdeb_cmdline_opts, stdeb_cmd_bool_opts
+diff -urN t/usr/lib/python3/dist-packages/stdeb/util.py t.patch/usr/lib/python3/dist-packages/stdeb/util.py
+--- t/usr/lib/python3/dist-packages/stdeb/util.py       2024-06-11 18:32:53.864776149 +0000
++++ t.patch/usr/lib/python3/dist-packages/stdeb/util.py 2024-06-11 18:33:02.063952870 +0000
+@@ -730,7 +730,7 @@
+     example.
+     """
+ 
+-    cfg = ConfigParser.SafeConfigParser()
++    cfg = ConfigParser.ConfigParser()
+     cfg.read(cfg_files)
+     if cfg.has_section(module_name):
+         section_items = cfg.items(module_name)
+@@ -801,7 +801,7 @@
+         if len(cfg_files):
+             check_cfg_files(cfg_files, module_name)
+ 
+-        cfg = ConfigParser.SafeConfigParser(cfg_defaults)
++        cfg = ConfigParser.ConfigParser(cfg_defaults)
+         for cfg_file in cfg_files:
+             with codecs.open(cfg_file, mode='r', encoding='utf-8') as fd:
+                 cfg.readfp(fd)
+

build/buildarm

@@ -0,0 +1,9 @@
+cd ~/confluent
+git pull
+rm ~/rpmbuild/RPMS/noarch/*osdeploy*
+rm ~/rpmbuild/SRPMS/*osdeploy*
+sh confluent_osdeploy/buildrpm-aarch64
+mkdir -p $HOME/el9/
+mkdir -p $HOME/el10/
+podman run --rm -it -v $HOME:/build el9build bash /build/confluent/confluent_vtbufferd/buildrpm /build/el9/
+

confluent_client/bin/nodeapply

@@ -36,6 +36,36 @@ import confluent.client as client
 import confluent.sortutil as sortutil
 devnull = None
 
+def run_automation(noderange, category, c):
+    automationbynode = {}
+    for res in c.update('/noderange/{0}/deployment/remote_config/run'.format(noderange), {
+            'category': category,
+        }):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                exitcode |= res.get('errorcode', 1)
+            if 'created' in res:
+                nodename = res['created'].split('/')[2]
+                automationbynode[nodename] = res['created']
+    while automationbynode:
+        for node in list(automationbynode):
+            for res in c.read(automationbynode[node]):
+                if 'error' in res:
+                    sys.stderr.write(res['error'] + '\n')
+                    exitcode |= res.get('errorcode', 1)
+                for result in res.get('results', []):
+                    sys.stdout.write('{0}: Task [{1}] {2}\n'.format(
+                        node, result['task_name'], result['state']))
+                    for warning in result.get('warnings', []):
+                        sys.stderr.write('{0}: [WARNING] {1}\n'.format(node, warning))
+                    if 'errorinfo' in result:
+                        for errorline in result['errorinfo'].splitlines():
+                            sys.stderr.write('{0}: [ERROR] {1}\n'.format(node, errorline))                
+                if res.get('complete', False):
+                    del automationbynode[node]
+                    sys.stdout.write('{0}: Automation complete\n'.format(node))
+                    
+
 def run():
     global devnull
     devnull = open(os.devnull, 'rb')
@@ -51,6 +81,8 @@ def run():
                          help='Run the syncfiles associated with the currently completed OS profile on the noderange')
     argparser.add_option('-P', '--scripts',
                          help='Re-run specified scripts, with full path under scripts, e.g. post.d/first,firstboot.d/second')
+    argparser.add_option('-A', '--automation',
+                         help='Run the automation scripts associated with the current OS profile on the noderange, specifying category (onboot.d/firstboot.d/post.d)')
     argparser.add_option('-m', '--maxnodes', type='int',
                     help='Specify a maximum number of '
                         'nodes to run remote ssh command to, '
@@ -74,16 +106,13 @@ def run():
     exitcode = 0
 
     c.stop_if_noderange_over(args[0], options.maxnodes)
+    if options.automation:
+        run_automation(args[0], options.automation, c)
+        
     nodemap = {}
     cmdparms = []
     nodes = []
-    for res in c.read('/noderange/{0}/nodes/'.format(args[0])):
-        if 'error' in res:
-            sys.stderr.write(res['error'] + '\n')
-            exitcode |= res.get('errorcode', 1)
-            break
-        node = res['item']['href'][:-1]
-        nodes.append(node)
+
     
     cmdstorun = []
     if options.security:
@@ -94,8 +123,17 @@ def run():
         for script in options.scripts.split(','):
             cmdstorun.append(['run_remote', script])
     if not cmdstorun:
+        if options.automation:
+            sys.exit(0)
         argparser.print_help()
         sys.exit(1)
+    for res in c.read('/noderange/{0}/nodes/'.format(args[0])):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode |= res.get('errorcode', 1)
+            break
+        node = res['item']['href'][:-1]
+        nodes.append(node)
     idxbynode = {}
     cmdvbase = ['bash', '/etc/confluent/functions']
     for sshnode in nodes:
@@ -145,7 +183,7 @@ def run():
                                 run_cmdv(node, cmdv, all, poller, pipedesc)
                             elif pendingexecs:
                                 node, cmdv = pendingexecs.popleft()
-                                run_cmdv(node, cmdv, all, poller. pipedesc)
+                                run_cmdv(node, cmdv, all, poller, pipedesc)
             singlepoller.close()
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:

confluent_client/bin/nodecertutil

@@ -76,6 +76,10 @@ if __name__ == '__main__':
 
     list_parser = subparsers.add_parser('listbmccacerts', help='List BMC CA certificates')
 
+    sign_bmc_parser = subparsers.add_parser('signbmccert', help='Sign BMC certificate')
+    sign_bmc_parser.add_argument('--days', type=int, help='Number of days the certificate is valid for')
+    sign_bmc_parser.add_argument('--added-names', type=str, help='Additional names to include in the certificate')
+
     args = parser.parse_args()
     c = client.Command()
     if args.command == 'installbmccacert':
@@ -84,6 +88,17 @@ if __name__ == '__main__':
         removebmccacert(args.noderange, args.id, c)
     elif args.command == 'listbmccacerts':
         listbmccacerts(args.noderange, c)
+    elif args.command == 'signbmccert':
+        payload = {}
+        if args.days is not None:
+            payload['days'] = args.days
+        else:
+            print("Error: --days is required for signbmccert", file=sys.stderr)
+            sys.exit(1)
+        if args.added_names:
+            payload['added_names'] = args.added_names
+        for res in c.update(f'/noderange/{args.noderange}/configuration/management_controller/certificate/sign', payload):
+            print(repr(res))
     else:
         parser.print_help()
         sys.exit(1)

confluent_client/bin/nodeconfig

@@ -62,7 +62,7 @@ argparser.add_option('-e', '--extra', dest='extra',
                            'to be extra configuration')
 argparser.add_option('-x', '--exclude', dest='exclude',
                      action='store_true', default=False,
-                     help='Treat positional arguments as items to not '
+                     help='Treat named settings as items to not '
                           'examine, compare, or restore default')
 argparser.add_option('-a', '--advanced', dest='advanced',
                      action='store_true', default=False,
@@ -73,7 +73,7 @@ argparser.add_option('-r', '--restoredefault', default=False,
                      dest='restoredefault', metavar="COMPONENT",
                      help='Restore the configuration of the node '
                           'to factory default for given component. '
-                          'Currently only uefi is supported')
+                          'Currently "uefi" and "bmc" are supported components')
 argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '
                           'nodes to configure, '

confluent_client/bin/nodehealth

@@ -84,13 +84,13 @@ def main():
                 healthexplanations[node] = []
                 for sensor in health[node]['sensors']:
                     explanation = sensor['name'] + ':'
-                    if sensor['value'] is not None:
+                    if sensor.get('value', None) is not None:
                         explanation += str(sensor['value'])
-                        if sensor['units'] is not None:
+                        if sensor.get('units', None) is not None:
                             explanation += sensor['units']
-                        if sensor['states']:
+                        if sensor.get('states', None):
                             explanation += ','
-                    if sensor['states']:
+                    if sensor.get('states', None):
                         explanation += ','.join(sensor['states'])
                     healthexplanations[node].append(explanation)
             if node in healthbynode and node in healthexplanations:

confluent_client/bin/nodeinventory

@@ -60,16 +60,21 @@ def print_mem_info(node, prefix, meminfo):
     elif 'DCPMM' in meminfo['memory_type']:
         memdescfmt = '{0}GB {1} '
         meminfo['module_type'] = 'DCPMM'
+    elif meminfo['memory_type'] == 'HBM':
+        memdescfmt = '{0}GB HBM '
     else:
         print('{0}: {1}: Unrecognized Memory'.format(node, prefix))
         return
     if meminfo.get('ecc', False):
         memdescfmt += 'ECC '
-    capacity = meminfo['capacity_mb'] / 1024
     modtype = meminfo.get('module_type', None)
     if modtype:
         memdescfmt += modtype
-    memdesc = memdescfmt.format(capacity, meminfo['speed'])
+    if meminfo.get('capacity_mb', None):
+        capacity = meminfo['capacity_mb'] // 1024
+        memdesc = memdescfmt.format(capacity, meminfo['speed'])
+    else:
+        memdesc = 'Unspecified Module'
     print('{0}: {1} description: {2}'.format(node, prefix, memdesc))
     print('{0}: {1} manufacturer: {2}'.format(
         node, prefix, meminfo['manufacturer']))

confluent_client/bin/nodesensors

@@ -123,7 +123,7 @@ def sensorpass(showout=True, appendtime=False):
                 if 'sensors' not in reading[node]:
                     continue
                 for sensedata in reading[node]['sensors']:
-                    if sensedata['value'] is None and options.skipnumberless:
+                    if sensedata.get('value', None) is None and options.skipnumberless:
                         continue
                     for redundant_state in ('Non-Critical', 'Critical'):
                         try:
@@ -134,17 +134,17 @@ def sensorpass(showout=True, appendtime=False):
                     resultdata[node][sensedata['name']] = sensedata
                     sensorname = sensedata['name']
                     sensorheaders[sensorname] = sensorname
-                    if sensedata['units'] not in (None, u''):
+                    if sensedata.get('units', None) not in (None, u''):
                         sensorheaders[sensorname] += u' ({0})'.format(
                             sensedata['units'])
                     if showout:
-                        if sensedata['value'] is None:
+                        if sensedata.get('value', None) is None:
                             showval = ''
                         elif isinstance(sensedata['value'], float):
                             showval = u' {0} '.format(floatformat(sensedata['value']))
                         else:
-                            showval = u' {0} '.format(sensedata['value'])
-                        if sensedata['units'] not in (None, u''):
+                            showval = u' {0} '.format(sensedata.get('value', ''))
+                        if sensedata.get('units', None) not in (None, u''):
                             showval += sensedata['units']
                         if sensedata.get('health', 'ok') != 'ok':
                             datadescription = [sensedata['health']]

confluent_client/confluent/client.py

@@ -391,8 +391,13 @@ class Command(object):
             cacert = None
             certreqs = ssl.CERT_NONE
             knownhosts = True
-        self.connection = ssl.wrap_socket(self.connection, ca_certs=cacert,
-                                          cert_reqs=certreqs)
+        tlsctx = ssl.create_default_context()
+        if certreqs == ssl.CERT_NONE:
+            tlsctx.check_hostname = False
+        tlsctx.verify_mode = certreqs
+        if cacert:
+            tlsctx.load_verify_locations(cacert)
+        self.connection = tlsctx.wrap_socket(self.connection, server_hostname=server)
         if knownhosts:
             certdata = self.connection.getpeercert(binary_form=True)
             fingerprint = 'sha512$' + hashlib.sha512(certdata).hexdigest()

confluent_client/doc/man/imgutil.ronn

@@ -0,0 +1,139 @@
+# imgutil(1) -- Work with confluent OS cloning and diskless images
+
+## SYNOPSIS
+
+`imgutil` `build` [<options>] <scratchdir>  
+`imgutil` `exec` [<options>] <scratchdir> [<cmd>...]  
+`imgutil` `unpack` <profilename> <scratchdir>  
+`imgutil` `pack` [<options>] <scratchdir> <profilename>  
+`imgutil` `capture` <node> <profilename>
+
+## DESCRIPTION
+
+**imgutil** is a utility for creating, managing, and deploying OS images for diskless boot and system cloning in a Confluent environment. It supports building images from scratch, capturing images from running systems, and packing/unpacking diskless profiles.
+
+## COMMANDS
+
+* `build`:
+  Build a new diskless image from scratch in the specified scratch directory.
+
+* `exec`:
+  Start the specified scratch directory as a container and optionally run a command inside it.
+
+* `unpack`:
+  Unpack a diskless image profile to a scratch directory for modification.
+
+* `pack`:
+  Pack a scratch directory into a diskless profile that can be deployed.
+
+* `capture`:
+  Capture an image for cloning from a running system.
+
+## BUILD OPTIONS
+
+* `-r`, `--addrepos` <repository>:
+  Repositories to add in addition to the main source. May be specified multiple times.
+
+* `-p`, `--packagelist` <file>:
+  Filename of package list to replace default pkglist.
+
+* `-a`, `--addpackagelist` <file>:
+  A list of additional packages to include. May be specified multiple times.
+
+* `-s`, `--source` <directory>:
+  Directory to pull installation from, typically a subdirectory of `/var/lib/confluent/distributions`. By default, the repositories for the build system are used. For Ubuntu, this is not supported; the build system repositories are always used.
+
+* `-y`, `--non-interactive`:
+  Avoid prompting for confirmation.
+
+* `-v`, `--volume` <mount>:
+  Directory to make available in the build environment. `-v /` will cause it to be mounted in image as `/run/external/`. `-v /:/run/root` will override the target to be `/run/root`. Something like `/var/lib/repository:-` will cause it to mount to the identical path inside the image. May be specified multiple times.
+
+* <scratchdir>:
+  Directory to build new diskless root in.
+
+## EXEC OPTIONS
+
+* `-v`, `--volume` <mount>:
+  Directory to make available in the build environment. `-v /` will cause it to be mounted in image as `/run/external/`. `-v /:/run/root` will override the target to be `/run/root`. May be specified multiple times.
+
+* <scratchdir>:
+  Directory of an unpacked diskless root.
+
+* <cmd>:
+  Optional command to run (defaults to a shell).
+
+## UNPACK OPTIONS
+
+* <profilename>:
+  The diskless OS profile to unpack.
+
+* <scratchdir>:
+  Directory to extract diskless root to.
+
+## PACK OPTIONS
+
+* `-b`, `--baseprofile` <profile>:
+  Profile to copy extra info from. For example, to make a new version of an existing profile, reference the previous one as baseprofile.
+
+* `-u`, `--unencrypted`:
+  Pack an unencrypted image rather than encrypting.
+
+* <scratchdir>:
+  Directory containing diskless root.
+
+* <profilename>:
+  The desired diskless OS profile name to pack the root into.
+
+## CAPTURE OPTIONS
+
+* <node>:
+  Node to capture image from.
+
+* <profilename>:
+  Profile name for captured image.
+
+## EXAMPLES
+
+Build a diskless image from a distribution:
+
+    imgutil build -s  alma-9.6-x86_64 /tmp/myimage
+
+Execute a shell in an unpacked image:
+
+    imgutil exec /tmp/myimage
+
+Execute a specific command in an image:
+
+    imgutil exec /tmp/myimage /bin/rpm -qa
+
+Unpack an existing profile for modification:
+
+    imgutil unpack myprofile /tmp/myimage
+
+Pack a modified image into a new profile:
+
+    imgutil pack /tmp/myimage myprofile-v2
+
+Capture an image from a running node:
+
+    imgutil capture node01 production-image
+
+## FILES
+
+* `/var/lib/confluent/public/os/`:
+  Default location for OS profiles.
+
+* `/var/lib/confluent/private/os/`:
+  Location for encrypted image keys and private data.
+
+* `/var/lib/confluent/distributions/`:
+  Default location for installation sources.
+
+## SEE ALSO
+
+osdeploy(8)
+
+## AUTHOR
+
+Written for the Confluent project.

confluent_client/doc/man/nodeattrib.ronn.tmpl

@@ -24,6 +24,8 @@ For a full list of attributes, run `nodeattrib <node> all` against a node.
 If `-c` is specified, this will set the nodeattribute to a null value.
 This is different from setting the value to an empty string.
 
+Arbitrary custom attributes can also be created with the `custom.` prefix.
+
 Attributes may be specified by wildcard, for example `net.*switch` will report
 all attributes that begin with `net.` and end with `switch`.
 

confluent_client/doc/man/nodeconfig.ronn

@@ -49,8 +49,7 @@ actually be in effect until a reboot.
 
 * `-r COMPONENT`, `--restoredefault=COMPONENT`:
   Request that the specified component of the targeted nodes will have its
-  configuration reset to default.  Currently the only component implemented
-  is uefi.
+  configuration reset to default.  The component may be "uefi" or "bmc".
 
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:
   Specify a maximum number of nodes to configure, prompting if over 

confluent_client/doc/man/nodegroupattrib.ronn.tmpl

@@ -11,7 +11,7 @@ nodegroupattrib(8) -- List or change confluent nodegroup attributes
 
 ## DESCRIPTION
 
-`nodegroupattrip` queries the confluent server to get information about nodes.
+`nodegroupattrib` queries the confluent server to get information about nodes.
 In the simplest form, it simply takes the given group and lists the attributes of that group.
 
 Contrasted with nodeattrib(8), settings managed by nodegroupattrib will be added

confluent_client/doc/man/nodemedia.ronn

@@ -32,6 +32,13 @@ BMCs map a virtual USB device to that url.  Content is loaded on demand, and
 as such that URL is referenced potentially once for every IO operation that
 the host platform attempts.
 
+## NOTES
+
+When doing an attach of an https:// url, you may hit an error if you have not enrolled your certificate authority.
+In a general confluent environment, you can usually address it by:
+  `# for cert in /var/lib/confluent/public/site/tls/*.pem; do nodecertutil s1-s4 installbmccacert $cert; done`  
+  
+
 ## OPTIONS
 
 * `-h`, `--help`:

confluent_osdeploy/bluefield/bfb-autoinstall

@@ -0,0 +1,74 @@
+#!/usr/bin/python3
+import glob
+import gzip
+import base64
+import os
+import subprocess
+import sys
+import tempfile
+
+def collect_certificates(tmpdir):
+    certdata = ''
+    for cacert in glob.glob(f'{tmpdir}/*.pem'):
+        with open(cacert, 'r') as f:
+            certdata += f.read()
+    return certdata
+
+def embed_certificates(incfg, certdata):
+    if not certdata:
+        raise Exception('No certificates found to embed')
+    incfg = incfg.replace('%CONFLUENTCERTCOLL%', certdata)
+    return incfg
+
+def embed_identity(incfg, identityjson):
+    incfg = incfg.replace('%IDENTJSON%', identityjson)
+    return incfg
+
+def embed_apiclient(incfg, apiclient):
+    with open(apiclient, 'r') as f:
+        apiclientdata = f.read()
+    compressed = gzip.compress(apiclientdata.encode())
+    encoded = base64.b64encode(compressed).decode()
+    incfg = incfg.replace('%APICLIENTZ64%', encoded)
+    return incfg
+
+def embed_data(tmpdir, outfile):
+    templatefile = f'{tmpdir}/bfb.cfg.template'
+    with open(templatefile, 'r') as f:
+        incfg = f.read()
+    
+    certdata = collect_certificates(tmpdir)
+    incfg = embed_certificates(incfg, certdata)
+    
+    with open(f'{tmpdir}/identity.json', 'r') as f:
+        identityjson = f.read()
+    
+    incfg = embed_identity(incfg, identityjson)
+    
+    incfg = embed_apiclient(incfg, f'{tmpdir}/../apiclient')
+    
+    with open(outfile, 'w') as f:
+        f.write(incfg)
+
+def get_identity_json(node):
+    identity_file = f'/var/lib/confluent/private/site/identity_files/{node}.json'
+    try:
+        with open(identity_file, 'r') as f:
+            return f.read()
+    except FileNotFoundError:
+        return None
+
+if __name__ == '__main__':
+    if len(sys.argv) != 4:
+        print("Usage: bfb-autoinstall <node> <bfbfile> <rshim>")
+        sys.exit(1)
+    
+    node = sys.argv[1]
+    bfbfile = sys.argv[2]
+    rshim = sys.argv[3]
+
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+    currdir = os.getcwd()
+    tempdir = tempfile.mkdtemp(prefix=f'bfb-autoinstall-{node}-')
+    embed_data(f'{currdir}/{node}', f'{tempdir}/bfb.cfg')
+    subprocess.check_call(['bfb-install', '-b', bfbfile, '-c', f'{tempdir}/bfb.cfg', '-r', rshim])

confluent_osdeploy/bluefield/hostscripts/bfb-autoinstall

@@ -0,0 +1,74 @@
+#!/usr/bin/python3
+import glob
+import gzip
+import base64
+import os
+import subprocess
+import sys
+import tempfile
+
+def collect_certificates(tmpdir):
+    certdata = ''
+    for cacert in glob.glob(f'{tmpdir}/*.pem'):
+        with open(cacert, 'r') as f:
+            certdata += f.read()
+    return certdata
+
+def embed_certificates(incfg, certdata):
+    if not certdata:
+        raise Exception('No certificates found to embed')
+    incfg = incfg.replace('%CONFLUENTCERTCOLL%', certdata)
+    return incfg
+
+def embed_identity(incfg, identityjson):
+    incfg = incfg.replace('%IDENTJSON%', identityjson)
+    return incfg
+
+def embed_apiclient(incfg, apiclient):
+    with open(apiclient, 'r') as f:
+        apiclientdata = f.read()
+    compressed = gzip.compress(apiclientdata.encode())
+    encoded = base64.b64encode(compressed).decode()
+    incfg = incfg.replace('%APICLIENTZ64%', encoded)
+    return incfg
+
+def embed_data(tmpdir, outfile):
+    templatefile = f'{tmpdir}/bfb.cfg.template'
+    with open(templatefile, 'r') as f:
+        incfg = f.read()
+    
+    certdata = collect_certificates(tmpdir)
+    incfg = embed_certificates(incfg, certdata)
+    
+    with open(f'{tmpdir}/identity.json', 'r') as f:
+        identityjson = f.read()
+    
+    incfg = embed_identity(incfg, identityjson)
+    
+    incfg = embed_apiclient(incfg, f'{tmpdir}/../apiclient')
+    
+    with open(outfile, 'w') as f:
+        f.write(incfg)
+
+def get_identity_json(node):
+    identity_file = f'/var/lib/confluent/private/site/identity_files/{node}.json'
+    try:
+        with open(identity_file, 'r') as f:
+            return f.read()
+    except FileNotFoundError:
+        return None
+
+if __name__ == '__main__':
+    if len(sys.argv) != 4:
+        print("Usage: bfb-autoinstall <node> <bfbfile> <rshim>")
+        sys.exit(1)
+    
+    node = sys.argv[1]
+    bfbfile = sys.argv[2]
+    rshim = sys.argv[3]
+
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+    currdir = os.getcwd()
+    tempdir = tempfile.mkdtemp(prefix=f'bfb-autoinstall-{node}-')
+    embed_data(f'{currdir}/{node}', f'{tempdir}/bfb.cfg')
+    subprocess.check_call(['bfb-install', '-b', bfbfile, '-c', f'{tempdir}/bfb.cfg', '-r', rshim])

confluent_osdeploy/bluefield/profiles/default/bluefield.cfg.template

@@ -0,0 +1,76 @@
+function bfb_modify_os() {
+     echo 'ubuntu:!' | chpasswd -e
+     mkdir -p /mnt/opt/confluent/bin/
+     cat > /mnt/opt/confluent/bin/confluentbootstrap.sh << 'END_OF_EMBED'
+#!/bin/bash
+     cat > /usr/local/share/ca-certificates/confluent.crt << 'END_OF_CERTS'
+%CONFLUENTCERTCOLL%
+END_OF_CERTS
+     update-ca-certificates
+     mkdir -p /opt/confluent/bin /etc/confluent/
+     cp /usr/local/share/ca-certificates/confluent.crt /etc/confluent/ca.pem
+     cat > /opt/confluent/bin/apiclient.gz.b64 << 'END_OF_CLIENT'
+%APICLIENTZ64%
+END_OF_CLIENT
+     base64 -d /opt/confluent/bin/apiclient.gz.b64 | gunzip > /opt/confluent/bin/apiclient
+     cat > /etc/confluent/ident.json << 'END_OF_IDENT'
+%IDENTJSON%
+END_OF_IDENT
+     python3 /opt/confluent/bin/apiclient -i /etc/confluent/ident.json /confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
+     PROFILE=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+     ROOTPASS=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg | awk '{print $2}'|grep -v null)
+     if [ -n "$ROOTPASS" ]; then
+          echo root:$ROOTPASS | chpasswd -e
+          echo "ubuntu:$ROOTPASS" | chpasswd -e
+     else
+          echo 'ubuntu:!' | chpasswd -e
+     fi
+     cntmp=$(mktemp -d)
+     cd "$cntmp" || { echo "Failed to cd to temporary directory $cntmp"; exit 1; }
+     touch /etc/confluent/confluent.deploycfg
+     python3 /opt/confluent/bin/apiclient /confluent-public/os/$PROFILE/scripts/confignet > confignet
+     python3 confignet
+     cd -
+     rm -rf "$cntmp"
+     python3 /opt/confluent/bin/apiclient /confluent-public/os/$PROFILE/scripts/functions > /etc/confluent/functions
+     bash /etc/confluent/functions run_remote setupssh
+     for cert in /etc/ssh/ssh*-cert.pub; do
+         if [ -s $cert ]; then
+             echo HostCertificate $cert >> /etc/ssh/sshd_config.d/90-confluent.conf
+         fi
+     done
+     mkdir -p /var/log/confluent
+     chmod 700 /var/log/confluent
+     touch /var/log/confluent/confluent-firstboot.log
+     touch /var/log/confluent/confluent-post.log
+     chmod 600 /var/log/confluent/confluent-post.log
+     chmod 600 /var/log/confluent/confluent-firstboot.log
+     exec >> /var/log/confluent/confluent-post.log
+     exec 2>> /var/log/confluent/confluent-post.log
+     bash /etc/confluent/functions run_remote_python syncfileclient
+     bash /etc/confluent/functions run_remote_parts post.d
+     bash /etc/confluent/functions run_remote_config post.d
+     exec >> /var/log/confluent/confluent-firstboot.log
+     exec 2>> /var/log/confluent/confluent-firstboot.log
+     bash /etc/confluent/functions run_remote_parts firstboot.d
+     bash /etc/confluent/functions run_remote_config firstboot.d
+     python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus  -d 'status: staged'
+     python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus  -d 'status: complete'
+     systemctl disable confluentbootstrap
+     rm /etc/systemd/system/confluentbootstrap.service
+END_OF_EMBED
+    chmod +x /mnt/opt/confluent/bin/confluentbootstrap.sh
+    cat > /mnt/etc/systemd/system/confluentbootstrap.service << EOS
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/confluentbootstrap.sh
+
+[Install]
+WantedBy=multi-user.target
+EOS
+    chroot /mnt systemctl enable confluentbootstrap
+}

confluent_osdeploy/bluefield/profiles/default/nodedeploy-bfb

@@ -0,0 +1,125 @@
+#!/usr/bin/python3
+
+import os
+import sys
+import tempfile
+import glob
+import shutil
+import shlex
+import subprocess
+import select
+
+sys.path.append('/opt/lib/confluent/python')
+
+import confluent.sortutil as sortutil
+import confluent.client as client
+
+
+def prep_outdir(node):
+    tmpdir = tempfile.mkdtemp()
+    for certfile in glob.glob('/var/lib/confluent/public/site/tls/*.pem'):
+        basename = os.path.basename(certfile)
+        destfile = os.path.join(tmpdir, basename)
+        shutil.copy2(certfile, destfile)
+    subprocess.check_call(shlex.split(f'confetty set /nodes/{node}/deployment/ident_image=create'))
+    shutil.copy2(f'/var/lib/confluent/private/identity_files/{node}.json', os.path.join(tmpdir, 'identity.json'))
+    return tmpdir
+
+def exec_bfb_install(host, nodetorshim, bfbfile, installprocs, pipedesc, all, poller):
+    remotedir = subprocess.check_output(shlex.split(f'ssh {host} mktemp -d /tmp/bfb.XXXXXX')).decode().strip()
+    bfbbasename = os.path.basename(bfbfile)
+    subprocess.check_call(shlex.split(f'rsync -avz --info=progress2 {bfbfile} {host}:{remotedir}/{bfbbasename}'))
+    subprocess.check_call(shlex.split(f'rsync -avc --info=progress2 /opt/lib/confluent/osdeploy/bluefield/hostscripts/ {host}:{remotedir}/'))
+    for node in nodetorshim:
+        rshim = nodetorshim[node]
+        nodeoutdir = prep_outdir(node)
+        nodeprofile = subprocess.check_output(shlex.split(f'nodeattrib {node} deployment.pendingprofile')).decode().strip().split(':', 2)[2].strip()
+        shutil.copy2(f'/var/lib/confluent/public/os/{nodeprofile}/bfb.cfg.template', os.path.join(nodeoutdir, 'bfb.cfg.template'))
+        subprocess.check_call(shlex.split(f'rsync -avz {nodeoutdir}/ {host}:{remotedir}/{node}/'))
+        shutil.rmtree(nodeoutdir)
+        run_cmdv(node, shlex.split(f'ssh {host} sh /etc/confluent/functions confluentpython {remotedir}/bfb-autoinstall {node} {remotedir}/{bfbbasename} {rshim}'), all, poller, pipedesc)
+
+
+def run_cmdv(node, cmdv, all, poller, pipedesc):
+    try:
+        nopen = subprocess.Popen(
+            cmdv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    except OSError as e:
+        if e.errno == 2:
+            sys.stderr.write('{0}: Unable to find local executable file "{1}"\n'.format(node, cmdv[0]))
+            return
+        raise
+    pipedesc[nopen.stdout.fileno()] = {'node': node, 'popen': nopen,
+                              'type': 'stdout', 'file': nopen.stdout}
+    pipedesc[nopen.stderr.fileno()] = {'node': node, 'popen': nopen,
+                              'type': 'stderr', 'file': nopen.stderr}
+    all.add(nopen.stdout)
+    poller.register(nopen.stdout, select.EPOLLIN)
+    all.add(nopen.stderr)
+    poller.register(nopen.stderr, select.EPOLLIN)
+
+if __name__ == '__main__':
+
+
+    if len(sys.argv) < 3:
+        print(f'Usage: {sys.argv[0]} <host> <bfbfile> <node1:rshim1> [<node2:rshim2> ...]')
+        sys.exit(1)
+
+    host = sys.argv[1]
+    bfbfile = sys.argv[2]
+    nodetorshim = {}
+    for arg in sys.argv[3:]:
+        node, rshim = arg.split(':')
+        nodetorshim[node] = rshim
+
+    installprocs = {}
+    pipedesc = {}
+    all = set()
+    poller = select.epoll()
+
+    exec_bfb_install(host, nodetorshim, bfbfile, installprocs, pipedesc, all, poller)
+    rdy = poller.poll(10)
+    pendingexecs = []
+    exitcode = 0
+    while all:
+        pernodeout = {}
+        for r in rdy:
+            r = r[0]
+            desc = pipedesc[r]
+            r = desc['file']
+            node = desc['node']
+            data = True
+            singlepoller = select.epoll()
+            singlepoller.register(r, select.EPOLLIN)
+            while data and singlepoller.poll(0):
+                data = r.readline()
+                if data:
+                    if desc['type'] == 'stdout':
+                        if node not in pernodeout:
+                            pernodeout[node] = []
+                        pernodeout[node].append(data)
+                    else:
+                        data = client.stringify(data)
+                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        sys.stderr.flush()
+                else:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        poller.unregister(r)
+                        r.close()
+                        if desc['type'] == 'stdout' and pendingexecs:
+                            node, cmdv = pendingexecs.popleft()
+                            run_cmdv(node, cmdv, all, poller, pipedesc)
+            singlepoller.close()
+        for node in sortutil.natural_sort(pernodeout):
+            for line in pernodeout[node]:
+                line = client.stringify(line)
+                sys.stdout.write('{0}: {1}'.format(node, line))
+        sys.stdout.flush()
+        if all:
+            rdy = poller.poll(10)
+
+

confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient

@@ -47,38 +47,113 @@ c_crypt.restype = ctypes.c_char_p
 
 
 def get_my_addresses():
-    nlhdrsz = struct.calcsize('IHHII')
-    ifaddrsz = struct.calcsize('BBBBI')
-    # RTM_GETADDR = 22
-    # nlmsghdr struct: u32 len, u16 type, u16 flags, u32 seq, u32 pid
-    nlhdr = struct.pack('IHHII', nlhdrsz + ifaddrsz, 22, 0x301, 0, 0)
-    # ifaddrmsg struct: u8 family, u8 prefixlen, u8 flags, u8 scope, u32 index
-    ifaddrmsg = struct.pack('BBBBI', 0, 0, 0, 0, 0)
-    s = socket.socket(socket.AF_NETLINK, socket.SOCK_RAW, socket.NETLINK_ROUTE)
-    s.bind((0, 0))
-    s.sendall(nlhdr + ifaddrmsg)
     addrs = []
-    while True:
-        pdata = s.recv(65536)
-        v = memoryview(pdata)
-        if struct.unpack('H', v[4:6])[0] == 3:  # netlink done message
-            break
-        while len(v):
-            length, typ = struct.unpack('IH', v[:6])
-            if typ == 20:
-                fam, plen, _, scope, ridx = struct.unpack('BBBBI', v[nlhdrsz:nlhdrsz+ifaddrsz])
-                if scope in (253, 0):
-                    rta = v[nlhdrsz+ifaddrsz:length]
-                    while len(rta):
-                        rtalen, rtatyp = struct.unpack('HH', rta[:4])
-                        if rtalen < 4:
-                            break
-                        if rtatyp == 1:
-                            addrs.append((fam, rta[4:rtalen], plen, ridx))
-                        rta = rta[msg_align(rtalen):]
-            v = v[msg_align(length):]
+    for ifa in get_ifaddrs():
+        if ifa[0] == 'ip':
+            addrs.append((ifa[1], ifa[2], ifa[3]))
     return addrs
 
+def get_mac_addresses():
+    macs = []
+    for ifa in get_ifaddrs():
+        if ifa[0] == 'ETHER':
+            macs.append((ifa[1], ifa[2]))
+    return macs
+
+def get_ifaddrs():
+    class sockaddr(ctypes.Structure):
+        _fields_ = [
+            ('sa_family', ctypes.c_uint16),
+            ('sa_data', ctypes.c_ubyte * 14),
+        ]
+    
+    class sockaddr_in(ctypes.Structure):
+        _fields_ = [
+            ('sin_family', ctypes.c_uint16),
+            ('sin_port', ctypes.c_uint16),
+            ('sin_addr', ctypes.c_ubyte * 4),
+            ('sin_zero', ctypes.c_ubyte * 8),
+        ]
+    
+    class sockaddr_in6(ctypes.Structure):
+        _fields_ = [
+            ('sin6_family', ctypes.c_uint16),
+            ('sin6_port', ctypes.c_uint16),
+            ('sin6_flowinfo', ctypes.c_uint32),
+            ('sin6_addr', ctypes.c_ubyte * 16),
+            ('sin6_scope_id', ctypes.c_uint32),
+        ]
+    
+    class sockaddr_ll(ctypes.Structure):
+        _fields_ = [
+            ('sll_family', ctypes.c_uint16),
+            ('sll_protocol', ctypes.c_uint16),
+            ('sll_ifindex', ctypes.c_int32),
+            ('sll_hatype', ctypes.c_uint16),
+            ('sll_pkttype', ctypes.c_uint8),
+            ('sll_halen', ctypes.c_uint8),
+            ('sll_addr', ctypes.c_ubyte * 8),
+        ]
+    
+    class ifaddrs(ctypes.Structure):
+        pass
+
+    ifaddrs._fields_ = [
+            ('ifa_next', ctypes.POINTER(ifaddrs)),
+            ('ifa_name', ctypes.c_char_p),
+            ('ifa_flags', ctypes.c_uint),
+            ('ifa_addr', ctypes.POINTER(sockaddr)),
+            ('ifa_netmask', ctypes.POINTER(sockaddr)),
+            ('ifa_ifu', ctypes.POINTER(sockaddr)),
+            ('ifa_data', ctypes.c_void_p),
+        ]
+    
+    libc = ctypes.CDLL(ctypes.util.find_library('c'))
+    libc.getifaddrs.argtypes = [ctypes.POINTER(ctypes.POINTER(ifaddrs))]
+    libc.getifaddrs.restype = ctypes.c_int
+    libc.freeifaddrs.argtypes = [ctypes.POINTER(ifaddrs)]
+    libc.freeifaddrs.restype = None
+    ifap = ctypes.POINTER(ifaddrs)()
+    result = libc.getifaddrs(ctypes.pointer(ifap))
+    if result != 0:
+        return []
+    addresses = []
+    ifa = ifap
+    try:
+        while ifa:
+            if ifa.contents.ifa_addr:
+                family = ifa.contents.ifa_addr.contents.sa_family
+                name = ifa.contents.ifa_name.decode('utf-8') if ifa.contents.ifa_name else None
+                if family in (socket.AF_INET, socket.AF_INET6):
+                    # skip loopback and non-multicast interfaces
+                    if ifa.contents.ifa_flags & 8 or not ifa.contents.ifa_flags & 0x1000:
+                        ifa = ifa.contents.ifa_next
+                        continue
+                if family == socket.AF_INET:
+                    addr_ptr = ctypes.cast(ifa.contents.ifa_addr, ctypes.POINTER(sockaddr_in))
+                    addr_bytes = bytes(addr_ptr.contents.sin_addr)
+                    if_index = socket.if_nametoindex(name) if name else 0
+                    addresses.append(('ip', family, addr_bytes, if_index))
+                elif family == socket.AF_INET6:
+                    addr_ptr = ctypes.cast(ifa.contents.ifa_addr, ctypes.POINTER(sockaddr_in6))
+                    addr_bytes = bytes(addr_ptr.contents.sin6_addr)
+                    scope_id = addr_ptr.contents.sin6_scope_id
+                    addresses.append(('ip', family, addr_bytes, scope_id))          
+                elif family == socket.AF_PACKET:
+                    addr_ptr = ctypes.cast(ifa.contents.ifa_addr, ctypes.POINTER(sockaddr_ll))
+                    halen = addr_ptr.contents.sll_halen
+                    if addr_ptr.contents.sll_hatype in (1, 32) and halen > 0:  # ARPHRD_ETHER or ARPHRD_INFINIBAND
+                        if addr_ptr.contents.sll_hatype == 1 and addr_ptr.contents.sll_addr[0] & 2:  # skip locally administered MACs
+                            ifa = ifa.contents.ifa_next
+                            continue
+                        mac_bytes = bytes(addr_ptr.contents.sll_addr[:halen])
+                        macaddr = ':'.join('{:02x}'.format(b) for b in mac_bytes)
+                        addresses.append(('ETHER', name, macaddr))                
+            ifa = ifa.contents.ifa_next
+    finally:
+        libc.freeifaddrs(ifap)
+    
+    return addresses
 
 def scan_confluents(confuuid=None):
     srvs = {}
@@ -92,22 +167,24 @@ def scan_confluents(confuuid=None):
     s4.bind(('0.0.0.0', 1900))
     doneidxs = set([])
     msg = 'M-SEARCH * HTTP/1.1\r\nST: urn:xcat.org:service:confluent:'
-    if not confuuid:
+    if not confuuid and os.path.exists('/etc/confluent/confluent.deploycfg'):
         with open('/etc/confluent/confluent.deploycfg') as dcfg:
             for line in dcfg.read().split('\n'):
                 if line.startswith('confluent_uuid:'):
                     confluentuuid = line.split(': ')[1]
                     msg += '/confluentuuid=' + confluentuuid
                     break
+    if not confuuid and os.path.exists('/confluent_uuid'):
+        with open('/confluent_uuid') as cuuidin:
+            confluentuuid = cuuidin.read().strip()
+            msg += '/confluentuuid=' + confluentuuid
     try:
         with open('/sys/devices/virtual/dmi/id/product_uuid') as uuidin:
             msg += '/uuid=' + uuidin.read().strip()
     except Exception:
         pass
-    for addrf in glob.glob('/sys/class/net/*/address'):
-        with open(addrf) as addrin:
-            hwaddr = addrin.read().strip()
-            msg += '/mac=' + hwaddr
+    for iface, hwaddr in get_mac_addresses():
+        msg += '/mac=' + hwaddr
     msg = msg.encode('utf8')
     for addr in get_my_addresses():
         if addr[0] == socket.AF_INET6:
@@ -155,7 +232,8 @@ def scan_confluents(confuuid=None):
             if currip.startswith('fe80::') and '%' not in currip:
                 currip = '{0}%{1}'.format(currip, peer[-1])
             srvs[currip] = current
-            srvlist.append(currip)
+            if currip not in srvlist:
+                srvlist.append(currip)
         r = select.select((s4, s6), (), (), 2)
         if r:
             r = r[0]

confluent_osdeploy/common/profile/scripts/confignet

@@ -356,7 +356,7 @@ class NetworkManager(object):
             currteam = deats.get('connection.master', None)
             if currteam == team:
                 return
-            for stg in ('ipv4.dhcp-hostname', 'ipv4.dns', 'ipv6.dns', 'ipv6.dhcp-hostname'):
+            for stg in ('ipv4.dhcp-hostname', 'ipv4.dns', 'ipv6.dns', 'ipv6.dhcp-hostname', 'ipv4.dns-search', 'ipv6.dns-search'):
                 if deats.get(stg, None):
                     bondcfg[stg] = deats[stg]
         if member in self.uuidbyname:
@@ -488,15 +488,16 @@ if __name__ == '__main__':
             continue
         myname = s.getsockname()
         s.close()
+        curridx = None
         if len(myname) == 4:
             curridx = myname[-1]
         else:
             myname = myname[0]
             myname = socket.inet_pton(socket.AF_INET, myname)
             for addr in myaddrs:
-                if myname == addr[1].tobytes():
+                if myname == addr[1]:
                     curridx = addr[-1]
-        if curridx in doneidxs:
+        if curridx is not None and curridx in doneidxs:
             continue
         for tries in (1, 2, 3):
             try:

confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl

@@ -27,12 +27,11 @@ mkdir -p stateless-bin
 cp -a el8bin/* .
 ln -s el8 el9
 ln -s el8 el10
-mv el10/initramfs/usr el10/initramfs/var
 cp -a debian debian13
 mkdir -p debian13/initramfs/usr
 mv debian13/initramfs/lib debian13/initramfs/usr/
 mv el10/initramfs/usr el10/initramfs/var
-for os in rhvh4 el7 genesis el8 suse15 debian debian13 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9 el10; do
+for os in rhvh4 el7 genesis el8 suse15 debian debian13 ubuntu20.04 ubuntu22.04 ubuntu24.04 ubuntu26.04 coreos el9 el10; do
     mkdir ${os}out
     cd ${os}out
     if [ -d ../${os}bin ]; then 
@@ -86,7 +85,10 @@ cp -a esxi7 esxi8
 %install
 mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
 #cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
-for os in rhvh4 el7 el8 el9 el10 debian debian13 genesis suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 esxi6 esxi7 esxi8 coreos; do
+mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/common
+cp common/initramfs/opt/confluent/bin/apiclient %{buildroot}/opt/confluent/lib/osdeploy/common
+
+for os in rhvh4 el7 el8 el9 el10 debian debian13 genesis suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 ubuntu26.04 esxi6 esxi7 esxi8 coreos; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs/aarch64/
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs/aarch64/
     if [ -d ${os}disklessout ]; then

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -33,7 +33,7 @@ cp -a debian debian13
 mkdir -p debian13/initramfs/usr
 mv debian13/initramfs/lib debian13/initramfs/usr/
 mv el10/initramfs/usr el10/initramfs/var
-for os in rhvh4 el7 genesis el8 suse15 debian debian13 ubuntu18.04 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9 el10; do
+for os in rhvh4 el7 genesis el8 suse15 debian debian13 ubuntu18.04 ubuntu20.04 ubuntu22.04 ubuntu24.04 ubuntu26.04 coreos el9 el10; do
     mkdir ${os}out
     cd ${os}out
     if [ -d ../${os}bin ]; then 
@@ -49,7 +49,7 @@ for os in rhvh4 el7 genesis el8 suse15 debian debian13 ubuntu18.04 ubuntu20.04 u
     mv ../addons.cpio .
     cd ..
 done
-for os in el7 el8 suse15 el9 el10 ubuntu20.04 ubuntu22.04 ubuntu24.04; do
+for os in el7 el8 suse15 el9 el10 ubuntu20.04 ubuntu22.04 ubuntu24.04 ubuntu26.04; do
     mkdir ${os}disklessout
     cd ${os}disklessout
     if [ -d ../${os}bin ]; then
@@ -89,7 +89,7 @@ cp -a esxi7 esxi9
 %install
 mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
 cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
-for os in rhvh4 el7 el8 el9 el10 genesis suse15 ubuntu20.04 debian debian13 ubuntu18.04 ubuntu22.04 ubuntu24.04 esxi6 esxi7 esxi8 esxi9 coreos; do
+for os in rhvh4 el7 el8 el9 el10 genesis suse15 ubuntu20.04 debian debian13 ubuntu18.04 ubuntu22.04 ubuntu24.04 ubuntu26.04 esxi6 esxi7 esxi8 esxi9 coreos; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs

confluent_osdeploy/el10-diskless/profiles/default/scripts/imageboot.sh

@@ -3,8 +3,8 @@ confluent_whost=$confluent_mgr
 if [[ "$confluent_whost" == *:* ]] && [[ "$confluent_whost" != "["* ]]; then
     confluent_whost="[$confluent_mgr]"
 fi
-mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
-if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
+mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay /sysroot
+if [ "untethered" = "$(getarg confluent_imagemethod)" -o "uncompressed" = "$(getarg confluent_imagemethod)" ]; then
     mount -t tmpfs untethered /mnt/remoteimg
     curl https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
 else
@@ -40,20 +40,53 @@ fi
 
 
 #mount -t tmpfs overlay /mnt/overlay
-modprobe zram
-memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
-memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
-echo $memtot > /sys/block/zram0/disksize
-mkfs.xfs /dev/zram0 > /dev/null
-mount -o discard /dev/zram0 /mnt/overlay
-if [ ! -f /tmp/mountparts.sh ]; then
-    mkdir -p /mnt/overlay/upper /mnt/overlay/work
-    mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
+if [ ! "uncompressed" = "$(getarg confluent_imagemethod)" ]; then
+    modprobe zram
+    memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
+    memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
+    echo $memtot > /sys/block/zram0/disksize
+    mkfs.xfs /dev/zram0 > /dev/null
+fi
+TETHERED=0
+if [ "untethered" = "$(getarg confluent_imagemethod)" -o "uncompressed" = "$(getarg confluent_imagemethod)" ]; then
+    if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
+        mount -o discard /dev/zram0 /sysroot
+    else
+        mount -t tmpfs disklessroot /sysroot
+    fi
+    echo -en "Decrypting and extracting root filesystem: 0%\r"
+    srcsz=$(du -sk /mnt/remote | awk '{print $1}')
+    while [ -f /mnt/remoteimg/rootimg.sfs ]; do
+        dstsz=$(du -sk /sysroot | awk '{print $1}')
+        pct=$((dstsz * 100 / srcsz))
+        if [ $pct -gt 99 ]; then
+            pct=99
+        fi
+        echo -en "Decrypting and extracting root filesystem: $pct%\r"
+        sleep 0.25
+    done &
+    cp -ax /mnt/remote/* /sysroot/
+    umount /mnt/remote
+    if [ -e /dev/mapper/cryptimg ]; then
+        dmsetup remove cryptimg
+    fi
+    losetup -d $loopdev
+    rm /mnt/remoteimg/rootimg.sfs
+    umount /mnt/remoteimg
+    wait
+    echo -e "Decrypting and extracting root filesystem: 100%"
 else
-    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
-        mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
-        mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
-    done
+    TETHERED=1
+    mount -o discard /dev/zram0 /mnt/overlay
+    if [ ! -f /tmp/mountparts.sh ]; then
+        mkdir -p /mnt/overlay/upper /mnt/overlay/work
+        mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
+    else
+        for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
+            mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
+            mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
+        done
+    fi
 fi
 mkdir -p /sysroot/etc/ssh
 mkdir -p /sysroot/etc/confluent
@@ -109,7 +142,7 @@ echo '    EnableSSHKeysign yes' >> $sshconf
 echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
 curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_whost/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
 cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
-chmod 640 /sysroot/etc/ssh/*_key
+chmod 600 /sysroot/etc/ssh/*_key
 cp /tls/*.pem /sysroot/etc/pki/ca-trust/source/anchors/
 chroot /sysroot/ update-ca-trust
 curl -sf https://$confluent_whost/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
@@ -129,10 +162,25 @@ ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
 mv /lib/firmware /lib/firmware-ramfs
 ln -s /sysroot/lib/firmware /lib/firmware
 kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
+if [ $TETHERED -eq 1 ]; then
+    (
+        sleep 86400 &
+        ONBOOTPID=$!
+        mkdir -p /run/confluent
+        echo $ONBOOTPID > /run/confluent/onboot_sleep.pid
+        wait $ONBOOTPID
+        dd if=/mnt/remoteimg/rootimg.sfs iflag=nocache count=0 >& /dev/null
+        rm -rf /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs /lib/firmware-ramfs /usr/lib64/libcrypto.so* /usr/lib64/systemd/ /kernel/ /usr/bin/ /usr/sbin/ /usr/libexec/
+    ) &
+    while [ ! -f /run/confluent/onboot_sleep.pid ]; do
+        sleep 0.1
+    done
+else
+    rm -rf /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs /lib/firmware-ramfs /usr/lib64/libcrypto.so* /usr/lib64/systemd/ /kernel/ /usr/bin/ /usr/sbin/ /usr/libexec/
+fi
 if grep debugssh /proc/cmdline >& /dev/null; then
     exec /opt/confluent/bin/start_root
 else
-    rm -rf /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs /lib/firmware-ramfs /usr/lib64/libcrypto.so* /usr/lib64/systemd/ /kernel/ /usr/bin/ /usr/sbin/ /usr/libexec/
     exec /opt/confluent/bin/start_root -s  # share mount namespace, keep kernel callbacks intact
 fi
 

confluent_osdeploy/el8-diskless/profiles/default/scripts/imageboot.sh

@@ -4,10 +4,12 @@ if [[ "$confluent_whost" == *:* ]] && [[ "$confluent_whost" != "["* ]]; then
     confluent_whost="[$confluent_mgr]"
 fi
 mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
+TETHERED=0
 if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
     mount -t tmpfs untethered /mnt/remoteimg
     curl https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
 else
+    TETHERED=1
     confluent_urls="$confluent_urls https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs"
     /opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg
 fi
@@ -130,4 +132,17 @@ ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
 mv /lib/firmware /lib/firmware-ramfs
 ln -s /sysroot/lib/firmware /lib/firmware
 kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
+if [ $TETHERED -eq 1 ]; then
+    (
+        sleep 86400 &
+        ONBOOTPID=$!
+        mkdir -p /run/confluent
+        echo $ONBOOTPID > /run/confluent/onboot_sleep.pid
+        wait $ONBOOTPID
+        dd if=/mnt/remoteimg/rootimg.sfs iflag=nocache count=0 >& /dev/null
+    ) &
+    while [ ! -f /run/confluent/onboot_sleep.pid ]; do
+        sleep 0.1
+    done
+fi
 exec /opt/confluent/bin/start_root

confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh

@@ -68,5 +68,14 @@ run_remote_parts onboot.d
 # Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/
 run_remote_config onboot.d
 
+if [ -f /run/confluent/onboot_sleep.pid ]; then
+    loopdev=$(losetup -j /mnt/remoteimg/rootimg.sfs|cut -d: -f 1)
+    if [ -n "$loopdev" ]; then
+        losetup "$loopdev" --direct-io=on
+    fi
+    sleeppid=$(cat /run/confluent/onboot_sleep.pid)
+    kill "$sleeppid"
+    rm -f /run/confluent/onboot_sleep.pid
+fi
 #curl -X POST -d 'status: booted' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
 kill $logshowpid

confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient

@@ -277,7 +277,10 @@ def synchronize():
                             try:
                                 uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid
                             except KeyError:
-                                uid = opts[fname][opt]['id']
+                                try:
+                                    uid = opts[fname][opt]['id']
+                                except KeyError:
+                                    raise Exception(f"Unable to map owner of {fname}") 
                         elif opt == 'group':
                             try:
                                 gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid

confluent_osdeploy/el8/profiles/default/initprofile.sh

@@ -3,7 +3,7 @@ sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/; s/oraclelinux/Oracle
 if grep Fedora $2/profile.yaml > /dev/null; then
 	sed -i 's/@^minimal-environment/#/' $2/packagelist
 fi
-if grep ^label: $2/profile.yaml | grep 10 > /dev/null; then
+if grep ^label: $2/profile.yaml | grep ' 10' > /dev/null; then
 	echo 'echo openssh-keysign >> /tmp/addonpackages' > $2/scripts/pre.d/enablekeysign
 	chmod 644 $2/scripts/pre.d/enablekeysign
 fi

confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py

@@ -467,7 +467,7 @@ def install_to_disk(imgpath):
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', vgname, lvmpart])
         vgroupmap = {}
-        if yaml and vgmap:
+        if yaml and vgmap and os.path.exists('/tmp/volumegroupmap.yml'):
             with open('/tmp/volumegroupmap.yml') as mapin:
                 vgroupmap = yaml.safe_load(mapin)
         donedisks = {}

confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh

@@ -3,8 +3,8 @@ confluent_whost=$confluent_mgr
 if [[ "$confluent_whost" == *:* ]] && [[ "$confluent_whost" != "["* ]]; then
     confluent_whost="[$confluent_mgr]"
 fi
-mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
-if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
+mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay /sysroot
+if [ "untethered" = "$(getarg confluent_imagemethod)" -o "uncompressed" = "$(getarg confluent_imagemethod)" ]; then
     mount -t tmpfs untethered /mnt/remoteimg
     curl https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
 else
@@ -40,20 +40,53 @@ fi
 
 
 #mount -t tmpfs overlay /mnt/overlay
-modprobe zram
-memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
-memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
-echo $memtot > /sys/block/zram0/disksize
-mkfs.xfs /dev/zram0 > /dev/null
-mount -o discard /dev/zram0 /mnt/overlay
-if [ ! -f /tmp/mountparts.sh ]; then
-    mkdir -p /mnt/overlay/upper /mnt/overlay/work
-    mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
+if [ ! "uncompressed" = "$(getarg confluent_imagemethod)" ]; then
+    modprobe zram
+    memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
+    memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
+    echo $memtot > /sys/block/zram0/disksize
+    mkfs.xfs /dev/zram0 > /dev/null
+fi
+TETHERED=0
+if [ "untethered" = "$(getarg confluent_imagemethod)" -o "uncompressed" = "$(getarg confluent_imagemethod)" ]; then
+    if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
+        mount -o discard /dev/zram0 /sysroot
+    else
+        mount -t tmpfs disklessroot /sysroot
+    fi
+    echo -en "Decrypting and extracting root filesystem: 0%\r"
+    srcsz=$(du -sk /mnt/remote | awk '{print $1}')
+    while [ -f /mnt/remoteimg/rootimg.sfs ]; do
+        dstsz=$(du -sk /sysroot | awk '{print $1}')
+        pct=$((dstsz * 100 / srcsz))
+        if [ $pct -gt 99 ]; then
+            pct=99
+        fi
+        echo -en "Decrypting and extracting root filesystem: $pct%\r"
+        sleep 0.25
+    done &
+    cp -ax /mnt/remote/* /sysroot/
+    umount /mnt/remote
+    if [ -e /dev/mapper/cryptimg ]; then
+        dmsetup remove cryptimg
+    fi
+    losetup -d $loopdev
+    rm /mnt/remoteimg/rootimg.sfs
+    umount /mnt/remoteimg
+    wait
+    echo -e "Decrypting and extracting root filesystem: 100%"
 else
-    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
-        mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
-        mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
-    done
+    TETHERED=1
+    mount -o discard /dev/zram0 /mnt/overlay
+    if [ ! -f /tmp/mountparts.sh ]; then
+        mkdir -p /mnt/overlay/upper /mnt/overlay/work
+        mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
+    else
+        for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
+            mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
+            mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
+        done
+    fi
 fi
 mkdir -p /sysroot/etc/ssh
 mkdir -p /sysroot/etc/confluent
@@ -109,8 +142,10 @@ echo '    EnableSSHKeysign yes' >> $sshconf
 echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
 curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_whost/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
 cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
-chmod 640 /sysroot/etc/ssh/*_key
-chroot /sysroot chgrp ssh_keys /etc/ssh/*_key
+if grep ^ssh_keys: /etc/group > /dev/null; then
+    chmod 640 /sysroot/etc/ssh/*_key
+    chroot /sysroot chgrp ssh_keys /etc/ssh/*_key
+fi
 cp /tls/*.pem /sysroot/etc/pki/ca-trust/source/anchors/
 chroot /sysroot/ update-ca-trust
 curl -sf https://$confluent_whost/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
@@ -129,10 +164,37 @@ mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
 mv /lib/firmware /lib/firmware-ramfs
 ln -s /sysroot/lib/firmware /lib/firmware
+rm -f /sysroot/etc/dracut.conf.d/diskless.conf # remove diskless dracut from runtime, to make kdump happier
 kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
+if grep debugssh /proc/cmdline >& /dev/null; then
+    debugssh=1
+else
+    debugssh=0
+fi
+if [ $TETHERED -eq 1 ]; then
+    # In tethered mode, the double-caching is useful to get through tricky part of
+    # onboot with confignet. After that, it's excessive cache usage.
+    # Give the onboot script a hook to have us come in and enable directio to the
+    # squashfs and drop the cache of the rootimg so far
+    (
+        sleep 86400 &
+        ONBOOTPID=$!
+        mkdir -p /run/confluent
+        echo $ONBOOTPID > /run/confluent/onboot_sleep.pid
+        wait $ONBOOTPID
+        dd if=/mnt/remoteimg/rootimg.sfs iflag=nocache count=0 >& /dev/null
+        if [ $debugssh -eq 0 ]; then
+            rm -rf /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs /lib/firmware-ramfs /usr/lib64/libcrypto.so* /usr/lib64/systemd/ /kernel/ /usr/bin/ /usr/sbin/ /usr/libexec/
+        fi
+    ) &
+    while [ ! -f /run/confluent/onboot_sleep.pid ]; do
+        sleep 0.1
+    done
+elif [ $debugssh -eq 0 ]; then
+    rm -rf /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs /lib/firmware-ramfs /usr/lib64/libcrypto.so* /usr/lib64/systemd/ /kernel/ /usr/bin/ /usr/sbin/ /usr/libexec/
+fi
 if grep debugssh /proc/cmdline >& /dev/null; then
     exec /opt/confluent/bin/start_root
 else
-    rm -rf /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs /lib/firmware-ramfs /usr/lib64/libcrypto.so* /usr/lib64/systemd/ /kernel/ /usr/bin/ /usr/sbin/ /usr/libexec/
     exec /opt/confluent/bin/start_root -s  # share mount namespace, keep kernel callbacks intact
 fi

confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh

@@ -62,5 +62,15 @@ run_remote_parts onboot.d
 # Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/
 run_remote_config onboot.d
 
+if [ -f /run/confluent/onboot_sleep.pid ]; then
+    loopdev=$(losetup -j /mnt/remoteimg/rootimg.sfs|cut -d: -f 1)
+    if [ -n "$loopdev" ]; then
+        losetup "$loopdev" --direct-io=on
+    fi
+    sleeppid=$(cat /run/confluent/onboot_sleep.pid)
+    kill "$sleeppid"
+    rm -f /run/confluent/onboot_sleep.pid
+fi
+
 #curl -X POST -d 'status: booted' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
 kill $logshowpid

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/getinstalldisk

@@ -96,22 +96,36 @@ def list_disks():
                 adp = ' '.join(line.split()[1:])
                 disks[dev]['adapter'] = adp
                 devbyadp.setdefault(adp, []).append(dev)
+    adapterlist = subprocess.check_output(['localcli', 'storage', 'core', 'adapter', 'list'])
+    if not isinstance(adapterlist, str):
+        adapterlist = adapterlist.decode('utf8')
+    driverbyadp = {}
+    linenum = 0
+    for line in adapterlist.split('\n'):
+        linenum += 1
+        if not line.strip():
+            continue
+        if linenum < 3:
+            continue
+        parts = line.split()
+        if len(parts) < 2:
+            continue
+        adp = parts[0]
+        driver = parts[1]
+        driverbyadp[adp] = driver
     for adp in devbyadp:
-        adaplist = subprocess.check_output(['localcli', 'storage', 'core', 'adapter', 'listdetailed', '--adapter', adp])
-        if not isinstance(adaplist, str):
-            adaplist = adaplist.decode('utf8')
-        for line in adaplist.split('\n'):
-            if not line.strip():
-                continue
-            if 'Driver Name:' in line:
-                driver = ' '.join(line.split()[2:])
-                for dev in devbyadp[adp]:
-                    disks[dev]['adapter_driver'] = driver
+        driver = driverbyadp.get(adp, 'Unknown')
+        for dev in devbyadp[adp]:
+            disks[dev]['adapter_driver'] = driver
     return disks
         
 def main():
     disks = []
-    alldisks = list_disks()
+    try:
+        alldisks = list_disks()
+    except Exception as e:
+        print("Error listing disks: {0}".format(str(e)))
+        alldisks = {}
     for disk in alldisks:
         try:
             disks.append(DiskInfo(disk, alldisks[disk]))
@@ -124,6 +138,11 @@ def main():
         with open('/tmp/storagecfg', 'w') as sc:
             sc.write(f'clearpart --all --drives={nd[0]} --overwritevmfs\n')
             sc.write(f'install --drive={nd[0]} --overwritevmfs\n')
+    else:
+        with open('/tmp/storagecfg', 'w') as sc:
+            sc.write(f'clearpart --firstdisk --overwritevmfs\n')
+            sc.write(f'install --firstdisk --overwritevmfs\n')
+
 
 if __name__ == '__main__':
     main()

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/makeksnet

@@ -45,7 +45,10 @@ try:
         cfg['ipv4_gateway'] = ncfg['ipv4_gateway']
 except Exception:
     pass
-netline = 'network --hostname={0} --bootproto={1}'.format(nodename, cfg['ipv4_method'])
+if cfg['ipv4_method'] == 'static':
+    netline = 'network --hostname={0} --bootproto={1}'.format(nodename, cfg['ipv4_method'])
+else:
+    netline = 'network --bootproto=dhcp'
 if vmnic:
     netline += ' --device={0}'.format(vmnic)
 if cfg['ipv4_method'] == 'static':

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/modinstall

@@ -5,6 +5,7 @@ mv /etc/confluent/confluent.deploycfg /etc/confluent/confluent.newdeploycfg
 mv /etc/confluent/confluent.newdeploycfg /etc/confluent/confluent.deploycfg
 chmod +x /tmp/makeksnet
 /tmp/makeksnet > /tmp/ksnet
+localcli system hostname set --host $node
 python3 /tmp/getinstalldisk
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg|sed -e 's/^rootpassword: //')
 echo rootpw --iscrypted $rootpw > /tmp/rootpw

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -56,7 +56,7 @@ cat /tls/*.pem > /etc/pki/tls/certs/ca-bundle.crt
 TRIES=0
 touch /etc/confluent/confluent.info
 TRIES=5
-echo -n "Waitiing for disks..."
+echo -n "Waiting for disks..."
 while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
     sleep 1
     TRIES=$((TRIES - 1))

confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent

@@ -37,10 +37,20 @@ else
         chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
         APIKEY=$(cat /root/custom-installation/confluent/confluent.apikey)
     done
-    MGR=[$MGR]
-    nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
-    nic=$(ip link |grep ^$nic:|awk '{print $2}')
-    DEVICE=${nic%:}
+    if echo "$MGR" | grep -q ':'; then
+        # IPv6 manager: wrap address in brackets and resolve interface from scoped manager entry.
+        MGR=[$MGR]
+        nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
+        nic=$(ip link |grep ^$nic:|awk '{print $2}')
+        DEVICE=${nic%:}
+    else
+        # IPv4 routed deployment: use previously detected NIC, fallback to route lookup.
+        if [ -f /tmp/autodetectnic ]; then
+            DEVICE=$(cat /tmp/autodetectnic)
+        else
+            DEVICE=$(ip route get ${MGR} 2>/dev/null | head -1 | sed -n 's/.*dev \([^ ]*\).*/\1/p')
+        fi
+    fi
     IP=done
 fi
 if [ -z "$MGTIFACE" ]; then

confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent

@@ -97,6 +97,62 @@ while ! grep NODENAME /custom-installation/confluent/confluent.info; do
         echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
         cd -
         umount $tmnt
+    elif confluentsrv=$(sed -n 's/.*confluent=\([^ ]*\).*/\1/p' /proc/cmdline); [ ! -z "$confluentsrv" ]; then
+        echo "confluent= kernel arg found: $confluentsrv" > /dev/console 2>&1
+        . /scripts/functions
+
+        rmmod cdc_ether 2> /dev/null
+        rm -rf /run/net* /run/dhcpcd /var/lib/dhcpcd
+        for dev in $(ip a|grep MULTICAST|awk '{print $2}'|sed -e s/://); do
+            ip a flush $dev
+            echo 1 > /proc/sys/net/ipv6/conf/$dev/addr_gen_mode 2>/dev/null
+            echo 0 > /proc/sys/net/ipv6/conf/$dev/addr_gen_mode 2>/dev/null
+        done
+        unset DEVICE DEVICE6 IP IP6 dev
+
+        echo "Starting DHCP configure_networking..." > /dev/console 2>&1
+        configure_networking
+        echo "DHCP done, DEVICE=$DEVICE" > /dev/console 2>&1
+        echo $DEVICE > /tmp/autodetectnic
+
+        RETRIES=0
+        while [ $RETRIES -lt 5 ]; do
+            if openssl s_client -connect $confluentsrv:443 </dev/null > /dev/null 2>&1; then
+                echo "TLS connectivity to $confluentsrv OK" > /dev/console 2>&1
+                break
+            fi
+            RETRIES=$((RETRIES + 1))
+            echo "Cannot reach $confluentsrv:443, retry $RETRIES/5..." > /dev/console 2>&1
+            sleep 3
+        done
+
+        if [ $RETRIES -ge 5 ]; then
+            echo "Failed to reach $confluentsrv after 5 retries, falling back to copernicus" > /dev/console 2>&1
+            /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+            continue
+        fi
+
+        myids="uuid=$(cat /sys/devices/virtual/dmi/id/product_uuid)"
+        for mac in $(ip link | grep 'link/ether' | awk '{print $2}'); do
+            myids="$myids/mac=$mac"
+        done
+        echo "Calling whoami with IDs: $myids" > /dev/console 2>&1
+
+        myname=$( (printf "GET /confluent-api/self/whoami HTTP/1.0\r\nHost: $confluentsrv\r\nCONFLUENT_IDS: $myids\r\n\r\n"; sleep 3) \
+            | openssl s_client -connect $confluentsrv:443 -quiet 2>/dev/null \
+            | tail -1 | tr -d '\r\n')
+
+        echo "whoami returned: '$myname'" > /dev/console 2>&1
+
+        if [ ! -z "$myname" ]; then
+            MGR=$confluentsrv
+            echo "NODENAME: $myname" > /custom-installation/confluent/confluent.info
+            echo "MANAGER: $confluentsrv" >> /custom-installation/confluent/confluent.info
+            echo "EXTMGRINFO: $confluentsrv||1" >> /custom-installation/confluent/confluent.info
+        else
+            echo "whoami returned empty, retrying in 10s..." > /dev/console 2>&1
+            sleep 10
+        fi
     else
         /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
     fi

confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh

@@ -1,16 +1,16 @@
 #!/bin/bash
 set -e
 sed -i 's/label: ubuntu/label: Ubuntu/' $2/profile.yaml
-if [ -e $1/casper/hwe-vmlinuz ]; then
-    ln -s $1/casper/hwe-vmlinuz $2/boot/kernel
-else
+#if [ -e $1/casper/hwe-vmlinuz ]; then
+#    ln -s $1/casper/hwe-vmlinuz $2/boot/kernel
+#else
     ln -s $1/casper/vmlinuz $2/boot/kernel
-fi
-if [ -e $1/casper/hwe-initrd ]; then
-    ln -s $1/casper/hwe-initrd $2/boot/initramfs/distribution
-else
+#fi
+#if [ -e $1/casper/hwe-initrd ]; then
+#    ln -s $1/casper/hwe-initrd $2/boot/initramfs/distribution
+#else
     ln -s $1/casper/initrd $2/boot/initramfs/distribution
-fi
+#fi
 mkdir -p $2/boot/efi/boot
 if [ -d $1/EFI/boot/ ]; then
     ln -s $1/EFI/boot/* $2/boot/efi/boot

confluent_osdeploy/ubuntu26.04

@@ -0,0 +1 @@
+ubuntu22.04/

confluent_osdeploy/ubuntu26.04-diskless

@@ -0,0 +1 @@
+ubuntu20.04-diskless/

confluent_osdeploy/utils/autocons.c

@@ -7,6 +7,7 @@
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
+#include <dirent.h>
 
 #define COM1 0x3f8
 #define COM2 0x2f8
@@ -19,6 +20,137 @@
 #define SPEED57600 6
 #define SPEED115200 7
 
+typedef struct {
+    char devnode[32];
+    speed_t speed;
+    int valid;
+} serial_port_t;
+
+serial_port_t process_spcr() {
+    serial_port_t result = {0};
+    char buff[128];
+    int fd;
+    uint64_t address;
+    int currspeed;
+    
+    result.valid = 0;
+    
+    fd = open("/sys/firmware/acpi/tables/SPCR", O_RDONLY);
+    if (fd < 0) {
+        return result;
+    }
+    
+    if (read(fd, buff, 80) < 80) {
+        close(fd);
+        return result;
+    }
+    close(fd);
+    
+    if (buff[8] != 2) return result; // revision 2
+    if (buff[36] != 0) return result; // 16550 only
+    if (buff[40] != 1) return result; // IO only
+    
+    address = *(uint64_t *)(buff + 44);
+    currspeed = buff[58];
+    
+    if (address == COM1) {
+        strncpy(result.devnode, "/dev/ttyS0", sizeof(result.devnode));
+    } else if (address == COM2) {
+        strncpy(result.devnode, "/dev/ttyS1", sizeof(result.devnode));
+    } else if (address == COM3) {
+        strncpy(result.devnode, "/dev/ttyS2", sizeof(result.devnode));
+    } else if (address == COM4) {
+        strncpy(result.devnode, "/dev/ttyS3", sizeof(result.devnode));
+    } else {
+        return result;
+    }
+    
+    if (currspeed == SPEED9600) {
+        result.speed = B9600;
+    } else if (currspeed == SPEED19200) {
+        result.speed = B19200;
+    } else if (currspeed == SPEED57600) {
+        result.speed = B57600;
+    } else if (currspeed == SPEED115200) {
+        result.speed = B115200;
+    } else {
+        return result;
+    }
+    
+    result.valid = 1;
+    return result;
+}
+
+serial_port_t identify_by_sys_vendor() {
+    serial_port_t result = {0};
+    char buff[128];
+    FILE *f;
+    
+    f = fopen("/sys/devices/virtual/dmi/id/sys_vendor", "r");
+    if (f) {
+        if (fgets(buff, sizeof(buff), f)) {
+            if (strstr(buff, "Supermicro")) {
+                strncpy(result.devnode, "/dev/ttyS1", sizeof(result.devnode));
+                result.speed = B115200;
+                result.valid = 1;
+            }
+        }
+        fclose(f);
+    }
+    return result;
+}
+
+serial_port_t search_serial_ports() {
+    serial_port_t result = {0};
+    DIR *dir;
+    struct dirent *entry;
+    int fd;
+    int status;
+    int numfound= 0;
+    int numpossible = 0;
+
+    dir = opendir("/dev");
+    if (!dir) {
+        return result;
+    }
+    
+    while ((entry = readdir(dir)) != NULL) {
+        if (strncmp(entry->d_name, "ttyS", 4) != 0) {
+            continue;
+        }
+        
+        char devpath[64];
+        snprintf(devpath, sizeof(devpath), "/dev/%s", entry->d_name);
+        
+        fd = open(devpath, O_RDWR | O_NOCTTY | O_NONBLOCK);
+        if (fd < 0) {
+            continue;
+        }
+        
+        if (ioctl(fd, TIOCMGET, &status) == 0) {
+            numpossible++;
+            if (numfound < 1) {
+                strncpy(result.devnode, devpath, sizeof(result.devnode));
+                result.speed = B115200;
+            }
+            if (status & TIOCM_CAR) {
+                strncpy(result.devnode, devpath, sizeof(result.devnode));
+                numfound++;
+                result.speed = B115200;
+                
+            }
+        }
+        
+        close(fd);
+    }
+    
+    closedir(dir);
+    if (numfound == 1 || numpossible == 1) {
+        result.valid = 1;
+    }
+    return result;
+}
+
 int main(int argc, char* argv[]) {
     struct termios tty;
     struct termios tty2;
@@ -36,46 +168,38 @@ int main(int argc, char* argv[]) {
     char* offset;
     uint64_t address;
     bufflen = 0;
-    tmpi = open("/sys/firmware/acpi/tables/SPCR", O_RDONLY);
-    if (tmpi < 0) {
-        exit(0);
-    }
-    if (read(tmpi, buff, 80) < 80) {
-        exit(0);
-    }
-    close(tmpi);
-    if (buff[8] != 2) exit(0); //revision 2
-    if (buff[36] != 0) exit(0); //16550 only
-    if (buff[40] != 1) exit(0); //IO only
-    address = *(uint64_t *)(buff + 44);
-    currspeed = buff[58];
-    offset = buff + 10;
-    if (address == COM1) {
-        strncpy(buff, "/dev/ttyS0", 128);
-    } else if (address == COM2) {
-        strncpy(buff, "/dev/ttyS1", 128);
-    } else if (address == COM3) {
-        strncpy(buff, "/dev/ttyS2", 128);
-    } else if (address == COM4) {
-        strncpy(buff, "/dev/ttyS3", 128);
-    } else {
+    #ifndef __x86_64__
+        // Only x86 needs autoconsole, other platforms have reasonable default serial console
+        exit(0);
+    #endif
+    serial_port_t spcr = process_spcr();
+    if (!spcr.valid) {
+        spcr = search_serial_ports();
+    }
+    if (!spcr.valid) {
+        spcr = identify_by_sys_vendor();
+    }
+    if (!spcr.valid) {
         exit(0);
     }
+    strncpy(buff, spcr.devnode, sizeof(buff));
+    offset = strchr(buff, 0);
+    currspeed = spcr.speed;
     ttyf = open(buff, O_RDWR | O_NOCTTY);
     if (ttyf < 0) {
         fprintf(stderr, "Unable to open tty\n");
         exit(1);
     }
-    if (currspeed == SPEED9600) {
+    if (currspeed == B9600) {
         cspeed = B9600;
         strncpy(offset, ",9600", 6);
-    } else if (currspeed == SPEED19200) {
+    } else if (currspeed == B19200) {
         cspeed = B19200;
         strncpy(offset, ",19200", 7);
-    } else if (currspeed == SPEED57600) {
+    } else if (currspeed == B57600) {
         cspeed = B57600;
         strncpy(offset, ",57600", 7);
-    } else if (currspeed == SPEED115200) {
+    } else if (currspeed == B115200) {
         cspeed = B115200;
         strncpy(offset, ",115200", 8);
     } else {

confluent_server/confluent/certutil.py

@@ -1,11 +1,22 @@
 import os
+if __name__ == '__main__':
+    import sys
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.realpath(os.path.join(path, '..'))
+    if path.startswith('/opt'):
+        sys.path.append(path)
 import confluent.collective.manager as collective
 import confluent.util as util
 from os.path import exists
+import datetime
 import shutil
 import socket
 import eventlet.green.subprocess as subprocess
 import tempfile
+try:
+    import cryptography.x509 as x509
+except ImportError:
+    x509 = None
 
 def mkdirp(targ):
     try:
@@ -178,6 +189,17 @@ def assure_tls_ca():
         os.symlink(certname, hashname)
     finally:
         os.seteuid(ouid)
+    return certout
+
+#def is_self_signed(pem):
+#    cert = ssl.PEM_cert_to_DER_cert(pem)
+#    return cert.get('subjectAltName', []) == cert.get('issuer', [])
+# x509 certificate issuer subject comparison..
+#>>> b.issuer
+#<Name(C=US,ST=NC,L=RTP,O=Lenovo,CN=XCC-7D9D-J102MM2T)>
+#>>> b.subject
+#<Name(C=US,ST=NC,L=RTP,O=Lenovo,CN=XCC-7D9D-J102MM2T)>
+
 
 def substitute_cfg(setting, key, val, newval, cfgfile, line):
     if key.strip() == setting:
@@ -222,7 +244,7 @@ def create_full_ca(certout):
                         cfgfile.write(line.strip() + '\n')
                     continue
                 cfgfile.write(line.strip() + '\n')
-            cfgfile.write('\n[CACert]\nbasicConstraints = CA:true\n\n[ca_confluent]\n')
+            cfgfile.write('\n[CACert]\nbasicConstraints = critical,CA:true\nkeyUsage = critical,keyCertSign,cRLSign\n[ca_confluent]\n')
     subprocess.check_call(
         ['openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
         keyout])
@@ -231,7 +253,7 @@ def create_full_ca(certout):
     subprocess.check_call(
         ['openssl', 'ca', '-config', newcfg, '-batch', '-selfsign',
         '-extensions', 'CACert', '-extfile', newcfg, 
-        '-notext', '-startdate',
+        '-notext', '-md', 'sha384', '-startdate',
          '19700101010101Z', '-enddate', '21000101010101Z', '-keyfile',
          keyout, '-out', '/etc/confluent/tls/ca/cacert.pem', '-in', csrout]
     )
@@ -257,7 +279,7 @@ def create_simple_ca(keyout, certout):
         if len(subj) > 68:
             subj = subj[:68]
         with open(tmpconfig, 'a') as cfgfile:
-            cfgfile.write('\n[CACert]\nbasicConstraints = CA:true\n')
+            cfgfile.write('\n[CACert]\nbasicConstraints = critical,CA:true\n')
         subprocess.check_call([
                 'openssl', 'req', '-new', '-x509', '-key', keyout, '-days',
                 '27300', '-out', certout, '-subj', subj,
@@ -266,70 +288,112 @@ def create_simple_ca(keyout, certout):
     finally:
         os.remove(tmpconfig)
 
-def create_certificate(keyout=None, certout=None, csrout=None):
-    if not keyout:
+def create_certificate(keyout=None, certout=None, csrfile=None, subj=None, san=None, backdate=True, days=None):
+    now_utc = datetime.datetime.now(datetime.timezone.utc)
+    if backdate:
+        # To deal with wildly off clocks, we backdate certificates.
+        startdate = '20000101010101Z'
+    else:
+        # apply a mild backdate anyway, even if these are supposed to be for more accurate clocks
+        startdate = (now_utc - datetime.timedelta(hours=24)).strftime('%Y%m%d%H%M%SZ')
+    if days is None:
+        enddate = '21000101010101Z'
+    else:
+        enddate = (now_utc + datetime.timedelta(days=days)).strftime('%Y%m%d%H%M%SZ')
+    tlsmateriallocation = {}
+    if not certout:
         tlsmateriallocation = get_certificate_paths()
         keyout = tlsmateriallocation.get('keys', [None])[0]
         certout = tlsmateriallocation.get('certs', [None])[0]
         if not certout:
             certout = tlsmateriallocation.get('bundles', [None])[0]
-    if not keyout or not certout:
+    if (not keyout and not csrfile) or not certout:
         raise Exception('Unable to locate TLS certificate path automatically')
-    assure_tls_ca()
-    shortname = socket.gethostname().split('.')[0]
-    longname = shortname # socket.getfqdn()
-    if not csrout:
+    cacertname = assure_tls_ca()
+    if not subj:
+        shortname = socket.gethostname().split('.')[0]
+        longname = shortname # socket.getfqdn()
+        subj = '/CN={0}'.format(longname)
+    elif '/CN=' not in subj:
+        subj = '/CN={0}'.format(subj)
+    if not csrfile:
         subprocess.check_call(
             ['openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
              keyout])
-    ipaddrs = list(get_ip_addresses())
-    san = ['IP:{0}'.format(x) for x in ipaddrs]
-    # It is incorrect to put IP addresses as DNS type.  However
-    # there exists non-compliant clients that fail with them as IP
-    # san.extend(['DNS:{0}'.format(x) for x in ipaddrs])
-    dnsnames = set(ipaddrs)
-    dnsnames.add(shortname)
-    for currip in ipaddrs:
-        dnsnames.add(socket.getnameinfo((currip, 0), 0)[0])
-    for currname in dnsnames:
-        san.append('DNS:{0}'.format(currname))
-    #san.append('DNS:{0}'.format(longname))
-    san = ','.join(san)
+    permitdomains = []
+    if x509:
+        # check if this CA has name constraints, and avoid violating them
+        with open(cacertname, 'rb') as f:
+            cer = x509.load_pem_x509_certificate(f.read())
+        for extension in cer.extensions:
+            if extension.oid == x509.ExtensionOID.NAME_CONSTRAINTS:
+                nc = extension.value
+                for pname in nc.permitted_subtrees:
+                    permitdomains.append(pname.value)
+    if not san:
+        ipaddrs = list(get_ip_addresses())
+        if not permitdomains:
+            san = ['IP:{0}'.format(x) for x in ipaddrs]
+            # It is incorrect to put IP addresses as DNS type.  However
+            # there exists non-compliant clients that fail with them as IP
+            # san.extend(['DNS:{0}'.format(x) for x in ipaddrs])
+            dnsnames = set(ipaddrs)
+            dnsnames.add(shortname)
+            dnsnames.add(longname)
+        else:
+            # nameconstraints preclude IP and shortname
+            san = []
+            dnsnames = set()
+            for suffix in permitdomains:
+                if longname.endswith(suffix):
+                    dnsnames.add(longname)
+                    break
+        for currip in ipaddrs:
+            currname = socket.getnameinfo((currip, 0), 0)[0]
+            for suffix in permitdomains:
+                if currname.endswith(suffix):
+                    dnsnames.add(currname)
+                    break
+            if not permitdomains:
+                dnsnames.add(currname)
+        for currname in dnsnames:
+            san.append('DNS:{0}'.format(currname))
+        #san.append('DNS:{0}'.format(longname))
+        san = ','.join(san)
     sslcfg = get_openssl_conf_location()
     tmphdl, tmpconfig = tempfile.mkstemp()
     os.close(tmphdl)
     tmphdl, extconfig = tempfile.mkstemp()
     os.close(tmphdl)
     needcsr = False
-    if csrout is None:
+    if csrfile is None:
         needcsr = True
-        tmphdl, csrout = tempfile.mkstemp()
+        tmphdl, csrfile = tempfile.mkstemp()
         os.close(tmphdl)
     shutil.copy2(sslcfg, tmpconfig)
     try:
+        with open(extconfig, 'a') as cfgfile:
+            cfgfile.write('\nbasicConstraints=critical,CA:false\nkeyUsage=critical,digitalSignature\nextendedKeyUsage=serverAuth,clientAuth\nsubjectAltName={0}'.format(san))
         if needcsr:
             with open(tmpconfig, 'a') as cfgfile:
                 cfgfile.write('\n[SAN]\nsubjectAltName={0}'.format(san))
-            with open(extconfig, 'a') as cfgfile:
-                cfgfile.write('\nbasicConstraints=CA:false\nsubjectAltName={0}'.format(san))
             subprocess.check_call([
-                'openssl', 'req', '-new', '-key', keyout, '-out', csrout, '-subj',
-                '/CN={0}'.format(longname),
-               '-extensions', 'SAN', '-config', tmpconfig
+                'openssl', 'req', '-new', '-key', keyout, '-out', csrfile, '-subj',
+                subj, '-extensions', 'SAN', '-config', tmpconfig
             ])
-        else:
-            # when used manually, allow the csr SAN to stand
-            # may add explicit subj/SAN argument, in which case we would skip copy
-            with open(tmpconfig, 'a') as cfgfile:
-                cfgfile.write('\ncopy_extensions=copy\n')
-            with open(extconfig, 'a') as cfgfile:
-                cfgfile.write('\nbasicConstraints=CA:false\n')
+        #else:
+        #    # when used manually, allow the csr SAN to stand
+        #    # may add explicit subj/SAN argument, in which case we would skip copy
+        #    #with open(tmpconfig, 'a') as cfgfile:
+        #    #    cfgfile.write('\ncopy_extensions=copy\n')
+        #    with open(extconfig, 'a') as cfgfile:
+        #        cfgfile.write('\nbasicConstraints=CA:false\n')
         if os.path.exists('/etc/confluent/tls/cakey.pem'):
             # simple style CA in effect, make a random serial number and
             # hope for the best, and accept inability to backdate the cert
             serialnum = '0x' + ''.join(['{:02x}'.format(x) for x in bytearray(os.urandom(20))])
             subprocess.check_call([
-                'openssl', 'x509', '-req', '-in', csrout,
+                'openssl', 'x509', '-req', '-in', csrfile,
                 '-CA', '/etc/confluent/tls/cacert.pem',
                 '-CAkey', '/etc/confluent/tls/cakey.pem',
                 '-set_serial', serialnum, '-out', certout, '-days', '27300',
@@ -348,12 +412,11 @@ def create_certificate(keyout=None, certout=None, csrout=None):
                 shutil.copy2(cacfgfile, tmpcafile)
                 os.close(tmphdl)
                 cacfgfile = tmpcafile
-            # with realcalock:  # if we put it in server, we must lock it
             subprocess.check_call([
-                'openssl', 'ca', '-config', cacfgfile,
-                '-in', csrout, '-out', certout, '-batch', '-notext',
-                '-startdate', '19700101010101Z', '-enddate', '21000101010101Z',
-                '-extfile', extconfig
+                'openssl', 'ca', '-config', cacfgfile, '-rand_serial',
+                '-in', csrfile, '-out', certout, '-batch', '-notext',
+                '-startdate', startdate, '-enddate', enddate, '-md', 'sha384',
+                '-extfile', extconfig, '-subj', subj
             ])
         for keycopy in tlsmateriallocation.get('keys', []):
             if keycopy != keyout:
@@ -381,18 +444,43 @@ def create_certificate(keyout=None, certout=None, csrout=None):
     finally:
         os.remove(tmpconfig)
         if needcsr:
-            os.remove(csrout)
-        print(extconfig)  # os.remove(extconfig)
+            os.remove(csrfile)
+        os.remove(extconfig)
 
 
 if __name__ == '__main__':
     import sys
+    import ipaddress
     outdir = os.getcwd()
     keyout = os.path.join(outdir, 'key.pem')
-    certout = os.path.join(outdir, sys.argv[2] + 'cert.pem')
+    certout = os.path.join(outdir, 'cert.pem')
     csrout = None
+    subj, san = (None, None)
+    try:
+        bindex = sys.argv.index('-b')
+        bmcnode = sys.argv.pop(bindex + 1)  # Remove bmcnode argument
+        sys.argv.pop(bindex)      # Remove -b flag
+        import confluent.config.configmanager as cfm
+        c = cfm.ConfigManager(None)
+        subj, san = util.get_bmc_subject_san(c, bmcnode)
+    except ValueError:
+        bindex = None
+    if subj is None:
+        try:
+            sans = set()
+            sindex = sys.argv.index('-s')
+            subj = sys.argv.pop(sindex + 1)  # Remove subject argument
+            sys.argv.pop(sindex)      # Remove -s flag
+            try:
+                ipaddress.ip_address(subj)
+                sans.add('IP:{0}'.format(subj))
+            except ValueError:
+                sans.add('DNS:{0}'.format(subj))
+            san = ','.join(sans) if sans else None
+        except ValueError:
+            pass
     try:
         csrout = sys.argv[1]
     except IndexError:
         csrout = None
-    create_certificate(keyout, certout, csrout)
+    create_certificate(keyout, certout, csrout, subj, san, backdate=False, days=3650)

confluent_server/confluent/collective/manager.py

@@ -716,6 +716,7 @@ def become_leader(connection):
     if reassimilate is not None:
         reassimilate.kill()
     reassimilate = eventlet.spawn(reassimilate_missing)
+    cfm._init_indexes()
     cfm._ready = True
     if _assimilate_missing(skipaddr):
         schedule_rebalance()

confluent_server/confluent/config/attributes.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2019 Lenovo
+# Copyright 2015-2025 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -215,6 +215,9 @@ node = {
                         'Using this requires that collective members be '
                         'defined as nodes for noderange expansion')
     },
+    'deployment.client_ip': {
+        'description': ('Client IP used when most recently reporting state.')
+    },
     'deployment.lock': {
         'description': ('Indicates whether deployment actions should be impeded. '
                         'If locked, it indicates that a pending profile should not be applied. '
@@ -405,6 +408,12 @@ node = {
                        'include /<prefixlen> CIDR suffix to indicate subnet length, which is '
                        'autodetected by default where possible.',
     },
+    'hardwaremanagement.manager_tls_name': {
+        'description': 'A name to use in lieu of the value in hardwaremanagement.manager for '
+                       'TLS certificate verification purposes. Some strategies involve a non-IP, '
+                       'non-resolvable name, or this can be used to access by IP while using name-based '
+                       'validation',
+    },
     'hardwaremanagement.method': {
         'description': 'The method used to perform operations such as power '
                        'control, get sensor data, get inventory, and so on. '
@@ -441,6 +450,9 @@ node = {
 #IBM Flex)''',
 #        'appliesto': ['system'],
 #    },
+    'id.index': {
+        'description': 'Confluent generated numeric index for the node.',
+    },
     'id.model': {
         'description': 'The model number of a node.  In scenarios where there '
                        'is both a name and a model number, it is generally '
@@ -466,17 +478,17 @@ node = {
                        'the discovery process to decide where to place the mac address of a detected PXE nic.',
     },
     'net.connection_name': {
-        'description': 'Name to use when specifiying a name for connection and/or interface name for a team.  This may be the name of a team interface, '
+        'description': 'Name to use when specifiying a name for connection and/or interface name for a team/bond.  This may be the name of a team/bond interface, '
                        'the connection name in network manager for the interface, or may be installed as an altname '
                        'as supported by the respective OS deployment profiles.  Default is to accept default name for '
-                       'a team consistent with the respective OS, or to use the matching original port name as connection name.'
+                       'a team/bond consistent with the respective OS, or to use the matching original port name as connection name.'
     },
     'net.interface_names': {
         'description': 'Interface name or comma delimited list of names to match for this interface. It is generally recommended '
                        'to leave this blank unless needing to set up interfaces that are not on a common subnet with a confluent server, '
                        'as confluent servers provide autodetection for matching the correct network definition to an interface. '
                        'This would be the default name per the deployed OS and can be a comma delimited list to denote members of '
-                       'a team or a single interface for VLAN/PKEY connections.'
+                       'a team/bond or a single interface for VLAN/PKEY connections.'
     },
     'net.mtu': {
             'description': 'MTU to apply to this connection',
@@ -562,7 +574,7 @@ node = {
                        'operating system',
     },
     'net.team_mode': {
-        'description': 'Indicates that this interface should be a team and what mode or runner to use when teamed. '
+        'description': 'Indicates that this interface should be a team/bond and what mode or runner to use when teamed or bonded. '
                        'If this covers a deployment interface, one of the member interfaces may be brought up as '
                        'a standalone interface until deployment is complete, as supported by the OS deployment profile. '
                        'To support this scenario, the switch should be set up to allow independent operation of member ports (e.g. lacp bypass mode or fallback mode).',
@@ -596,6 +608,10 @@ node = {
         'description': ('SNMPv1 community string, it is highly recommended to'
                         'step up to SNMPv3'),
     },
+    'snmp.privacyprotocol': {
+        'description': 'The privacy protocol to use for SNMPv3',
+        'valid_values': ('aes', 'des'),
+    },
 #    'secret.snmplocalizedkey': {
 #        'description': ("SNMPv3 key localized to this node's SNMP Engine id"
 #                        'This can be used in lieu of snmppassphrase to avoid'

confluent_server/confluent/config/configmanager.py

@@ -164,6 +164,45 @@ def _mkpath(pathname):
             raise
 
 
+def _count_freeindexes(freeindexes):
+    count = 0
+    for idx in freeindexes:
+        if isinstance(idx, list):
+            for subidx in range(idx[0], idx[1] + 1):
+                count += 1
+        else:
+            count += 1
+    return count
+
+def _is_free_index(freeindexes, idx):
+    for freeidx in freeindexes:
+        if isinstance(freeidx, list):
+            if freeidx[0] <= idx <= freeidx[1]:
+                return True
+        else:
+            if freeidx == idx:
+                return True
+    return False
+
+def _remove_free_index(freeindexes, idx):
+    for i, freeidx in enumerate(freeindexes):
+        if isinstance(freeidx, list):
+            if freeidx[0] <= idx <= freeidx[1]:
+                if freeidx[0] == freeidx[1]:
+                    del freeindexes[i]
+                elif freeidx[0] == idx:
+                    freeindexes[i][0] += 1
+                elif freeidx[1] == idx:
+                    freeindexes[i][1] -= 1
+                else:
+                    freeindexes.insert(i + 1, [idx + 1, freeidx[1]])
+                    freeindexes[i][1] = idx - 1
+                return
+        else:
+            if freeidx == idx:
+                del freeindexes[i]
+                return
+
 def _derive_keys(password, salt):
     #implement our specific combination of pbkdf2 transforms to get at
     #key.  We bump the iterations up because we can afford to
@@ -334,7 +373,7 @@ def _rpc_rename_nodes(tenant, renamemap):
 
 
 def _rpc_rename_nodegroups(tenant, renamemap):
-    ConfigManager(tenant)._true_rename_nodegroups(renamemap)
+    ConfigManager(tenant)._true_rename_groups(renamemap)
 
 
 def _rpc_set_group_attributes(tenant, attribmap, autocreate):
@@ -974,6 +1013,14 @@ def del_collective_member(name):
     if cfgstreams:
         exec_on_followers_unconditional('_true_del_collective_member', name)
     _true_del_collective_member(name)
+    if cfgstreams:
+        _hasquorum = has_quorum()
+        pushes = eventlet.GreenPool()
+        payload = msgpack.packb({'quorum': _hasquorum}, use_bin_type=False)
+        for _ in pushes.starmap(
+                _push_rpc,
+                [(cfgstreams[s]['stream'], payload) for s in cfgstreams]):
+            pass
 
 def _true_del_collective_member(name, sync=True):
     global cfgleader
@@ -1142,7 +1189,10 @@ class _ExpressionFormat(string.Formatter):
                 # such as 'net.pxe.hwaddr'
                 key = '.' + left.attr + key
                 left = left.value
-            key = left.id + key
+            if isinstance(left, ast.Name):
+                key = left.id + key
+            else:
+                raise ValueError("Invalid AST structure: expected ast.Name at end of attribute chain")
             if (not key.startswith('custom.') and
                         _get_valid_attrname(key) not in allattributes.node):
                 raise ValueError(
@@ -1203,6 +1253,32 @@ class _ExpressionFormat(string.Formatter):
                 return strval[index]
         elif isinstance(node, ast.Constant):
             return node.value
+        elif isinstance(node, ast.Call):
+            key = ''
+            baseval = ''
+            if isinstance(node.func, ast.Attribute):
+                fun_name = node.func.attr
+                baseval = self._handle_ast_node(node.func.value)
+            else:
+                raise ValueError("Invalid function call syntax in expression")
+            if fun_name == 'replace':
+                if len(node.args) != 2:
+                    raise ValueError("Invalid number of arguments to replace")
+                arg1 = self._handle_ast_node(node.args[0])
+                arg2 = self._handle_ast_node(node.args[1])
+                return baseval.replace(arg1, arg2)
+            elif fun_name == 'upper':
+                return baseval.upper()
+            elif fun_name == 'lower':
+                return baseval.lower()
+            elif fun_name == 'block_number':
+                chunk_size = self._handle_ast_node(node.args[0])
+                return (int(baseval) - 1) // chunk_size + 1
+            elif fun_name == 'block_offset':
+                chunk_size = self._handle_ast_node(node.args[0])
+                return (int(baseval) - 1) % chunk_size + 1
+            else:
+                raise ValueError("Unsupported function in expression")
         else:
             raise ValueError("Unrecognized expression syntax")
 
@@ -2206,7 +2282,7 @@ class ConfigManager(object):
                 watcher = self._nodecollwatchers[self.tenant][watcher]
                 watcher(added=(), deleting=nodes, renamed=(), configmanager=self)
         changeset = {}
-        for node in nodes:
+        for node in confluent.util.natural_sort(nodes):
             # set a reserved attribute for the sake of the change notification
             # framework to trigger on
             changeset[node] = {'_nodedeleted': 1}
@@ -2214,6 +2290,29 @@ class ConfigManager(object):
             if node in self._cfgstore['nodes']:
                 self._sync_groups_to_node(node=node, groups=[],
                                           changeset=changeset)
+                nidx = self._cfgstore['nodes'][node].get('id.index', {}).get('value', None)
+                if nidx is not None:
+                    currmaxidx = get_global('max_node_index')
+                    freeindexes = get_global('free_node_indexes')
+                    if not freeindexes:
+                        freeindexes = []
+                    if nidx == currmaxidx - 1:
+                        currmaxidx = currmaxidx - 1
+                        while _is_free_index(freeindexes, currmaxidx - 1):
+                            _remove_free_index(freeindexes, currmaxidx - 1)
+                            currmaxidx = currmaxidx - 1
+                        set_global('max_node_index', currmaxidx)
+                    else:
+                        lastindex = freeindexes[-1] if freeindexes else [-2, -2]
+                        if not isinstance(lastindex, list):
+                            lastindex = [lastindex, lastindex]
+                        if nidx == lastindex[1] + 1:
+                            lastindex[1] = nidx
+                            if freeindexes:
+                                freeindexes[-1] = lastindex
+                        else:
+                            freeindexes.append(nidx)
+                    set_global('free_node_indexes', freeindexes)
                 del self._cfgstore['nodes'][node]
                 _mark_dirtykey('nodes', node, self.tenant)
         self._notif_attribwatchers(changeset)
@@ -2491,12 +2590,29 @@ class ConfigManager(object):
                             attrname, node)
                         raise ValueError(errstr)
                     attribmap[node][attrname] = attrval
-        for node in attribmap:
+        for node in confluent.util.natural_sort(attribmap):
             node = confluent.util.stringify(node)
             exprmgr = None
             if node not in self._cfgstore['nodes']:
                 newnodes.append(node)
-                self._cfgstore['nodes'][node] = {}
+                freeindexes = get_global('free_node_indexes')
+                if not freeindexes:
+                    freeindexes = []
+                if _count_freeindexes(freeindexes) > 128:  # tend to leave freed indexes disused until a lot have accumulated
+                    if isinstance(freeindexes[0], list):
+                        nidx = freeindexes[0][0]
+                        freeindexes[0][0] = nidx + 1
+                        if freeindexes[0][0] == freeindexes[0][1]:
+                            freeindexes[0] = freeindexes[0][0]
+                    else:
+                        nidx = freeindexes.pop(0)
+                    set_global('free_node_indexes', freeindexes)
+                else:
+                    nidx = get_global('max_node_index')
+                    if nidx is None:
+                        nidx = 1
+                    set_global('max_node_index', nidx + 1)
+                self._cfgstore['nodes'][node] = {'id.index': {'value': nidx}}
             cfgobj = self._cfgstore['nodes'][node]
             recalcexpressions = False
             for attrname in attribmap[node]:
@@ -3129,6 +3245,29 @@ def get_globals():
         bkupglobals[globvar] = _cfgstore['globals'][globvar]
     return bkupglobals
 
+def _init_indexes():
+    maxidx = get_global('max_node_index')
+    if maxidx is not None or 'main' not in _cfgstore:
+        return
+    maxidx = 1
+    maincfgstore = _cfgstore['main']
+    nodes_without_index = []
+    for node in confluent.util.natural_sort(maincfgstore.get('nodes', {})):
+        nidx = maincfgstore['nodes'][node].get('id.index', {}).get('value', None)
+        if nidx is not None:
+            if nidx >= maxidx:
+                maxidx = nidx + 1
+        else:
+            nodes_without_index.append(node)
+    for node in nodes_without_index:
+        maincfgstore['nodes'][node]['id.index'] = {'value': maxidx}
+        maxidx += 1
+        _mark_dirtykey('nodes', node, None)
+    set_global('max_node_index', maxidx)
+    set_global('free_node_indexes', [])
+    ConfigManager._bg_sync_to_file()
+
+
 def init(stateless=False):
     global _cfgstore
     global _ready
@@ -3141,6 +3280,7 @@ def init(stateless=False):
         _cfgstore = {}
     members = list(list_collective())
     if len(members) < 2:
+        _init_indexes()
         _ready = True
 
 

confluent_server/confluent/core.py

@@ -76,7 +76,7 @@ import shutil
 
 vinz = None
 pluginmap = {}
-dispatch_plugins = (b'ipmi', u'ipmi', b'redfish', u'redfish', b'tsmsol', u'tsmsol', b'geist', u'geist', b'deltapdu', u'deltapdu', b'eatonpdu', u'eatonpdu', b'affluent', u'affluent', b'cnos', u'cnos', b'enos', u'enos')
+dispatch_plugins = (b'remoteconfig', b'ipmi', u'ipmi', b'redfish', u'redfish', b'tsmsol', u'tsmsol', b'geist', u'geist', b'deltapdu', u'deltapdu', b'eatonpdu', u'eatonpdu', b'affluent', u'affluent', b'cnos', u'cnos', b'enos', u'enos')
 
 PluginCollection = plugin.PluginCollection
 
@@ -300,6 +300,20 @@ def _init_core():
                         'default': 'ipmi',
                     }),
                 },
+                'certificate': {
+                    'sign': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                    'generate_csr': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                    'install': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                },
                 'certificate_authorities': PluginCollection({
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
@@ -462,7 +476,15 @@ def _init_core():
             }),
             'ident_image': PluginRoute({
                 'handler': 'identimage'
-            })
+            }),
+            'remote_config': {
+                'run': PluginRoute({
+                    'handler': 'remoteconfig'
+                }),
+                'active': PluginCollection({
+                    'handler': 'remoteconfig'
+                }),
+            },
         },
         'events': {
             'hardware': {

confluent_server/confluent/discovery/core.py

@@ -98,6 +98,7 @@ import eventlet
 import eventlet.greenpool
 import eventlet.semaphore
 
+
 autosensors = set()
 scanner = None
 
@@ -106,6 +107,11 @@ try:
 except NameError:
     unicode = str
 
+try:
+    import cryptography.x509.verification as verification
+except ImportError:
+    verification = None
+
 class nesteddict(dict):
 
     def __missing__(self, key):
@@ -1059,8 +1065,12 @@ def get_nodename_sysdisco(cfg, handler, info):
         return currnode
     else:
         baynum = info['bay']
-        nl = cfg.filter_node_attributes('enclosure.manager=' + currnode)
-        nl = list(cfg.filter_node_attributes('enclosure.bay={0}'.format(baynum), nl))
+        alphabaynum = '{}{}'.format((int(baynum) + 1) // 2, 'ab'[(int(baynum) - 1) % 2])
+        onl = cfg.filter_node_attributes('enclosure.manager=' + currnode)
+        nl = list(cfg.filter_node_attributes('enclosure.bay={}'.format(baynum), onl))
+        if len(nl) == 1:
+            return nl[0]
+        nl = list(cfg.filter_node_attributes('enclosure.bay={}'.format(alphabaynum), onl))
         if len(nl) == 1:
             return nl[0]
 
@@ -1182,14 +1192,21 @@ def get_nodename_from_enclosures(cfg, info):
         encl = nodes_by_uuid[cuuid]
         bay = info.get('enclosure.bay', None)
         if bay:
-            tnl = cfg.filter_node_attributes('enclosure.manager=' + encl)
+            otnl = cfg.filter_node_attributes('enclosure.manager=' + encl)
             tnl = list(
-                cfg.filter_node_attributes('enclosure.bay={0}'.format(bay),
-                                           tnl))
+                cfg.filter_node_attributes('enclosure.bay={}'.format(bay), otnl))
             if len(tnl) == 1:
                 # This is not a secure assurance, because it's by
                 # uuid instead of a key
                 nodename = tnl[0]
+            else:
+                alphabay = '{}{}'.format((int(bay) + 1) // 2, 'ab'[(int(bay) - 1) % 2])
+                tnl = list(
+                    cfg.filter_node_attributes('enclosure.bay={}'.format(alphabay), otnl))
+                if len(tnl) == 1:
+                    # Fallback alpha-bay mapping resolved to a single node
+                    nodename = tnl[0]
+
     return nodename
 
 
@@ -1215,10 +1232,15 @@ def search_smms_by_cert(currsmm, cert, cfg):
             port = neigh.get('port', None)
             if port is not None:
                 bay = port + 1
-                nl = list(
+                onl = list(
                     cfg.filter_node_attributes('enclosure.manager=' + currsmm))
                 nl = list(
-                    cfg.filter_node_attributes('enclosure.bay={}'.format(bay), nl))
+                    cfg.filter_node_attributes('enclosure.bay={}'.format(bay), onl))
+                if len(nl) == 1:
+                    return currsmm, bay, nl[0]
+                alphabay = '{}{}'.format((bay + 1) // 2, 'ab'[(bay - 1) % 2])
+                nl = list(
+                    cfg.filter_node_attributes('enclosure.bay={}'.format(alphabay), onl))
                 if len(nl) == 1:
                     return currsmm, bay, nl[0]
                 return currsmm, bay, None
@@ -1311,8 +1333,15 @@ def eval_node(cfg, handler, info, nodename, manual=False):
                 return
         # search for nodes fitting our description using filters
         # lead with the most specific to have a small second pass
-        nl = list(cfg.filter_node_attributes(
-            'enclosure.bay={0}'.format(info['enclosure.bay']), nl))
+        baynum = info.get('enclosure.bay', None)
+        if baynum:
+            nnl = list(cfg.filter_node_attributes(
+                'enclosure.bay={}'.format(baynum), nl))
+            if len(nnl) == 0:
+                alphabaynum = '{}{}'.format((int(baynum) + 1) // 2, 'ab'[(int(baynum) - 1) % 2])
+                nnl = list(cfg.filter_node_attributes(
+                    'enclosure.bay={}'.format(alphabaynum), nl))
+            nl = nnl
         if len(nl) != 1:
             info['discofailure'] = 'ambigconfig'
             if len(nl):
@@ -1323,7 +1352,7 @@ def eval_node(cfg, handler, info, nodename, manual=False):
                 errorstr = 'The {0} in enclosure {1} bay {2} does not ' \
                            'seem to be a defined node ({3})'.format(
                                         handler.devname, nodename,
-                                        info['enclosure.bay'],
+                                        baynum,
                                         handler.ipaddr,
                                     )
             if manual:
@@ -1472,7 +1501,7 @@ def discover_node(cfg, handler, info, nodename, manual):
                         break
             log.log({'info': 'Discovered {0} ({1})'.format(nodename,
                                                           handler.devname)})
-            if nodeconfig:
+            if nodeconfig or handler.current_cert_self_signed():
                 bmcaddr = cfg.get_node_attributes(nodename, 'hardwaremanagement.manager')
                 bmcaddr = bmcaddr.get(nodename, {}).get('hardwaremanagement.manager', {}).get('value', '')
                 if not bmcaddr:
@@ -1481,9 +1510,12 @@ def discover_node(cfg, handler, info, nodename, manual):
                     bmcaddr = bmcaddr.split('/', 1)[0]
                     wait_for_connection(bmcaddr)
                     socket.getaddrinfo(bmcaddr, 443)
+            if nodeconfig:
                     subprocess.check_call(['/opt/confluent/bin/nodeconfig', nodename] + nodeconfig)
                     log.log({'info': 'Configured {0} ({1})'.format(nodename,
                                                           handler.devname)})
+            if verification and handler.current_cert_self_signed():
+                handler.autosign_certificate()
 
         info['discostatus'] = 'discovered'
         for i in pending_by_uuid.get(curruuid, []):

confluent_server/confluent/discovery/handlers/generic.py

@@ -17,6 +17,10 @@ import errno
 import eventlet
 import socket
 webclient = eventlet.import_patched('pyghmi.util.webclient')
+try:
+    import cryptography.x509 as x509
+except ImportError:
+    x509 = None
 
 class NodeHandler(object):
     https_supported = True
@@ -59,6 +63,40 @@ class NodeHandler(object):
         # may occur against the target in a short while
         return True
 
+    def current_cert_self_signed(self):
+        if not x509:
+            return
+        if not self._ipaddr:
+            return
+        try:
+            wc = webclient.SecureHTTPConnection(self._ipaddr, verifycallback=self._savecert, port=443)
+            wc.connect()
+            wc.close()
+            if not self._fp:
+                return False
+            # Check if certificate is self-signed by comparing issuer and subject
+            cert = self._fp
+            certobj = x509.load_der_x509_certificate(cert)
+            skid = None
+            akid = None
+            for ext in certobj.extensions:
+                if ext.oid == x509.ExtensionOID.SUBJECT_KEY_IDENTIFIER:
+                    skid = ext.value
+                elif ext.oid == x509.ExtensionOID.AUTHORITY_KEY_IDENTIFIER:
+                    akid = ext.value
+            if akid:
+                if skid.digest == akid.key_identifier:
+                    return True
+            elif certobj.issuer == certobj.subject:
+                return True
+        except Exception:
+            pass
+        return False
+    
+    def autosign_certificate(self):
+        # A no-op by default
+        return
+
     def scan(self):
         # Do completely passive things to enhance data.
         # Probe is permitted to for example attempt a login

confluent_server/confluent/discovery/handlers/redfishbmc.py

@@ -23,6 +23,7 @@ try:
     from urllib import urlencode
 except ImportError:
     from urllib.parse import urlencode
+import eventlet.green.subprocess as subprocess
 
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
@@ -326,6 +327,14 @@ class NodeHandler(generic.NodeHandler):
             raise exc.TargetEndpointUnreachable(
                 'hardwaremanagement.manager must be set to desired address (No IPv6 Link Local detected)')
 
+    def autosign_certificate(self):
+        nodename = self.nodename
+        hwmgt_method = self.configmanager.get_node_attributes(
+            nodename, 'hardwaremanagement.method').get(
+                nodename, {}).get('hardwaremanagement.method', {}).get('value', 'ipmi')
+        if hwmgt_method != 'redfish':
+            return
+        subprocess.check_call(['/opt/confluent/bin/nodecertutil', nodename, 'signbmccert', '--days', '47'])
 
 def remote_nodecfg(nodename, cfm):
     cfg = cfm.get_node_attributes(

confluent_server/confluent/discovery/handlers/xcc.py

@@ -29,6 +29,7 @@ import eventlet.green.socket as socket
 webclient = eventlet.import_patched('pyghmi.util.webclient')
 import struct
 getaddrinfo = eventlet.support.greendns.getaddrinfo
+import eventlet.green.subprocess as subprocess
 
 
 def fixuuid(baduuid):
@@ -490,7 +491,7 @@ class NodeHandler(immhandler.NodeHandler):
                             {'UserName': username}, method='PATCH')
                         if status != 200:
                             rsp = json.loads(rsp)
-                            if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError', 'Base.1.14.GeneralError', 'Base.1.18.GeneralError'):
+                            if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError', 'Base.1.14.GeneralError', 'Base.1.18.GeneralError', 'Base.1.21.GeneralError'):
                                 if tries:
                                     eventlet.sleep(4)
                                 elif tmpaccount:
@@ -522,7 +523,7 @@ class NodeHandler(immhandler.NodeHandler):
             if userent['users_user_name'] == user:
                 curruser = userent
                 break
-        if curruser.get('users_pass_is_sha256', 0):
+        if curruser and curruser.get('users_pass_is_sha256', 0):
             self._wc = None
             wc = self.wc
             nwc = wc.dupe()
@@ -704,6 +705,15 @@ class NodeHandler(immhandler.NodeHandler):
             if em:
                 self.configmanager.set_node_attributes(
                     {em: {'id.uuid': enclosureuuid}})
+    def autosign_certificate(self):
+        nodename = self.nodename
+        hwmgt_method = self.configmanager.get_node_attributes(
+            nodename, 'hardwaremanagement.method').get(
+                nodename, {}).get('hardwaremanagement.method', {}).get('value', 'ipmi')
+        if hwmgt_method != 'redfish':
+            return
+        subprocess.check_call(['/opt/confluent/bin/nodecertutil', nodename, 'signbmccert', '--days', '47'])
+
 
 def remote_nodecfg(nodename, cfm):
     cfg = cfm.get_node_attributes(

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -441,7 +441,7 @@ def _find_service(service, target):
             continue
         else:
             for targurl in peerdata[nid]['urls']:
-                if '/eth' in targurl and targurl.endswith('.xml'):
+                if targurl and targurl.endswith('.xml'):
                     pooltargs.append(('/redfish/v1/', peerdata[nid], 'megarac-bmc'))
         # For now, don't interrogate generic redfish bmcs
         # This is due to a need to deduplicate from some supported SLP
@@ -491,7 +491,7 @@ def check_fish(urldata, port=443, verifycallback=None):
             data['services'] = ['lenovo-xcc'] if 'xcc-variant' not in peerinfo else ['lenovo-xcc' + peerinfo['xcc-variant']]
             return data
         except (IndexError, KeyError):
-            if 'type' in peerinfo and peerinfo['type'].lower() == 'lenovo-smm3':
+            if 'type' in peerinfo and peerinfo['type'].lower() in ('lenovo-smm3', 'smm3'):
                 del peerinfo['xcc-variant']
                 data['uuid'] = peerinfo['enclosure-uuid']
                 data['services'] = ['lenovo-smm3']

confluent_server/confluent/messages.py

@@ -519,6 +519,8 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
         return InputAlertDestination(path, nodes, inputdata, multinode)
     elif len(path) == 3 and path[:3] == ['configuration', 'management_controller', 'certificate_authorities'] and operation not in ('retrieve', 'delete'):
         return InputCertificateAuthority(path, nodes, inputdata)
+    elif len(path) == 4 and path[:4] == ['configuration', 'management_controller', 'certificate', 'sign'] and operation not in ('retrieve', 'delete'):
+        return InputSigningParameters(path, inputdata, nodes, configmanager)
     elif path == ['identify'] and operation != 'retrieve':
         return InputIdentifyMessage(path, nodes, inputdata)
     elif path == ['events', 'hardware', 'decode']:
@@ -578,6 +580,8 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
         return InputLicense(path, nodes, inputdata, configmanager)
     elif path == ['deployment', 'lock'] and inputdata:
         return InputDeploymentLock(path, nodes, inputdata)
+    elif path == ['deployment', 'remote_config', 'run'] and inputdata:
+        return InputRemoteConfig(path, nodes, inputdata)
     elif path == ['deployment', 'ident_image']:
         return InputIdentImage(path, nodes, inputdata)
     elif path == ['console', 'ikvm']:
@@ -721,6 +725,9 @@ class InputConfigChangeSet(InputExpression):
         endattrs = {}
         for attr in attrs:
             origval = attrs[attr]
+            if isinstance(origval, int):
+                endattrs[attr] = origval
+                continue
             if isinstance(origval, bytes) or isinstance(origval, unicode):
                 origval = {'expression': origval}
             if 'expression' not in origval:
@@ -956,6 +963,20 @@ class ConfluentInputMessage(ConfluentMessage):
     def is_valid_key(self, key):
         return key in self.valid_values
 
+class InputSigningParameters(InputConfigChangeSet):
+
+    def get_days(self, node):
+        attribs = self.get_attributes(node)
+        return int(attribs['days'])
+
+    def get_added_names(self, node):
+        attribs = self.get_attributes(node)
+        addnames = []
+        for subj in (attribs.get('added_names') or '').split(','):
+            if subj:
+                addnames.append(subj.strip())
+        return addnames
+
 
 class InputCertificateAuthority(ConfluentInputMessage):
     keyname = 'pem'
@@ -975,6 +996,10 @@ class InputDeploymentLock(ConfluentInputMessage):
     keyname = 'lock'
     valid_values = ['autolock', 'unlocked', 'locked']
 
+class InputRemoteConfig(ConfluentInputMessage):
+    keyname = 'category'
+    valid_values = ['post.d', 'firstboot.d', 'onboot.d']
+
 class DeploymentLock(ConfluentChoiceMessage):
     valid_values = set([
         'autolock',
@@ -1703,6 +1728,7 @@ class Disk(ConfluentMessage):
     state_aliases = {
         'unconfigured bad': 'fault',
         'unconfigured good': 'unconfigured',
+        'unconfiguredgood': 'unconfigured',
         'global hot spare': 'hotspare',
         'dedicated hot spare': 'hotspare',
     }

confluent_server/confluent/networking/lldp.py

@@ -35,6 +35,7 @@ if __name__ == '__main__':
     import confluent.config.configmanager as cfm
 import base64
 import confluent.networking.nxapi as nxapi
+import confluent.networking.srlinux as srlinux
 import confluent.exceptions as exc
 import confluent.log as log
 import confluent.messages as msg
@@ -189,6 +190,10 @@ def detect_backend(switch, verifier):
             apicheck, retcode = wc.grab_json_response_with_status('/api/')
             if retcode == 400 and apicheck.startswith(b'{"imdata":['):
                 _fastbackends[switch] = 'nxapi'
+            else:
+                rsp = wc.grab_json_response_with_status('/jsonrpc', {'dummy': 'data'}, returnheaders=True)
+                if rsp[1] == 401 and rsp[2].get('WWW-Authenticate', '').startswith('Basic realm="SRLinux"'):
+                    _fastbackends[switch] = 'srlinux'
         return _fastbackends.get(switch, None)
 
 def _extract_neighbor_data_https(switch, user, password, cfm, lldpdata):
@@ -203,6 +208,8 @@ def _extract_neighbor_data_https(switch, user, password, cfm, lldpdata):
         return _extract_neighbor_data_affluent(switch, user, password, cfm, lldpdata, wc)
     elif backend == 'nxapi':
         return _extract_neighbor_data_nxapi(switch, user, password, cfm, lldpdata, wc)
+    elif backend == 'srlinux':
+        return _extract_neighbor_data_srlinux(switch, user, password, cfm, lldpdata, wc)
 
 
 
@@ -217,10 +224,28 @@ def _extract_neighbor_data_nxapi(switch, user, password, cfm, lldpdata, wc):
         )
         portdata['peerid'] = peerid
         _extract_extended_desc(portdata, portdata['peerdescription'], True)
+        portdata['switch'] = switch
         _neighbypeerid[peerid] = portdata
         lldpdata[port] = portdata
     _neighdata[switch] = lldpdata
 
+def _extract_neighbor_data_srlinux(switch, user, password, cfm, lldpdata, wc):
+    cli = srlinux.SRLinuxClient(switch, user, password, cfm)
+    lldpinfo = cli.get_lldp()
+    for port in lldpinfo:
+        portdata = lldpinfo[port]
+        peerid = '{0}.{1}'.format(
+            portdata.get('peerchassisid', '').replace(':', '-').replace('/', '-'),
+            portdata.get('peerportid', '').replace(':', '-').replace('/', '-'),
+        )
+        portdata['peerid'] = peerid
+        _extract_extended_desc(portdata, portdata['peerdescription'], True)
+        portdata['switch'] = switch
+        _neighbypeerid[peerid] = portdata
+        lldpdata[port] = portdata
+    _neighdata[switch] = lldpdata
+
+
 def _extract_neighbor_data_affluent(switch, user, password, cfm, lldpdata, wc):
     wc.set_basic_credentials(user, password)
     neighdata = wc.grab_json_response('/affluent/lldp/all')
@@ -255,7 +280,13 @@ def _extract_neighbor_data_b(args):
 
     args are carried as a tuple, because of eventlet convenience
     """
-    switch, password, user, cfm, force = args[:5]
+    # Safely unpack args with defaults to avoid IndexError
+    switch = args[0] if len(args) > 0 else None
+    password = args[1] if len(args) > 1 else None
+    user = args[2] if len(args) > 2 else None
+    cfm = args[3] if len(args) > 3 else None
+    privproto = args[4] if len(args) > 4 else None
+    force = args[5] if len(args) > 5 else False
     vintage = _neighdata.get(switch, {}).get('!!vintage', 0)
     now = util.monotonic_time()
     if vintage > (now - 60) and not force:
@@ -265,7 +296,7 @@ def _extract_neighbor_data_b(args):
         return _extract_neighbor_data_https(switch, user, password, cfm, lldpdata)
     except Exception as e:
         pass
-    conn = snmp.Session(switch, password, user)
+    conn = snmp.Session(switch, password, user, privacy_protocol=privproto)
     sid = None
     for sysid in conn.walk('1.3.6.1.2.1.1.2'):
         sid = str(sysid[1][6:])
@@ -364,8 +395,8 @@ def _extract_neighbor_data(args):
             return _extract_neighbor_data_b(args)
     except Exception as e:
         yieldexc = False
-        if len(args) >= 6:
-            yieldexc = args[5]
+        if len(args) >= 7:
+            yieldexc = args[6]
         if yieldexc:
             return e
         else:

confluent_server/confluent/networking/macmap.py

@@ -55,6 +55,7 @@ import confluent.log as log
 import confluent.messages as msg
 import confluent.noderange as noderange
 import confluent.networking.nxapi as nxapi
+import confluent.networking.srlinux as srlinux
 import confluent.util as util
 from eventlet.greenpool import GreenPool
 import eventlet.green.subprocess as subprocess
@@ -153,7 +154,7 @@ def _nodelookup(switch, ifname):
     return None
 
 def _fast_map_switch(args):
-    switch, password, user, cfgm = args
+    switch, password, user, cfgm = args[:4]
     macdata = None
     kv = util.TLSCertVerifier(cfgm, switch,
                               'pubkeys.tls_hardwaremanager').verify_cert
@@ -162,8 +163,16 @@ def _fast_map_switch(args):
         return _affluent_map_switch(switch, password, user, cfgm, macdata)
     elif backend == 'nxapi':
         return _nxapi_map_switch(switch, password, user, cfgm)
+    elif backend == 'srlinux':
+        return _srlinux_map_switch(switch, password, user, cfgm)
     raise Exception("No fast backend match")
 
+def _srlinux_map_switch(switch, password, user, cfgm):
+    cli = srlinux.SRLinuxClient(switch, user, password, cfgm)
+    mt = cli.get_mac_table()
+    _macsbyswitch[switch] = mt
+    _fast_backend_fixup(mt, switch)
+
 def _nxapi_map_switch(switch, password, user, cfgm):
         cli = nxapi.NxApiClient(switch, user, password, cfgm)
         mt = cli.get_mac_table()
@@ -213,14 +222,14 @@ def _fast_backend_fixup(macs, switch):
                 else:
                     _nodesbymac[mac] = (nodename, nummacs)
 
-def _offload_map_switch(switch, password, user):
+def _offload_map_switch(switch, password, user, privprotocol=None):
     if _offloader is None:
         _start_offloader()
     evtid = random.randint(0, 4294967295)
     while evtid in _offloadevts:
         evtid = random.randint(0, 4294967295)
     _offloadevts[evtid] = eventlet.Event()
-    _offloader.stdin.write(msgpack.packb((evtid, switch, password, user),
+    _offloader.stdin.write(msgpack.packb((evtid, switch, password, user, privprotocol),
                                          use_bin_type=True))
     _offloader.stdin.flush()
     result = _offloadevts[evtid].wait()
@@ -280,12 +289,11 @@ def _map_switch_backend(args):
     #   fallback if ifName is empty
     #
     global _macmap
-    if len(args) == 4:
-        switch, password, user, _ = args  # 4th arg is for affluent only
-        if not user:
-            user = None
-    else:
-        switch, password = args
+    switch = args[0] if len(args) > 0 else None
+    password = args[1] if len(args) > 1 else None
+    user = args[2] if len(args) > 2 else None
+    privprotocol = args[4] if len(args) > 4 else None
+    if not user:  # make '' be treated as None
         user = None
     if switch not in noaffluent:
         try:
@@ -298,7 +306,7 @@ def _map_switch_backend(args):
         except Exception as e:
             pass
     mactobridge, ifnamemap, bridgetoifmap = _offload_map_switch(
-        switch, password, user)
+        switch, password, user, privprotocol)
     maccounts = {}
     bridgetoifvalid = False
     for mac in mactobridge:
@@ -367,9 +375,9 @@ def _map_switch_backend(args):
                 _nodesbymac[mac] = (nodename, maccounts[ifname])
     _macsbyswitch[switch] = newmacs
 
-def _snmp_map_switch_relay(rqid, switch, password, user):
+def _snmp_map_switch_relay(rqid, switch, password, user, privprotocol=None):
     try:
-        res = _snmp_map_switch(switch, password, user)
+        res = _snmp_map_switch(switch, password, user, privprotocol)
         payload = msgpack.packb((rqid,) + res, use_bin_type=True)
         try:
             sys.stdout.buffer.write(payload)
@@ -391,10 +399,10 @@ def _snmp_map_switch_relay(rqid, switch, password, user):
     finally:
         sys.stdout.flush()
 
-def _snmp_map_switch(switch, password, user):
+def _snmp_map_switch(switch, password, user, privprotocol=None):
     haveqbridge = False
     mactobridge = {}
-    conn = snmp.Session(switch, password, user)
+    conn = snmp.Session(switch, password, user, privacy_protocol=privprotocol)
     ifnamemap = get_portnamemap(conn)
     for vb in conn.walk('1.3.6.1.2.1.17.7.1.2.2.1.2'):
         haveqbridge = True

confluent_server/confluent/networking/netutil.py

@@ -21,7 +21,7 @@ import confluent.collective.manager as collective
 def get_switchcreds(configmanager, switches):
     switchcfg = configmanager.get_node_attributes(
         switches, ('secret.hardwaremanagementuser', 'secret.snmpcommunity',
-                   'secret.hardwaremanagementpassword',
+                   'secret.hardwaremanagementpassword', 'snmp.privacyprotocol',
                    'collective.managercandidates'), decrypt=True)
     switchauth = []
     for switch in switches:
@@ -39,6 +39,7 @@ def get_switchcreds(configmanager, switches):
         user = None
         password = switchparms.get(
             'secret.snmpcommunity', {}).get('value', None)
+        privacy_protocol = None
         if not password:
             password = switchparms.get(
                 'secret.hardwaremanagementpassword', {}).get('value',
@@ -47,7 +48,9 @@ def get_switchcreds(configmanager, switches):
                 'secret.hardwaremanagementuser', {}).get('value', None)
             if not user:
                 user = None
-        switchauth.append((switch, password, user, configmanager))
+            privacy_protocol = switchparms.get(
+                'snmp.privacyprotocol', {}).get('value', None)
+        switchauth.append((switch, password, user, configmanager, privacy_protocol))
     return switchauth
 
 

confluent_server/confluent/networking/srlinux.py

@@ -0,0 +1,252 @@
+import confluent.util as util
+import eventlet
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+
+
+
+class SRLinuxClient:
+    def __init__(self, switch, user, password, configmanager):
+        self.cachedurls = {}
+        self.switch = switch
+        if configmanager:
+            cv = util.TLSCertVerifier(
+                configmanager, switch, 'pubkeys.tls_hardwaremanager'
+            ).verify_cert
+        else:
+            cv = lambda x: True
+        self.user = user
+        self.password = password
+        try:
+            self.user = self.user.decode()
+            self.password = self.password.decode()
+        except Exception:
+            pass
+        self.wc = webclient.SecureHTTPConnection(switch, port=443, verifycallback=cv)
+        self.wc.set_basic_credentials(self.user, self.password)
+        self.rpc_id = 1
+        self.login()
+
+    def login(self):
+        # Just a quick query to validate that credentials are correct and device is reachable and TLS works out however it is supposed to
+        self._get_state('/system/information')
+
+        
+        
+
+    def _rpc_call(self, method, params=None):
+        """Make a JSON-RPC call to SR-Linux"""
+        payload = {
+            'jsonrpc': '2.0',
+            'id': self.rpc_id,
+            'method': method,
+        }
+        if params:
+            payload['params'] = params
+        
+        self.rpc_id += 1
+        
+        rsp = self.wc.grab_json_response_with_status('/jsonrpc', payload)
+        if rsp[1] != 200:
+            raise Exception(f"Failed RPC call: {method}, status: {rsp[1]}")
+        
+        result = rsp[0]
+        if 'error' in result:
+            raise Exception(f"RPC error: {result['error']}")
+        
+        return result.get('result', {})
+
+    def _get_state(self, path, datastore='state'):
+        """Get state data from SR-Linux using JSON-RPC get method"""
+        params = {
+            'commands': [
+                {
+                    'path': path,
+                    'datastore': datastore
+                }
+            ]
+        }
+        result = self._rpc_call('get', params)
+        return result
+
+    def get_firmware(self):
+        """Get firmware/software version information"""
+        firmdata = {}
+        result = self._get_state('/system/information')
+        for item in result:
+            if 'version' in item:
+                firmdata['SR-Linux'] = {'version': item['version']}
+            if 'build-date' in item:
+                if 'SR-Linux' in firmdata:
+                        firmdata['SR-Linux']['date'] = item['build-date']
+        return firmdata
+
+    def get_sensors(self):
+        """Get sensor readings from the device"""
+        sensedata = []
+        result = self._get_state('/platform/control/temperature')
+        for item in result:
+            for pcc in item:
+                currreading = {}
+                for reading in item[pcc]:
+                    if reading.get('temperature', {}).get('alarm-status', False):
+                        currreading['health'] = 'critical'
+                    else:
+                        currreading['health'] = 'ok'
+                    states = []
+                    if reading.get('oper-state', 'up',) != 'up':
+                        states = [reading.get('oper-reason', 'unknown')]
+                    currreading['states'] = states
+                    currreading['name'] = 'Slot {} Temperature'.format(reading.get('slot', 'Unknown'))
+                    currreading['value'] = reading.get('temperature', {}).get('instant', 'Unknown')
+                    currreading['units'] = '°C'
+                    sensedata.append(currreading)
+
+        result = self._get_state('/platform/fan-tray')
+        for item in result:
+            for pft in item:
+                currreading = {}
+                for reading in item[pft]:
+                    if reading.get('srl_nokia-platform-healthz:healthz', {}).get('status', 'healthy') != 'healthy':
+                        currreading['health'] = 'critical'
+                    else:
+                        currreading['health'] = 'ok'
+                    states = []
+                    if reading.get('oper-state', 'up',) != 'up':
+                        states = [reading.get('oper-reason', 'unknown')]
+                    currreading['states'] = states
+                    currreading['name'] = 'Fan Tray {}'.format(reading.get('id', 'Unknown'))
+                    currreading['value'] = reading.get('fan', {}).get('speed', 'Unknown')
+                    currreading['units'] = '%'
+                    sensedata.append(currreading)
+
+        result = self._get_state('/platform/power-supply')
+        for item in result:
+            for pps in item:
+                for reading in item[pps]:
+                    currreading = {}
+                    if reading.get('srl_nokia-platform-healthz:healthz', {}).get('status', 'healthy') != 'healthy':
+                        currreading['health'] = 'critical'
+                    else:
+                        currreading['health'] = 'ok'
+                    states = []
+                    if reading.get('oper-state', 'up',) != 'up':
+                        states = [reading.get('oper-reason', 'unknown')]
+                    currreading['states'] = states
+                    currreading['name'] = 'Power Supply {} Health'.format(reading.get('id', 'Unknown'))
+                    sensedata.append(currreading)
+                    tempreading = {'health': 'ok'}
+                    tempreading['name'] = 'Power Supply {} Temperature'.format(reading.get('id', 'Unknown'))
+                    tempreading['value'] = reading.get('temperature', {}).get('instant', 'Unknown')
+                    tempreading['units'] = '°C'
+                    sensedata.append(tempreading)
+                    for powstat in 'current', 'power', 'voltage':
+                        powreading = {'health': 'ok'}
+                        powreading['name'] = 'Power Supply {} {}'.format(reading.get('id', 'Unknown'), powstat.capitalize())
+                        powreading['value'] = reading.get('input', {}).get(powstat, 'Unknown')
+                        if powstat == 'current':
+                            powreading['units'] = 'A'
+                        elif powstat == 'power':
+                            powreading['units'] = 'W'
+                        elif powstat == 'voltage':
+                            powreading['units'] = 'V'
+                        sensedata.append(powreading)
+        return sensedata
+
+
+
+    def get_health(self):
+        healthdata = {'health': 'ok', 'sensors': []}
+        sensors = self.get_sensors()
+        
+        for sensor in sensors:
+            currhealth = sensor.get('health', 'ok')
+            if currhealth != 'ok':
+                healthdata['sensors'].append(sensor)
+                if sensor['health'] == 'critical':
+                    healthdata['health'] = 'critical'
+                elif sensor['health'] == 'warning' and healthdata['health'] != 'critical':
+                    healthdata['health'] = 'warning'
+        
+        return healthdata
+
+    def get_inventory(self):
+        invdata = []        
+        results = self._get_state('/platform/chassis')
+        for result in results:
+            invinfo = {'name': 'System', 'present': True}
+            invinfo['information'] = {'Manufacturer': 'Nokia'}
+            
+            if isinstance(result, dict):
+                for key, value in result.items():
+                    if key == 'serial-number':
+                        invinfo['information']['Serial Number'] = value
+                    elif key == 'part-number':
+                        invinfo['information']['Part Number'] = value
+                    elif key == 'type':
+                        invinfo['information']['Model'] = value
+            
+            if invinfo['information']:
+                invdata.append(invinfo)
+        return invdata
+
+    def get_mac_table(self):
+        macdict = {}
+        response = self._get_state('/network-instance/bridge-table/mac-table/mac')
+        for datum in response:
+            for niname in datum:
+                for nin in datum[niname]:
+                    btable = nin.get('bridge-table', {})
+                    for btab in btable:
+                        macs = btable[btab].get('mac', [])
+                        for macent in macs:
+                            macaddr = macent.get('address', None)
+                            if macaddr:
+                                macport = macent.get('destination', None)
+                                if macport:
+                                    macdict.setdefault(macport, []).append(macaddr)
+        return macdict
+
+    def get_lldp(self):
+        lldpbyport = {}
+        
+        response = self._get_state('/system/lldp/interface')
+        for datum in response:
+            for intfname in datum:
+                lldpallinfo = datum[intfname]
+                for lldpdatum in lldpallinfo:
+                    myportname = lldpdatum.get('name', None)
+                    for neighinfo in lldpdatum.get('neighbor', []):
+                        peerdesc = neighinfo.get('system-description', 'Unknown')
+                        peername = neighinfo.get('system-name', 'Unknown')
+                        peerchassisid = neighinfo.get('chassis-id', 'Unknown')
+                        peerportid = neighinfo.get('port-id', 'Unknown')
+                        lldpinfo = {
+                            'verified': True, # Data provided with authentication over TLS
+                            'peerdescription': peerdesc,
+                            'peername': peername,
+                            'peerchassisid': peerchassisid,
+                            'peerportid': peerportid,
+                            'portid': myportname,
+                            'port': myportname,
+                        }
+                        lldpbyport[myportname] = lldpinfo
+        return lldpbyport
+
+
+if __name__ == '__main__':
+    import sys
+    import os
+    from pprint import pprint
+    myuser = os.environ.get('SWITCHUSER')
+    mypass = os.environ.get('SWITCHPASS')
+    if not myuser or not mypass:
+        print("Set SWITCHUSER and SWITCHPASS environment variables")
+        sys.exit(1)
+    
+    srl = SRLinuxClient(sys.argv[1], myuser, mypass, None)
+    pprint(srl.get_firmware())
+    pprint(srl.get_inventory())
+    pprint(srl.get_sensors())
+    pprint(srl.get_health())
+    pprint(srl.get_lldp())
+    pprint(srl.get_mac_table())

confluent_server/confluent/osimage.py

@@ -1137,6 +1137,8 @@ def generate_stock_profiles(defprofile, distpath, targpath, osname,
         initrds = ['{0}/initramfs/{1}'.format(defprofile, initrd) for initrd in os.listdir('{0}/initramfs'.format(defprofile))]
         if os.path.exists('{0}/initramfs/{1}'.format(defprofile, arch)):
             initrds.extend(['{0}/initramfs/{1}/{2}'.format(defprofile, arch, initrd) for initrd in os.listdir('{0}/initramfs/{1}'.format(defprofile, arch))])
+        elif arch == 'arm64' and os.path.exists('{0}/initramfs/aarch64'.format(defprofile)):
+            initrds.extend(['{0}/initramfs/aarch64/{1}'.format(defprofile, initrd) for initrd in os.listdir('{0}/initramfs/aarch64'.format(defprofile))])
         for fullpath in initrds:
             initrd = os.path.basename(fullpath)
             if os.path.isdir(fullpath):

confluent_server/confluent/plugins/deployment/remoteconfig.py

@@ -0,0 +1,38 @@
+import confluent.selfservice as selfservice
+import confluent.messages as msg
+import confluent.runansible as runansible
+
+_user_initiated_runs = {}
+def update(nodes, element, configmanager, inputdata):
+    if element[-1] != 'run':
+        raise ValueError('Invalid element for remoteconfig plugin')
+    for node in nodes:
+        category = inputdata.inputbynode[node]
+        playlist = selfservice.list_ansible_scripts(configmanager, node, category)
+        if playlist:
+            _user_initiated_runs[node] = True
+            runansible.run_playbooks(playlist, [node])
+            yield msg.CreatedResource(
+                    '/nodes/{0}/deployment/remote_config/active/{0}'.format(node))
+        else:
+            yield msg.ConfluentNodeError('No remote configuration for category "{0}"', node)
+
+def retrieve(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        if element[-1] == 'active':
+            rst = runansible.running_status.get(node, None)
+            if not rst:
+                return
+            yield msg.ChildCollection(node)
+        elif element[-2] == 'active' and element[-1] == node:
+            rst = runansible.running_status.get(node, None)
+            if not rst:
+                return
+            playstatus = rst.dump_dict()
+            if playstatus['complete'] and _user_initiated_runs.get(node, False):
+                del runansible.running_status[node]
+                del _user_initiated_runs[node]
+            yield msg.KeyValueData(playstatus, node)
+
+
+

confluent_server/confluent/plugins/hardwaremanagement/redfish.py

@@ -20,6 +20,7 @@ import confluent.messages as msg
 import confluent.util as util
 import copy
 import errno
+from confluent import certutil
 import eventlet
 import eventlet.event
 import eventlet.green.threading as threading
@@ -37,6 +38,7 @@ ipmicommand = eventlet.import_patched('pyghmi.redfish.command')
 import socket
 import ssl
 import traceback
+import tempfile
 
 if not hasattr(ssl, 'SSLEOFError'):
     ssl.SSLEOFError = None
@@ -184,8 +186,12 @@ class IpmiCommandWrapper(ipmicommand.Command):
             (node,), ('secret.hardwaremanagementuser', 'collective.manager',
                       'secret.hardwaremanagementpassword', 
                       'hardwaremanagement.manager'), self._attribschanged)
+        htn = cfm.get_node_attributes(node, 'hardwaremanagement.manager_tls_name')
+        subject = htn.get(node, {}).get('hardwaremanagement.manager_tls_name', {}).get('value', None)
+        if not subject:
+            subject = kwargs['bmc']
         kv = util.TLSCertVerifier(cfm, node,
-                                  'pubkeys.tls_hardwaremanager').verify_cert
+                                  'pubkeys.tls_hardwaremanager', subject).verify_cert
         kwargs['verifycallback'] = kv
         try:
             super(IpmiCommandWrapper, self).__init__(**kwargs)
@@ -528,6 +534,8 @@ class IpmiHandler(object):
             return self.handle_alerts()
         elif self.element[1:3] == ['management_controller', 'certificate_authorities']:
             return self.handle_cert_authorities()
+        elif self.element[1:3] == ['management_controller', 'certificate']:
+            return self.handle_certificate()
         elif self.element[1:3] == ['management_controller', 'users']:
             return self.handle_users()
         elif self.element[1:3] == ['management_controller', 'net_interfaces']:
@@ -578,6 +586,26 @@ class IpmiHandler(object):
         self.pyghmi_event_to_confluent(event)
         self.output.put(msg.EventCollection((event,), name=self.node))
 
+    def handle_certificate(self):
+        self.element = self.element[3:]
+        if len(self.element) != 1:
+            raise Exception('Not implemented')
+        if self.element[0] == 'sign' and self.op == 'update':
+            csr = self.ipmicmd.get_bmc_csr()
+            subj, san = util.get_bmc_subject_san(self.cfm, self.node, self.inputdata.get_added_names(self.node))
+            with tempfile.NamedTemporaryFile() as tmpfile:
+                tmpfile.write(csr.encode())
+                tmpfile.flush()
+                certfile = tempfile.NamedTemporaryFile(delete=False)
+                certname = certfile.name
+                certfile.close()
+                certutil.create_certificate(None, certname, tmpfile.name, subj, san, backdate=False,
+                                            days=self.inputdata.get_days(self.node))
+            with open(certname, 'rb') as certf:
+                cert = certf.read()
+            os.unlink(certname)
+            self.ipmicmd.install_bmc_certificate(cert)
+
     def handle_cert_authorities(self):
         if len(self.element) == 3:
             if self.op == 'read':

confluent_server/confluent/plugins/hardwaremanagement/srlinux.py

@@ -0,0 +1,99 @@
+import confluent.exceptions as exc
+import confluent.networking.srlinux as srlinux
+import eventlet
+import eventlet.queue as queue
+import confluent.messages as msg
+
+
+def retrieve_node(node, element, user, pwd, configmanager, inputdata, results):
+    try:
+        retrieve_node_backend(node, element, user, pwd, configmanager, inputdata, results)
+    except exc.PubkeyInvalid as e:
+        results.put(msg.ConfluentNodeError(node, 'Mismatch detected between target certificate fingerprint '
+                'and pubkeys.tls_hardwaremanager attribute'))
+    except Exception as e:
+        results.put(e)
+
+
+def simplify_name(name):
+    return name.lower().replace(' ', '_').replace('/', '-').replace(
+        '_-_', '-')
+
+
+def retrieve_node_backend(node, element, user, pwd, configmanager, inputdata, results):
+    cli = srlinux.SRLinuxClient(node, user, pwd, configmanager)
+    if element == ['power', 'state']:  # client initted successfully, must be on
+        results.put(msg.PowerState(node, 'on'))
+    elif element == ['health', 'hardware']:
+        hinfo = cli.get_health()
+        results.put(msg.HealthSummary(hinfo.get('health', 'unknown'), name=node))
+        results.put(msg.SensorReadings(hinfo.get('sensors', []), name=node))
+    elif element[:3] == ['inventory', 'hardware', 'all']:
+        if len(element) == 3:
+            results.put(msg.ChildCollection('all'))
+            return
+        invinfo = cli.get_inventory()
+        if invinfo:
+            results.put(msg.KeyValueData({'inventory': invinfo}, node))
+    elif element[:3] == ['inventory', 'firmware', 'all']:
+        if len(element) == 3:
+            results.put(msg.ChildCollection('all'))
+            return
+        fwinfo = []
+        for fwnam, fwdat in cli.get_firmware().items():
+            fwinfo.append({fwnam: fwdat})
+        if fwinfo:
+            results.put(msg.Firmware(fwinfo, node))
+    elif element == ['sensors', 'hardware', 'all']:
+        sensors = cli.get_sensors()
+        for sensor in sensors:
+            results.put(msg.ChildCollection(simplify_name(sensor['name'])))
+    elif element[:3] == ['sensors', 'hardware', 'all']:
+        sensors = cli.get_sensors()
+        for sensor in sensors:
+            if element[-1] == 'all' or simplify_name(sensor['name']) == element[-1]:
+                results.put(msg.SensorReadings([sensor], node))
+    else:
+        results.put(msg.ConfluentNodeError(node, 'Not supported'))
+
+
+def retrieve(nodes, element, configmanager, inputdata):
+    results = queue.LightQueue()
+    workers = set([])
+    creds = configmanager.get_node_attributes(
+        nodes, ['secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword'], decrypt=True)
+    for node in nodes:
+        cred = creds.get(node, {})
+        user = cred.get('secret.hardwaremanagementuser', {}).get('value')
+        pwd = cred.get('secret.hardwaremanagementpassword', {}).get('value')
+        try:
+            user = user.decode()
+            pwd = pwd.decode()
+        except Exception:
+            pass
+        if not user or not pwd:
+            yield msg.ConfluentTargetInvalidCredentials(node)
+            continue
+        workers.add(eventlet.spawn(retrieve_node, node, element, user, pwd, configmanager, inputdata, results))
+    while workers:
+        try:
+            datum = results.get(block=True, timeout=10)
+            while datum:
+                if isinstance(datum, Exception):
+                    raise datum
+                if datum:
+                    yield datum
+                datum = results.get_nowait()
+        except queue.Empty:
+            pass
+        eventlet.sleep(0.001)
+        for t in list(workers):
+            if t.dead:
+                workers.discard(t)
+    try:
+        while True:
+            datum = results.get_nowait()
+            if datum:
+                yield datum
+    except queue.Empty:
+        pass

confluent_server/confluent/runansible.py

@@ -21,12 +21,16 @@ try:
     import eventlet.green.subprocess as subprocess
 except ImportError:
     pass
+import base64
+import eventlet.green.select as select
 import shutil
 import json
+import socket
 import msgpack
 import os
 import struct
 import sys
+import tempfile
 
 anspypath = None
 running_status = {}
@@ -38,6 +42,7 @@ class PlayRunner(object):
         self.worker = None
         self.results = []
         self.complete = False
+        self.stdout = ''
 
     def _start_playbooks(self):
         self.worker = eventlet.spawn(self._really_run_playbooks)
@@ -49,6 +54,7 @@ class PlayRunner(object):
 
     def dump_text(self):
         stderr = self.stderr
+        stdout = self.stdout
         retinfo = self.dump_dict()
         textout = ''
         for result in retinfo['results']:
@@ -65,6 +71,9 @@ class PlayRunner(object):
                 else:
                     textout += result['state'] + '\n'
             textout += '\n'
+        if stdout:
+            textout += "OUTPUT **********************************\n"
+            textout += stdout
         if stderr:
             textout += "ERRORS **********************************\n"
             textout += stderr
@@ -88,25 +97,58 @@ class PlayRunner(object):
                 if ansloc:
                     with open(ansloc, 'r') as onsop:
                         shebang = onsop.readline()
-                        anspypath = shebang.strip().replace('#!', '')
+                        anspypath = shebang.strip().replace('#!', '').strip()
                         mypath = anspypath
             if not mypath:
                 mypath = sys.executable
-            with open(os.devnull, 'w+') as devnull:
-                targnodes = ','.join(self.nodes)
-                for playfilename in self.playfiles:
-                    worker = subprocess.Popen(
-                        [mypath, __file__, targnodes, playfilename],
-                        stdin=devnull, stdout=subprocess.PIPE,
-                        stderr=subprocess.PIPE)
-                    stdout, stder = worker.communicate()
-                    self.stderr += stder.decode('utf8')
-                    current = memoryview(stdout)
-                    while len(current):
-                        sz = struct.unpack('=q', current[:8])[0]
-                        result = msgpack.unpackb(current[8:8+sz], raw=False)
-                        self.results.append(result)
-                        current = current[8+sz:]
+            with tempfile.TemporaryDirectory() as tmpdir:
+                feedback = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+                suffix = base64.urlsafe_b64encode(os.urandom(6)).decode('ascii')
+                sockpath = os.path.join(tmpdir, 'feedback.sock.' + suffix)
+                localenv = os.environ.copy()
+                localenv['FEEDBACK_SOCK'] = sockpath
+                feedback.bind(sockpath)
+                feedback.listen(1)
+                with feedback:
+                    with open(os.devnull, 'w+') as devnull:
+                        targnodes = ','.join(self.nodes)
+                        for playfilename in self.playfiles:
+                            worker = subprocess.Popen(
+                                [mypath, __file__, targnodes, playfilename],
+                                stdin=devnull, stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE, env=localenv)
+                            rlist, _, _ = select.select([feedback], [], [], 10)
+                            if not rlist:
+                                raise RuntimeError(
+                                    f"Timed out waiting for feedback socket connection for playbook '{playfilename}'"
+                                )
+                            
+                            def _recv_results(sock, timeout=0.1):
+                                while select.select([sock], [], [], timeout)[0]:
+                                    hdr = b''
+                                    while len(hdr) < 8:
+                                        chunk = sock.recv(8 - len(hdr))
+                                        if not chunk:
+                                            if not hdr:
+                                                return
+                                            raise RuntimeError("Socket closed while receiving message header")
+                                        hdr += chunk
+                                    msglen = struct.unpack('=q', hdr)[0]
+                                    msg = b''
+                                    while len(msg) < msglen:
+                                        chunk = sock.recv(msglen - len(msg))
+                                        if not chunk:
+                                            raise RuntimeError("Socket closed while receiving message")
+                                        msg += chunk
+                                    self.results.append(msgpack.unpackb(msg, raw=False))
+                            conn, _ = feedback.accept()
+                            with conn:
+                                while worker.poll() is None:
+                                    _recv_results(conn)
+                                _recv_results(conn, timeout=0)
+                            stdout, stder = worker.communicate()
+                            self.stderr += stder.decode('utf8')
+                            self.stdout += stdout.decode('utf8')
         finally:
             self.complete = True
 
@@ -119,7 +161,7 @@ def run_playbooks(playfiles, nodes):
     runner._start_playbooks()
 
 
-def print_result(result, state, collector=None):
+def print_result(result, state, collector=None, callbacksock=None):
     output = {
         'task_name': result.task_name,
         'changed': result._result.get('changed', ''),
@@ -130,12 +172,12 @@ def print_result(result, state, collector=None):
         del result._result['warnings']
     except KeyError:
         pass
-    if collector:
+    if state != 'ok' and collector and hasattr(collector, '_dump_results'):
         output['errorinfo'] = collector._dump_results(result._result)
     msg = msgpack.packb(output, use_bin_type=True)
     msglen = len(msg)
-    sys.stdout.buffer.write(struct.pack('=q', msglen))
-    sys.stdout.buffer.write(msg)
+    callbacksock.sendall(struct.pack('=q', msglen))
+    callbacksock.sendall(msg)
 
 if __name__ == '__main__':
     from ansible.inventory.manager import InventoryManager
@@ -149,16 +191,25 @@ if __name__ == '__main__':
     import ansible.plugins.loader
     import yaml
 
+    sockpath = os.environ.get('FEEDBACK_SOCK')
+    if not sockpath:
+        sys.stderr.write('No feedback socket specified\n')
+        sys.exit(1)
+
+    callbacksock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    callbacksock.connect(sockpath)
+
+
     class ResultsCollector(CallbackBase):
 
         def v2_runner_on_unreachable(self, result):
-            print_result(result, 'UNREACHABLE', self)
+            print_result(result, 'UNREACHABLE', self, callbacksock)
 
         def v2_runner_on_ok(self, result, *args, **kwargs):
-            print_result(result, 'ok')
+            print_result(result, 'ok', self, callbacksock)
 
         def v2_runner_on_failed(self, result, *args, **kwargs):
-            print_result(result, 'FAILED', self)
+            print_result(result, 'FAILED', self, callbacksock)
 
     context.CLIARGS = ImmutableDict(
         connection='smart', module_path=['/usr/share/ansible'], forks=10,

confluent_server/confluent/selfservice.py

@@ -395,7 +395,7 @@ def handle_request(env, start_response):
                         keymap = ckeymap
             try:
                 tdc = util.run(['timedatectl'])[0].split(b'\n')
-            except subprocess.CalledProcessError:
+            except (subprocess.CalledProcessError, FileNotFoundError):
                 tdc = []
                 currtzvintage = time.time()
                 ncfg['timezone'] = currtz
@@ -466,6 +466,9 @@ def handle_request(env, start_response):
         statusstr = update.get('state', None)
         statusdetail = update.get('state_detail', None)
         didstateupdate = False
+        if statusstr or 'status' in update:
+            cfg.set_node_attributes({nodename: {
+                'deployment.client_ip': {'value': clientip}}})
         if statusstr:
             cfg.set_node_attributes({nodename: {'deployment.state': statusstr}})
             didstateupdate = True
@@ -516,19 +519,7 @@ def handle_request(env, start_response):
         yield ''
     elif env['PATH_INFO'].startswith('/self/remoteconfig/') and 'POST' == operation:
         scriptcat = env['PATH_INFO'].replace('/self/remoteconfig/', '')
-        playlist = []
-        for privacy in ('public', 'private'):
-            slist, profile = get_scriptlist(
-                scriptcat, cfg, nodename,
-                '/var/lib/confluent/{0}/os/{{0}}/ansible/{{1}}'.format(privacy))
-            dirname = '/var/lib/confluent/{2}/os/{0}/ansible/{1}/'.format(
-                profile, scriptcat, privacy)
-            if not os.path.isdir(dirname):
-                dirname = '/var/lib/confluent/{2}/os/{0}/ansible/{1}.d/'.format(
-                    profile, scriptcat, privacy)
-            for filename in slist:
-                if filename.endswith('.yaml') or filename.endswith('.yml'):
-                    playlist.append(os.path.join(dirname, filename))
+        playlist = list_ansible_scripts(cfg, nodename, scriptcat)
         if playlist:
             runansible.run_playbooks(playlist, [nodename])
             start_response('202 Queued', ())
@@ -587,8 +578,13 @@ def handle_request(env, start_response):
             yield 'No profile'
             return
         fname = '/var/lib/confluent/private/os/{}/{}'.format(profile, fname)
+        fullpath = os.path.abspath(fname)
+        if not fullpath.startswith('/var/lib/confluent/private/os/{}/'.format(profile)):
+            start_response('400 Bad Request', ())
+            yield 'Bad Request'
+            return
         try:
-            with open(fname, 'rb') as privdata:
+            with open(fullpath, 'rb') as privdata:
                 start_response('200 OK', ())
                 yield privdata.read()
                 return
@@ -600,6 +596,22 @@ def handle_request(env, start_response):
         start_response('404 Not Found', ())
         yield 'Not found'
 
+def list_ansible_scripts(cfg, nodename, scriptcat):
+    playlist = []
+    for privacy in ('public', 'private'):
+        slist, profile = get_scriptlist(
+                scriptcat, cfg, nodename,
+                '/var/lib/confluent/{0}/os/{{0}}/ansible/{{1}}'.format(privacy))
+        dirname = '/var/lib/confluent/{2}/os/{0}/ansible/{1}/'.format(
+                profile, scriptcat, privacy)
+        if not os.path.isdir(dirname):
+            dirname = '/var/lib/confluent/{2}/os/{0}/ansible/{1}.d/'.format(
+                    profile, scriptcat, privacy)
+        for filename in slist:
+            if filename.endswith('.yaml') or filename.endswith('.yml'):
+                playlist.append(os.path.join(dirname, filename))
+    return playlist
+
 def get_scriptlist(scriptcat, cfg, nodename, pathtemplate):
     if '..' in scriptcat:
         return None, None
@@ -616,6 +628,11 @@ def get_scriptlist(scriptcat, cfg, nodename, pathtemplate):
         'deployment.profile', {}).get('value', '')
     slist = []
     target = pathtemplate.format(profile, scriptcat)
+    target = os.path.abspath(target)
+    allowedbase = os.path.abspath(pathtemplate.format(profile, '').rstrip('/'))
+    allowedbaseprefix = os.path.join(allowedbase, '')
+    if not target.startswith(allowedbaseprefix):
+        return None, None
     if not os.path.isdir(target) and os.path.isdir(target + '.d'):
         target = target + '.d'
     try:

confluent_server/confluent/snmputil.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016 Lenovo
+# Copyright 2016-2025 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,21 +26,59 @@ import eventlet
 from eventlet.support.greendns import getaddrinfo
 import pysnmp.smi.error as snmperr
 import socket
+import asyncio
 snmp = eventlet.import_patched('pysnmp.hlapi')
+asyn = False
+if not hasattr(snmp, 'UsmUserData'):
+    # pysnmp that dropped the sync support
+    import pysnmp.hlapi.v3arch.asyncio as snmp
+    asyn = True
+    import pysnmp.smi.rfc1902 as rfc1902
 
 
+def get_loop():
+    try:
+        return asyncio.get_event_loop()
+    except RuntimeError:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        return loop
+
+def _run_coro(coro):
+    loop = get_loop()
+    fun = asyncio.wait_for(coro, None)
+    if loop.is_running():
+        task = asyncio.ensure_future(fun)
+        return loop.run_until_complete(task)
+    return loop.run_until_complete(fun)
+
+async def _agen_to_list(agen):
+    out = []
+    async for item in agen:
+        out.append(item)
+    return out
+
+def _sync_gen(agen):
+    return _run_coro(_agen_to_list(agen))
+
 def _get_transport(name):
     # Annoyingly, pysnmp does not automatically determine ipv6 v ipv4
     res = getaddrinfo(name, 161, 0, socket.SOCK_DGRAM)
     if res[0][0] == socket.AF_INET6:
-        return snmp.Udp6TransportTarget(res[0][4], 2)
+        if asyn:
+            return _run_coro(snmp.Udp6TransportTarget.create(res[0][4], 2))
+        else:
+            return snmp.Udp6TransportTarget(res[0][4], 2)
     else:
-        return snmp.UdpTransportTarget(res[0][4], 2)
+        if asyn:
+            return _run_coro(snmp.UdpTransportTarget.create(res[0][4], 2))
+        else:
+            return snmp.UdpTransportTarget(res[0][4], 2)
 
 
 class Session(object):
 
-    def __init__(self, server, secret, username=None, context=None):
+    def __init__(self, server, secret, username=None, context=None, privacy_protocol=None):
         """Create a new session to interrogate a switch
 
         If username is not given, it is assumed that
@@ -59,9 +97,17 @@ class Session(object):
             # SNMP v2c
             self.authdata = snmp.CommunityData(secret, mpModel=1)
         else:
+            if privacy_protocol == 'aes':
+                privproto = snmp.usmAesCfb128Protocol
+            elif privacy_protocol in ('des', None):
+                privproto = snmp.usmDESPrivProtocol
+            else:
+                raise exc.ConfluentException('Unsupported SNMPv3 privacy protocol '
+                                             '{0}'.format(privacy_protocol))
             self.authdata = snmp.UsmUserData(
                 username, authKey=secret, privKey=secret,
-                authProtocol=snmp.usmHMACSHAAuthProtocol)
+                authProtocol=snmp.usmHMACSHAAuthProtocol,
+                privProtocol=privproto)
         self.eng = snmp.SnmpEngine()
 
     def walk(self, oid):
@@ -83,12 +129,22 @@ class Session(object):
         if '::' in oid:
             resolvemib = True
             mib, field = oid.split('::')
-            obj = snmp.ObjectType(snmp.ObjectIdentity(mib, field))
+            if asyn:
+                obj = rfc1902.ObjectType(rfc1902.ObjectIdentity(mib, field))
+            else:
+                obj = snmp.ObjectType(snmp.ObjectIdentity(mib, field))
         else:
-            obj = snmp.ObjectType(snmp.ObjectIdentity(oid))
-
-        walking = snmp.bulkCmd(self.eng, self.authdata, tp, ctx, 0, 10, obj,
-                               lexicographicMode=False, lookupMib=resolvemib)
+            if asyn:
+                obj = rfc1902.ObjectType(rfc1902.ObjectIdentity(oid))
+            else:
+                obj = snmp.ObjectType(snmp.ObjectIdentity(oid))
+        if asyn:
+            walking = snmp.bulk_walk_cmd(self.eng, self.authdata, tp, ctx, 0, 10, obj,
+                                   lexicographicMode=False, lookupMib=resolvemib)
+            walking = _sync_gen(walking)
+        else:
+            walking = snmp.bulkCmd(self.eng, self.authdata, tp, ctx, 0, 10, obj,
+                                   lexicographicMode=False, lookupMib=resolvemib)
         try:
             for rsp in walking:
                 errstr, errnum, erridx, answers = rsp

confluent_server/confluent/util.py

@@ -19,7 +19,9 @@
 import base64
 import confluent.exceptions as cexc
 import confluent.log as log
+import glob
 import hashlib
+import ipaddress
 try:
     import psutil
 except ImportError:
@@ -31,6 +33,12 @@ import socket
 import ssl
 import struct
 import eventlet.green.subprocess as subprocess
+import cryptography.x509 as x509
+try:
+    import cryptography.x509.verification as verification
+except ImportError:
+    verification = None
+
 
 
 def mkdirp(path, mode=0o777):
@@ -86,6 +94,53 @@ def list_interface_indexes():
         return
 
 
+def get_bmc_subject_san(configmanager, nodename, addnames=()):
+    bmc_san = []
+    subject = ''
+    ipas = set([])
+    dnsnames = set([])
+    for addname in addnames:
+        try:
+            addr = ipaddress.ip_address(addname)
+            ipas.add(addname)
+        except Exception:
+            dnsnames.add(addname)
+    nodecfg = configmanager.get_node_attributes(nodename,
+                                             ('dns.domain', 'hardwaremanagement.manager', 'hardwaremanagement.manager_tls_name'))
+    bmcaddr = nodecfg.get(nodename, {}).get('hardwaremanagement.manager', {}).get('value', '')
+    domain = nodecfg.get(nodename, {}).get('dns.domain', {}).get('value', '')
+    isipv4 = False
+    if bmcaddr:
+        bmcaddr = bmcaddr.split('/', 1)[0]
+        bmcaddr = bmcaddr.split('%', 1)[0]
+        dnsnames.add(bmcaddr)
+        subject = bmcaddr
+        if ':' in bmcaddr:
+            ipas.add(bmcaddr)
+            dnsnames.add('{0}.ipv6-literal.net'.format(bmcaddr.replace(':', '-')))
+        else:
+            try:
+                socket.inet_aton(bmcaddr)
+                isipv4 = True
+                ipas.add(bmcaddr)
+            except socket.error:
+                pass
+            if not isipv4: # neither ipv6 nor ipv4, should be a name
+                if domain and domain not in bmcaddr:
+                    dnsnames.add('{0}.{1}'.format(bmcaddr, domain))
+    bmcname = nodecfg.get(nodename, {}).get('hardwaremanagement.manager_tls_name', {}).get('value', '')
+    if bmcname:
+        subject = bmcname
+        dnsnames.add(bmcname)
+        if domain and domain not in bmcname:
+            dnsnames.add('{0}.{1}'.format(bmcname, domain))
+    for dns in dnsnames:
+        bmc_san.append('DNS:{0}'.format(dns))
+    for ip in ipas:
+        bmc_san.append('IP:{0}'.format(ip))
+    return subject, ','.join(bmc_san)
+
+
 def list_ips():
     # Used for getting addresses to indicate the multicast address
     # as well as getting all the broadcast addresses
@@ -184,15 +239,51 @@ def cert_matches(fingerprint, certificate):
     return newfp and fingerprint == newfp
 
 
+_polbuilder = None
+
+
 class TLSCertVerifier(object):
-    def __init__(self, configmanager, node, fieldname):
+    def __init__(self, configmanager, node, fieldname, subject=None):
         self.cfm = configmanager
         self.node = node
         self.fieldname = fieldname
+        self.subject = subject
+
+    def verify_by_ca(self, certificate):
+        global _polbuilder
+        _polbuilder = None
+        if not _polbuilder:
+            certs = []
+            for cert in glob.glob('/var/lib/confluent/public/site/tls/*.pem'):
+                with open(cert, 'rb') as certfile:
+                    certs.extend(x509.load_pem_x509_certificates(certfile.read()))
+            if not certs:
+                return False
+            castore = verification.Store(certs)
+            _polbuilder = verification.PolicyBuilder()
+            eep = verification.ExtensionPolicy.permit_all().require_present(
+                x509.SubjectAlternativeName, verification.Criticality.AGNOSTIC, None).may_be_present(
+                x509.KeyUsage, verification.Criticality.AGNOSTIC, None)
+            cap = verification.ExtensionPolicy.webpki_defaults_ca().require_present(
+                x509.BasicConstraints, verification.Criticality.AGNOSTIC, None).may_be_present(
+                x509.KeyUsage, verification.Criticality.AGNOSTIC, None)
+            _polbuilder = _polbuilder.store(castore).extension_policies(
+                ee_policy=eep, ca_policy=cap)
+        try:
+            addr = ipaddress.ip_address(self.subject)
+            subject = x509.IPAddress(addr)
+        except ValueError:
+            subject = x509.DNSName(self.subject)
+        cert = x509.load_der_x509_certificate(certificate)
+        _polbuilder.build_server_verifier(subject).verify(cert, [])
+        return True
+        
+
 
     def verify_cert(self, certificate):
         storedprint = self.cfm.get_node_attributes(self.node, (self.fieldname,)
                                                    )
+        
         if (self.fieldname not in storedprint[self.node] or
                 storedprint[self.node][self.fieldname]['value'] == ''):
             # no stored value, check policy for next action
@@ -220,6 +311,18 @@ class TLSCertVerifier(object):
                           certificate):
             return True
         fingerprint = get_fingerprint(certificate, 'sha256')
+        # Mismatches, but try more traditional validation using the site CAs
+        if self.subject:
+            try:
+                if verification and self.verify_by_ca(certificate):
+                    auditlog = log.Logger('audit')
+                    auditlog.log({'node': self.node, 'event': 'certautoupdate',
+                                  'fingerprint': fingerprint})
+                    self.cfm.set_node_attributes(
+                        {self.node: {self.fieldname: fingerprint}})
+                    return True
+            except Exception:
+                pass
         raise cexc.PubkeyInvalid(
             'Mismatched certificate detected', certificate, fingerprint,
             self.fieldname, 'mismatch')

confluent_server/confluent_server.spec.tmpl

@@ -25,7 +25,7 @@ Requires: python-pyghmi >= 1.5.71, python-eventlet, python-greenlet, python-pycr
 Requires: python3-pyghmi >= 1.5.71, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute
 %else
 %if "%{dist}" == ".el9"
-Requires: python3-pyghmi >= 1.5.71, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-webauthn, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute
+Requires: python3-pyghmi >= 1.5.71, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-webauthn, python3-netifaces, python3-pyasn1 >= 0.2.3, ((python3-pysnmp >= 4.3.4 and python3-pysnmp < 5.0.0) or python3-pysnmp >= 7.1.21), python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute
 %else
 %if "%{dist}" == ".el10"
 Requires: python3-pyghmi >= 1.5.71, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-webauthn, python3-psutil, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute

confluent_vtbufferd/buildrpm

@@ -1,3 +1,4 @@
+cd $(dirname $0)
 VERSION=`git describe|cut -d- -f 1`
 NUMCOMMITS=`git describe|cut -d- -f 2`
 if [ "$NUMCOMMITS" != "$VERSION"  ]; then
@@ -12,6 +13,7 @@ cp ../LICENSE NOTICE *.c *.h Makefile dist/confluent_vtbufferd-$VERSION
 cd dist
 tar czf confluent_vtbufferd-$VERSION.tar.gz confluent_vtbufferd-$VERSION
 cd -
+mkdir -p ~/rpmbuild/SOURCES ~/rpmbuild/SPECS
 cp dist/confluent_vtbufferd-$VERSION.tar.gz ~/rpmbuild/SOURCES
 sed -e 's/#VERSION#/'$VERSION/  confluent_vtbufferd.spec.tmpl > ~/rpmbuild/SPECS/confluent_vtbufferd.spec
 rpmbuild -ba ~/rpmbuild/SPECS/confluent_vtbufferd.spec  2> /dev/null |grep ^Wrote:
@@ -22,5 +24,10 @@ else
     # Clean up the generated files in this directory
     rm -rf dist 
 fi
+ARCH=$(uname -m)
+if [ ! -z "$1" ]; then
+    cp /root/rpmbuild/RPMS/$ARCH/confluent_vtbufferd-*.$ARCH.rpm $1
+fi
+
 
 

container/Dockerfile

@@ -1,7 +1,14 @@
-FROM almalinux:8
+FROM almalinux:10
+VOLUME /var/lib/confluent
+VOLUME /var/run/confluent
+VOLUME /etc/confluent
+VOLUME /var/lib/tftpboot
+ENV EMBEDHTTP=1
+ENV EMBEDTFTP=0
 RUN ["yum", "-y", "update"]
-RUN ["rpm", "-ivh", "https://hpc.lenovo.com/yum/latest/el8/x86_64/lenovo-hpc-yum-1-1.x86_64.rpm"]
-RUN ["yum", "-y", "install", "lenovo-confluent", "tftp-server", "openssh-clients", "openssl", "vim-enhanced", "iproute"]
+RUN ["rpm", "-ivh", "https://hpc.lenovo.com/yum/latest/el10/x86_64/lenovo-hpc-yum-1-1.x86_64.rpm"]
+RUN ["yum", "-y", "install", "lenovo-confluent", "tftp-server", "openssh-clients", "openssl", "vim-enhanced", "iproute", "policycoreutils", "selinux-policy-targeted", "device-mapper"]
+RUN ["chown", "-r", "root", "/etc/confluent"]
 ADD runconfluent.sh /bin/
 CMD ["/bin/bash", "/bin/runconfluent.sh"]
 

genesis/confluent-genesis.spec

@@ -1,5 +1,5 @@
 %define arch x86_64
-Version: 3.14.1
+Version: 3.14.2
 Release:  1
 Name: confluent-genesis-%{arch}
 BuildArch: noarch

imgutil/el10/pkglist

@@ -19,4 +19,8 @@ fuse-libs
 libnl3
 dhcpcd
 openssh-keysign
-chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr shim-x64.x86_64 grub2-efi-x64 attr
+chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr attr
+%onlyarch x86_64
+shim-x64.x86_64 grub2-efi-x64
+%onlyarch aarch64
+shim-aa64.aarch64 grub2-efi-aa64

imgutil/el9/pkglist

@@ -1,3 +1,4 @@
+system-release
 dnf
 hostname
 irqbalance
@@ -17,4 +18,8 @@ xfsprogs
 e2fsprogs
 fuse-libs
 libnl3
-chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr shim-x64.x86_64 grub2-efi-x64 attr
+chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr attr
+%onlyarch x86_64
+shim-x64.x86_64 grub2-efi-x64
+%onlyarch aarch64
+shim-aa64.aarch64 grub2-efi-aa64

imgutil/imgutil

@@ -214,7 +214,7 @@ def capture_remote(args):
     subprocess.check_call(['rsync', __file__, '{0}:/run/imgutil/capenv/'.format(targ)])
     finfo = subprocess.check_output(['ssh', targ, 'python3', '/run/imgutil/capenv/imgutil', 'getfingerprint']).decode('utf8')
     finfo = json.loads(finfo)
-    if finfo['oscategory'] not in ('el8', 'el9', 'ubuntu20.04', 'ubuntu22.04'):
+    if finfo['oscategory'] not in ('el8', 'el9', 'ubuntu20.04', 'ubuntu22.04', 'ubuntu24.04', 'ubuntu26.04'):
         sys.stderr.write('Not yet supported for capture: ' + repr(finfo) + '\n')
         sys.exit(1)
     unmet = finfo.get('unmetprereqs', [])
@@ -386,6 +386,7 @@ def encrypt_image(plainfile, cryptfile, keyfile):
     loopdev = subprocess.check_output(['losetup', '-f']).decode('utf8').strip()
     subprocess.check_call(['losetup', loopdev, cryptfile])
     subprocess.check_call(['dmsetup', 'create', dmname, '--table', '0 {} crypt aes-xts-plain64 {} 0 {} 8'.format(neededblocks, key, loopdev)])
+    subprocess.check_call(['dmsetup', 'mknodes', dmname])
     with open('/dev/mapper/{}'.format(dmname), 'wb') as cryptout:
         with open(plainfile, 'rb+') as plainin:
             lastoffset = 0
@@ -519,9 +520,18 @@ class OsHandler(object):
             pkglistfile = pkglistfile[:-1]
         with open(pkglistfile, 'r') as pkglist:
             pkgs = ''
+            ignoremode = False
             for line in pkglist.readlines():
                 line = line.split('#', 1)[0].strip()
-                pkgs += line + ' '
+                if line.startswith(r'%onlyarch '):
+                    archs = line[len(r'%onlyarch '):].split()
+                    if self.arch not in archs:
+                        ignoremode = True
+                    else:
+                        ignoremode = False
+                    continue
+                if not ignoremode:
+                    pkgs += line + ' '
             pkgs = pkgs.split()
         retpkgs = []
         for pkg in pkgs:
@@ -764,7 +774,12 @@ class ElHandler(OsHandler):
                 if line.startswith('selinuxfs '):
                     break
             else:
-                self.relabel_targdir()
+                try:
+                    self.relabel_targdir()
+                except subprocess.CalledProcessError:
+                    # Some filesystem contexts can not accommodate the selinux labels, warn that
+                    # this failed, but allow it to proceed in case it can boot anyway
+                    sys.stderr.write('Warning: could not relabel target filesystem for SELinux\n')
 
     def relabel_targdir(self):
         subprocess.check_call(
@@ -810,11 +825,14 @@ def version_sort(iterable):
 
 def get_kern_version(filename):
     with open(filename, 'rb') as kernfile:
-        checkgzip = kernfile.read(2)
-        if checkgzip == b'\x1f\x8b':
-            # gzipped... this would probably be aarch64
+        header = kernfile.read(2)
+        if header == b'\x1f\x8b':
+            # gzipped... we can't process this right now,
             # assume the filename has the version embedded
             return os.path.basename(filename).replace('vmlinuz-', '')
+        if header == b'MZ':
+            # PE format, no easy way to get version, assume filename has it
+            return os.path.basename(filename).replace('vmlinuz-', '')                
         kernfile.seek(0x20e)
         offset = struct.unpack('<H', kernfile.read(2))[0] + 0x200
         kernfile.seek(offset)
@@ -894,7 +912,7 @@ def main():
     buildp.add_argument('-p', '--packagelist', help='Filename of package list to replace default pkglist', default='')
     buildp.add_argument('-a', '--addpackagelist', action='append', default=[],
                         help='A list of additional packages to include, may be specified multiple times')
-    buildp.add_argument('-s', '--source', help='Directory to pull installation from, typically a subdirectory of /var/lib/confluent/distributions. By default, the repositories for the build system are used.')
+    buildp.add_argument('-s', '--source', help='Directory to pull installation from, typically a subdirectory of /var/lib/confluent/distributions. By default, the repositories for the build system are used.  For Ubuntu, this is not supported, the build system repositories are always used.')
     buildp.add_argument('-y', '--non-interactive', help='Avoid prompting for confirmation', action='store_true')
     buildp.add_argument('-v', '--volume',
         help='Directory to make available in the build environment.  -v / will '
@@ -1278,6 +1296,7 @@ def prep_decrypt(indir):
     tempfile.mktemp()
     subprocess.check_call(['dmsetup', 'create', dmname, '--table', '0 {0} crypt {1} {2} 0 {3} 8'.format(
         imglen, cipher, key, loopdev)])
+    subprocess.check_call(['dmsetup', 'mknodes', dmname])
     return '/dev/mapper/{0}'.format(dmname), loopdev
 
 

misc/adoptnode.sh

@@ -9,7 +9,13 @@ if [ -z "$TARGPROF" ]; then
 	echo "Target profile must be specified"
 	exit 1
 fi
-nodedefine $TARGNODE deployment.apiarmed=once deployment.profile=$TARGPROF deployment.useinsecureprotocols= deployment.pendingprofile=$TARGPROF
+OLDINSECURE=$(nodeattrib $TARGNODE deployment.useinsecureprotocols -b 2> /dev/null |grep -v inherited|awk '{print $3}')
+nodedefine $TARGNODE deployment.profile=$TARGPROF deployment.useinsecureprotocols= deployment.pendingprofile=$TARGPROF
+confetty set /nodes/$TARGNODE/deployment/ident_image=create
+REMTMP=$(ssh $TARGNODE mktemp -d)
+scp /var/lib/confluent/private/identity_files/$TARGNODE.json $TARGNODE:$REMTMP
+rm /var/lib/confluent/private/identity_files/$TARGNODE.*
+rm /var/lib/confluent/private/identity_images/$TARGNODE.*
 cat /var/lib/confluent/public/site/ssh/*pubkey | ssh $TARGNODE "mkdir -p /root/.ssh/; cat - >> /root/.ssh/authorized_keys"
 ssh $TARGNODE mkdir -p /etc/confluent /opt/confluent/bin
 cat /var/lib/confluent/public/site/tls/*.pem | ssh $TARGNODE "cat - >> /etc/confluent/ca.pem"
@@ -17,8 +23,11 @@ cat /var/lib/confluent/public/site/tls/*.pem | ssh $TARGNODE "cat - >> /etc/pki/
 nodeattrib $TARGNODE id.uuid=$(ssh $TARGNODE cat /sys/devices/virtual/dmi/id/product_uuid)
 scp prepadopt.sh $TARGNODE:/tmp/
 scp finalizeadopt.sh $TARGNODE:/tmp/
-ssh $TARGNODE bash /tmp/prepadopt.sh $TARGNODE $TARGPROF
+ssh $TARGNODE bash /tmp/prepadopt.sh $TARGNODE $TARGPROF $REMTMP/$TARGNODE.json
 nodeattrib $TARGNODE deployment.pendingprofile=
 nodeattrib $TARGNODE -c deployment.useinsecureprotocols
+if [ ! -z "$OLDINSECURE" ]; then
+	nodeattrib $TARGNODE $OLDINSECURE
+fi
 nodeapply $TARGNODE -k
 ssh $TARGNODE sh /tmp/finalizeadopt.sh

misc/prepadopt.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 TARGNODE=$1
 TARGPROF=$2
+TARGIDENT=$3
 TMPDIR=$(mktemp -d)
 cd $TMPDIR
 DEPLOYSRV=$(echo $SSH_CLIENT|awk '{print $1}')
@@ -16,10 +17,12 @@ cpio -dumi < addons.cpio
 systemctl status firewalld >& /dev/null && FWACTIVE=1
 if [ "$FWACTIVE" == 1 ]; then systemctl stop firewalld; fi
 opt/confluent/bin/copernicus  > /etc/confluent/confluent.info
-opt/confluent/bin/clortho $TARGNODE $DEPLOYSRV > /etc/confluent/confluent.apikey
+#opt/confluent/bin/clortho $TARGNODE $DEPLOYSRV > /etc/confluent/confluent.apikey
+. /etc/confluent/functions
+confluentpython opt/confluent/bin/apiclient -i $TARGIDENT /confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
 if [ "$FWACTIVE" == 1 ]; then systemctl start firewalld; fi
 cp opt/confluent/bin/apiclient /opt/confluent/bin
-curl -sg -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -H "CONFLUENT_NODENAME: $TARGNODE" https://$UDEPLOYSRV/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
+#curl -sg -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -H "CONFLUENT_NODENAME: $TARGNODE" https://$UDEPLOYSRV/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
 # python3 /opt/confluent/bin/apiclient /confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
 cd -
 echo rm -rf $TMPDIR

misc/setupnokia.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+if [[ -z "$SWITCHUSER" || -z "$SWITCHPASS" ]]; then
+    echo "Error: SWITCHUSER and SWITCHPASS environment variables must be set"
+    exit 1
+fi
+
+if [[ $# -eq 0 ]]; then
+    echo "Error: At least one argument is required"
+    exit 1
+fi
+
+SWITCHNAME="$1"
+
+# Create temporary SSH ASKPASS script
+ASKPASS_SCRIPT=$(mktemp)
+trap "rm -f $ASKPASS_SCRIPT" EXIT
+
+cat > "$ASKPASS_SCRIPT" << 'EOF'
+#!/bin/bash
+echo "$SWITCHPASS"
+EOF
+
+chmod 700 "$ASKPASS_SCRIPT"
+
+# Set SSH_ASKPASS environment variable
+export SSH_ASKPASS="$ASKPASS_SCRIPT"
+export SSH_ASKPASS_REQUIRE=force
+export DISPLAY=none
+
+CERTDIR=$(mktemp -d)
+trap "rm -rf $CERTDIR" EXIT
+cd "$CERTDIR"
+python3 /opt/confluent/lib/python/confluent/certutil.py -s "$SWITCHNAME"
+setsid ssh $SWITCHUSER@"$SWITCHNAME" <<EOC
+enter candidate exclusive
+/ system json-rpc-server admin-state enable
+/ system tls server-profile fullcert key "$(<$CERTDIR/key.pem)"
+/ system tls server-profile fullcert certificate "$(<$CERTDIR/cert.pem)"
+/ system json-rpc-server network-instance mgmt https admin-state enable
+/ system json-rpc-server network-instance mgmt https tls-profile fullcert
+commit save
+EOC