Handle broader memory information being returned from confluent

It fouls run of kdump building the kdump image.

confluent_client/bin/nodeinventory

@@ -60,16 +60,21 @@ def print_mem_info(node, prefix, meminfo):
     elif 'DCPMM' in meminfo['memory_type']:
         memdescfmt = '{0}GB {1} '
         meminfo['module_type'] = 'DCPMM'
+    elif meminfo['memory_type'] == 'HBM':
+        memdescfmt = '{0}GB HBM '
     else:
         print('{0}: {1}: Unrecognized Memory'.format(node, prefix))
         return
     if meminfo.get('ecc', False):
         memdescfmt += 'ECC '
-    capacity = meminfo['capacity_mb'] / 1024
     modtype = meminfo.get('module_type', None)
     if modtype:
         memdescfmt += modtype
-    memdesc = memdescfmt.format(capacity, meminfo['speed'])
+    if meminfo.get('capacity_mb', None):
+        capacity = meminfo['capacity_mb'] // 1024
+        memdesc = memdescfmt.format(capacity, meminfo['speed'])
+    else:
+        memdesc = 'Unspecified Module'
     print('{0}: {1} description: {2}'.format(node, prefix, memdesc))
     print('{0}: {1} manufacturer: {2}'.format(
         node, prefix, meminfo['manufacturer']))

confluent_client/doc/man/imgutil.ronn

@@ -0,0 +1,139 @@
+# imgutil(1) -- Work with confluent OS cloning and diskless images
+
+## SYNOPSIS
+
+`imgutil` `build` [<options>] <scratchdir>  
+`imgutil` `exec` [<options>] <scratchdir> [<cmd>...]  
+`imgutil` `unpack` <profilename> <scratchdir>  
+`imgutil` `pack` [<options>] <scratchdir> <profilename>  
+`imgutil` `capture` <node> <profilename>
+
+## DESCRIPTION
+
+**imgutil** is a utility for creating, managing, and deploying OS images for diskless boot and system cloning in a Confluent environment. It supports building images from scratch, capturing images from running systems, and packing/unpacking diskless profiles.
+
+## COMMANDS
+
+* `build`:
+  Build a new diskless image from scratch in the specified scratch directory.
+
+* `exec`:
+  Start the specified scratch directory as a container and optionally run a command inside it.
+
+* `unpack`:
+  Unpack a diskless image profile to a scratch directory for modification.
+
+* `pack`:
+  Pack a scratch directory into a diskless profile that can be deployed.
+
+* `capture`:
+  Capture an image for cloning from a running system.
+
+## BUILD OPTIONS
+
+* `-r`, `--addrepos` <repository>:
+  Repositories to add in addition to the main source. May be specified multiple times.
+
+* `-p`, `--packagelist` <file>:
+  Filename of package list to replace default pkglist.
+
+* `-a`, `--addpackagelist` <file>:
+  A list of additional packages to include. May be specified multiple times.
+
+* `-s`, `--source` <directory>:
+  Directory to pull installation from, typically a subdirectory of `/var/lib/confluent/distributions`. By default, the repositories for the build system are used. For Ubuntu, this is not supported; the build system repositories are always used.
+
+* `-y`, `--non-interactive`:
+  Avoid prompting for confirmation.
+
+* `-v`, `--volume` <mount>:
+  Directory to make available in the build environment. `-v /` will cause it to be mounted in image as `/run/external/`. `-v /:/run/root` will override the target to be `/run/root`. Something like `/var/lib/repository:-` will cause it to mount to the identical path inside the image. May be specified multiple times.
+
+* <scratchdir>:
+  Directory to build new diskless root in.
+
+## EXEC OPTIONS
+
+* `-v`, `--volume` <mount>:
+  Directory to make available in the build environment. `-v /` will cause it to be mounted in image as `/run/external/`. `-v /:/run/root` will override the target to be `/run/root`. May be specified multiple times.
+
+* <scratchdir>:
+  Directory of an unpacked diskless root.
+
+* <cmd>:
+  Optional command to run (defaults to a shell).
+
+## UNPACK OPTIONS
+
+* <profilename>:
+  The diskless OS profile to unpack.
+
+* <scratchdir>:
+  Directory to extract diskless root to.
+
+## PACK OPTIONS
+
+* `-b`, `--baseprofile` <profile>:
+  Profile to copy extra info from. For example, to make a new version of an existing profile, reference the previous one as baseprofile.
+
+* `-u`, `--unencrypted`:
+  Pack an unencrypted image rather than encrypting.
+
+* <scratchdir>:
+  Directory containing diskless root.
+
+* <profilename>:
+  The desired diskless OS profile name to pack the root into.
+
+## CAPTURE OPTIONS
+
+* <node>:
+  Node to capture image from.
+
+* <profilename>:
+  Profile name for captured image.
+
+## EXAMPLES
+
+Build a diskless image from a distribution:
+
+    imgutil build -s  alma-9.6-x86_64 /tmp/myimage
+
+Execute a shell in an unpacked image:
+
+    imgutil exec /tmp/myimage
+
+Execute a specific command in an image:
+
+    imgutil exec /tmp/myimage /bin/rpm -qa
+
+Unpack an existing profile for modification:
+
+    imgutil unpack myprofile /tmp/myimage
+
+Pack a modified image into a new profile:
+
+    imgutil pack /tmp/myimage myprofile-v2
+
+Capture an image from a running node:
+
+    imgutil capture node01 production-image
+
+## FILES
+
+* `/var/lib/confluent/public/os/`:
+  Default location for OS profiles.
+
+* `/var/lib/confluent/private/os/`:
+  Location for encrypted image keys and private data.
+
+* `/var/lib/confluent/distributions/`:
+  Default location for installation sources.
+
+## SEE ALSO
+
+osdeploy(8)
+
+## AUTHOR
+
+Written for the Confluent project.

confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh

@@ -129,6 +129,7 @@ mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
 mv /lib/firmware /lib/firmware-ramfs
 ln -s /sysroot/lib/firmware /lib/firmware
+rm -f /sysroot/etc/dracut.conf.d/diskless.conf # remove diskless dracut from runtime, to make kdump happier
 kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
 if grep debugssh /proc/cmdline >& /dev/null; then
     exec /opt/confluent/bin/start_root

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/getinstalldisk

@@ -96,22 +96,36 @@ def list_disks():
                 adp = ' '.join(line.split()[1:])
                 disks[dev]['adapter'] = adp
                 devbyadp.setdefault(adp, []).append(dev)
+    adapterlist = subprocess.check_output(['localcli', 'storage', 'core', 'adapter', 'list'])
+    if not isinstance(adapterlist, str):
+        adapterlist = adapterlist.decode('utf8')
+    driverbyadp = {}
+    linenum = 0
+    for line in adapterlist.split('\n'):
+        linenum += 1
+        if not line.strip():
+            continue
+        if linenum < 3:
+            continue
+        parts = line.split()
+        if len(parts) < 2:
+            continue
+        adp = parts[0]
+        driver = parts[1]
+        driverbyadp[adp] = driver
     for adp in devbyadp:
-        adaplist = subprocess.check_output(['localcli', 'storage', 'core', 'adapter', 'listdetailed', '--adapter', adp])
-        if not isinstance(adaplist, str):
-            adaplist = adaplist.decode('utf8')
-        for line in adaplist.split('\n'):
-            if not line.strip():
-                continue
-            if 'Driver Name:' in line:
-                driver = ' '.join(line.split()[2:])
-                for dev in devbyadp[adp]:
-                    disks[dev]['adapter_driver'] = driver
+        driver = driverbyadp.get(adp, 'Unknown')
+        for dev in devbyadp[adp]:
+            disks[dev]['adapter_driver'] = driver
     return disks
         
 def main():
     disks = []
-    alldisks = list_disks()
+    try:
+        alldisks = list_disks()
+    except Exception as e:
+        print("Error listing disks: {0}".format(str(e)))
+        alldisks = {}
     for disk in alldisks:
         try:
             disks.append(DiskInfo(disk, alldisks[disk]))
@@ -124,6 +138,11 @@ def main():
         with open('/tmp/storagecfg', 'w') as sc:
             sc.write(f'clearpart --all --drives={nd[0]} --overwritevmfs\n')
             sc.write(f'install --drive={nd[0]} --overwritevmfs\n')
+    else:
+        with open('/tmp/storagecfg', 'w') as sc:
+            sc.write(f'clearpart --firstdisk --overwritevmfs\n')
+            sc.write(f'install --firstdisk --overwritevmfs\n')
+
 
 if __name__ == '__main__':
     main()

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/makeksnet

@@ -45,7 +45,10 @@ try:
         cfg['ipv4_gateway'] = ncfg['ipv4_gateway']
 except Exception:
     pass
-netline = 'network --hostname={0} --bootproto={1}'.format(nodename, cfg['ipv4_method'])
+if cfg['ipv4_method'] == 'static':
+    netline = 'network --hostname={0} --bootproto={1}'.format(nodename, cfg['ipv4_method'])
+else:
+    netline = 'network --bootproto=dhcp'
 if vmnic:
     netline += ' --device={0}'.format(vmnic)
 if cfg['ipv4_method'] == 'static':

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/modinstall

@@ -5,6 +5,7 @@ mv /etc/confluent/confluent.deploycfg /etc/confluent/confluent.newdeploycfg
 mv /etc/confluent/confluent.newdeploycfg /etc/confluent/confluent.deploycfg
 chmod +x /tmp/makeksnet
 /tmp/makeksnet > /tmp/ksnet
+localcli system hostname set --host $node
 python3 /tmp/getinstalldisk
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg|sed -e 's/^rootpassword: //')
 echo rootpw --iscrypted $rootpw > /tmp/rootpw

confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh

@@ -1,16 +1,16 @@
 #!/bin/bash
 set -e
 sed -i 's/label: ubuntu/label: Ubuntu/' $2/profile.yaml
-if [ -e $1/casper/hwe-vmlinuz ]; then
-    ln -s $1/casper/hwe-vmlinuz $2/boot/kernel
-else
+#if [ -e $1/casper/hwe-vmlinuz ]; then
+#    ln -s $1/casper/hwe-vmlinuz $2/boot/kernel
+#else
     ln -s $1/casper/vmlinuz $2/boot/kernel
-fi
-if [ -e $1/casper/hwe-initrd ]; then
-    ln -s $1/casper/hwe-initrd $2/boot/initramfs/distribution
-else
+#fi
+#if [ -e $1/casper/hwe-initrd ]; then
+#    ln -s $1/casper/hwe-initrd $2/boot/initramfs/distribution
+#else
     ln -s $1/casper/initrd $2/boot/initramfs/distribution
-fi
+#fi
 mkdir -p $2/boot/efi/boot
 if [ -d $1/EFI/boot/ ]; then
     ln -s $1/EFI/boot/* $2/boot/efi/boot

confluent_server/confluent/config/attributes.py

@@ -215,6 +215,9 @@ node = {
                         'Using this requires that collective members be '
                         'defined as nodes for noderange expansion')
     },
+    'deployment.client_ip': {
+        'description': ('Client IP used when most recently reporting state.')
+    },
     'deployment.lock': {
         'description': ('Indicates whether deployment actions should be impeded. '
                         'If locked, it indicates that a pending profile should not be applied. '

confluent_server/confluent/discovery/handlers/xcc.py

@@ -490,7 +490,7 @@ class NodeHandler(immhandler.NodeHandler):
                             {'UserName': username}, method='PATCH')
                         if status != 200:
                             rsp = json.loads(rsp)
-                            if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError', 'Base.1.14.GeneralError', 'Base.1.18.GeneralError'):
+                            if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError', 'Base.1.14.GeneralError', 'Base.1.18.GeneralError', 'Base.1.21.GeneralError'):
                                 if tries:
                                     eventlet.sleep(4)
                                 elif tmpaccount:
@@ -522,7 +522,7 @@ class NodeHandler(immhandler.NodeHandler):
             if userent['users_user_name'] == user:
                 curruser = userent
                 break
-        if curruser.get('users_pass_is_sha256', 0):
+        if curruser and curruser.get('users_pass_is_sha256', 0):
             self._wc = None
             wc = self.wc
             nwc = wc.dupe()

confluent_server/confluent/selfservice.py

@@ -466,6 +466,9 @@ def handle_request(env, start_response):
         statusstr = update.get('state', None)
         statusdetail = update.get('state_detail', None)
         didstateupdate = False
+        if statusstr or 'status' in update:
+            cfg.set_node_attributes({nodename: {
+                'deployment.client_ip': {'value': clientip}}})
         if statusstr:
             cfg.set_node_attributes({nodename: {'deployment.state': statusstr}})
             didstateupdate = True

genesis/confluent-genesis.spec

@@ -1,5 +1,5 @@
 %define arch x86_64
-Version: 3.14.1
+Version: 3.14.2
 Release:  1
 Name: confluent-genesis-%{arch}
 BuildArch: noarch

imgutil/imgutil

@@ -894,7 +894,7 @@ def main():
     buildp.add_argument('-p', '--packagelist', help='Filename of package list to replace default pkglist', default='')
     buildp.add_argument('-a', '--addpackagelist', action='append', default=[],
                         help='A list of additional packages to include, may be specified multiple times')
-    buildp.add_argument('-s', '--source', help='Directory to pull installation from, typically a subdirectory of /var/lib/confluent/distributions. By default, the repositories for the build system are used.')
+    buildp.add_argument('-s', '--source', help='Directory to pull installation from, typically a subdirectory of /var/lib/confluent/distributions. By default, the repositories for the build system are used.  For Ubuntu, this is not supported, the build system repositories are always used.')
     buildp.add_argument('-y', '--non-interactive', help='Avoid prompting for confirmation', action='store_true')
     buildp.add_argument('-v', '--volume',
         help='Directory to make available in the build environment.  -v / will '