From 94d396a8cbd20c304a5b1ebd6b98a5d893e96343 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 3 Nov 2026 15:16:26 +0100 Subject: [PATCH 1/8] Leave host specific leftovers out of captured images Capture copied everything on the local filesystems, including user files and sockets in /tmp and /var/tmp, crash dumps, logs and the persistent journal, sssd cached credentials, package caches, the random seed, DHCP leases and root's shell history. All of it ended up on every node deployed from the profile. Drop the contents of scratch, crash, journal, mail spool, sssd cache and package cache directories, keep only the directory tree under /var/log and /var/cache/apt since daemons expect their subdirectories there, and drop a short list of individual files. The directories themselves are kept, so their mode, ownership and labels survive. mksquashfs gets the paths as an exclude file, so this applies the same way whether or not a directory is its own mount. --- imgutil/imgutil | 70 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index f7a19ce3..a5a7b2bb 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -149,6 +149,59 @@ class FileMasker(): secontext = secontext.split(b'\x00', 1)[0].decode('utf8') subprocess.check_call(['chcon', secontext, tmaskwith]) +# Directories whose contents are host specific leftovers (scratch space, +# crash dumps, logs, caches, cached credentials). The directories themselves +# are kept so their mode, ownership and labels survive. +CAPTURE_EMPTY_DIRS = ( + '/tmp', '/var/tmp', '/var/crash', '/var/log/journal', + '/var/lib/systemd/coredump', '/var/spool/abrt', '/var/spool/mail', + '/var/mail', '/var/lib/sss/db', '/var/cache/dnf', '/var/cache/yum') +# Directories where daemons expect the subdirectories to exist, only the +# files are dropped +CAPTURE_DIRS_ONLY = ('/var/log', '/var/cache/apt') +# Individual files, may be globs +CAPTURE_DROP_FILES = ( + '/var/lib/systemd/random-seed', '/var/lib/rsyslog/imjournal.state', + '/var/lib/NetworkManager/*.lease', '/var/lib/dhclient/*.lease', + '/root/.*_history', '/root/.lesshst', '/root/.viminfo') + +def get_capture_excludes(mdir): + prefix = mdir.rstrip('/') + '/' + + def capin_path(path): + if path != mdir and not path.startswith(prefix): + return None + return os.path.normpath(os.path.join( + '/run/imgutil/capin', os.path.relpath(path, mdir))) + + excludes = [] + emptied = set() + for scratch in CAPTURE_EMPTY_DIRS: + top = capin_path(scratch) + if top is None or os.path.islink(top) or not os.path.isdir(top): + continue + emptied.add(top) + excludes.extend(os.path.join(top, x) for x in os.listdir(top)) + for scratch in CAPTURE_DIRS_ONLY: + top = capin_path(scratch) + if top is None or os.path.islink(top) or not os.path.isdir(top): + continue + pending = [top] + while pending: + for entry in os.scandir(pending.pop()): + if not entry.is_dir(follow_symlinks=False): + excludes.append(entry.path) + elif entry.path not in emptied: + pending.append(entry.path) + for dropglob in CAPTURE_DROP_FILES: + top = capin_path(dropglob) + if top is None: + continue + for dropfile in glob.glob(top): + if os.path.islink(dropfile) or not os.path.isdir(dropfile): + excludes.append(dropfile) + return excludes + def capture_fs(args): fsinfo, fname = args _mount(fsinfo['mount'], '/run/imgutil/capin', flags=MS_BIND|MS_RDONLY) @@ -165,7 +218,22 @@ def capture_fs(args): masker.mask('/etc/pki/tls/private/*') masker.mask('/root/.ssh/id_*') masker.mask('/etc/netplan/*.yaml') - subprocess.check_call(['mksquashfs', '/run/imgutil/capin', fname + '.sfs', '-comp', 'xz']) + mksquashargs = ['mksquashfs', '/run/imgutil/capin', fname + '.sfs', '-comp', 'xz'] + excludes = get_capture_excludes(mdir) + exfile = None + if excludes: + # absolute paths, matched by inode, one per line + exfd, exfile = tempfile.mkstemp(dir='/run/imgutil/captmp') + with os.fdopen(exfd, 'w') as exout: + for exclude in excludes: + if '\n' not in exclude: + exout.write(exclude + '\n') + mksquashargs += ['-ef', exfile] + try: + subprocess.check_call(mksquashargs) + finally: + if exfile: + os.remove(exfile) def capture_local_cleanup(): try: From 3cf66843226151d41e8863e0731e8cadace6e070 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 3 Nov 2026 17:18:35 +0100 Subject: [PATCH 2/8] Keep hard linked files and odd names right in capture excludes mksquashfs matches excluded paths by inode, so excluding a hard linked file also dropped its other links elsewhere in the filesystem. Keep such files in the image. squashfs-tools 4.3 also splits exclude file lines on whitespace, so pass paths with whitespace or backslashes with -e instead. --- imgutil/imgutil | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index a5a7b2bb..175d75c6 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -18,6 +18,7 @@ import os import pwd import re import shutil +import stat import struct import subprocess import sys @@ -168,6 +169,15 @@ CAPTURE_DROP_FILES = ( def get_capture_excludes(mdir): prefix = mdir.rstrip('/') + '/' + def droppable(path): + # mksquashfs excludes by inode, which would also drop the other + # links of a hard linked file + try: + pinfo = os.lstat(path) + except OSError: + return False + return stat.S_ISDIR(pinfo.st_mode) or pinfo.st_nlink == 1 + def capin_path(path): if path != mdir and not path.startswith(prefix): return None @@ -181,7 +191,9 @@ def get_capture_excludes(mdir): if top is None or os.path.islink(top) or not os.path.isdir(top): continue emptied.add(top) - excludes.extend(os.path.join(top, x) for x in os.listdir(top)) + for entry in os.listdir(top): + if droppable(os.path.join(top, entry)): + excludes.append(os.path.join(top, entry)) for scratch in CAPTURE_DIRS_ONLY: top = capin_path(scratch) if top is None or os.path.islink(top) or not os.path.isdir(top): @@ -190,7 +202,8 @@ def get_capture_excludes(mdir): while pending: for entry in os.scandir(pending.pop()): if not entry.is_dir(follow_symlinks=False): - excludes.append(entry.path) + if droppable(entry.path): + excludes.append(entry.path) elif entry.path not in emptied: pending.append(entry.path) for dropglob in CAPTURE_DROP_FILES: @@ -198,7 +211,7 @@ def get_capture_excludes(mdir): if top is None: continue for dropfile in glob.glob(top): - if os.path.islink(dropfile) or not os.path.isdir(dropfile): + if (os.path.islink(dropfile) or not os.path.isdir(dropfile)) and droppable(dropfile): excludes.append(dropfile) return excludes @@ -221,14 +234,20 @@ def capture_fs(args): mksquashargs = ['mksquashfs', '/run/imgutil/capin', fname + '.sfs', '-comp', 'xz'] excludes = get_capture_excludes(mdir) exfile = None + # older mksquashfs splits exclude file lines on whitespace, pass such + # paths on the command line instead + oddexcludes = [x for x in excludes if re.search(r'[\s\\]', x)] + excludes = [x for x in excludes if not re.search(r'[\s\\]', x)] if excludes: # absolute paths, matched by inode, one per line exfd, exfile = tempfile.mkstemp(dir='/run/imgutil/captmp') with os.fdopen(exfd, 'w') as exout: for exclude in excludes: - if '\n' not in exclude: - exout.write(exclude + '\n') + exout.write(exclude + '\n') mksquashargs += ['-ef', exfile] + if oddexcludes: + # -e takes the rest of the command line, so it goes last + mksquashargs += ['-e'] + oddexcludes try: subprocess.check_call(mksquashargs) finally: From 8fd70fd49a15114155d5bc1f507cc76893e62783 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 3 Nov 2026 17:27:51 +0100 Subject: [PATCH 3/8] Leave the source network profiles out of captured images Captured images kept the NetworkManager connection profiles and ifcfg files of the source host. A diskless boot kept them too, and confignet only rewrites the profiles named like the interfaces it configures, so every other interface came up with the source host's settings, such as its static addresses. Install to disk already removes them, and capture already masks netplan files for the same reason. Drop them from the image, except ifcfg-lo. --- imgutil/imgutil | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 175d75c6..d117e42e 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -160,11 +160,17 @@ CAPTURE_EMPTY_DIRS = ( # Directories where daemons expect the subdirectories to exist, only the # files are dropped CAPTURE_DIRS_ONLY = ('/var/log', '/var/cache/apt') -# Individual files, may be globs +# Individual files, may be globs. The network profiles of the source would +# keep its addresses on interfaces confignet does not configure, deployment +# writes the node's own (as with the netplan files masked in capture_fs) CAPTURE_DROP_FILES = ( '/var/lib/systemd/random-seed', '/var/lib/rsyslog/imjournal.state', '/var/lib/NetworkManager/*.lease', '/var/lib/dhclient/*.lease', - '/root/.*_history', '/root/.lesshst', '/root/.viminfo') + '/root/.*_history', '/root/.lesshst', '/root/.viminfo', + '/etc/NetworkManager/system-connections/*', + '/etc/sysconfig/network-scripts/ifcfg-*') +# Matches of the globs above to keep +CAPTURE_KEEP_FILES = ('/etc/sysconfig/network-scripts/ifcfg-lo',) def get_capture_excludes(mdir): prefix = mdir.rstrip('/') + '/' @@ -210,7 +216,10 @@ def get_capture_excludes(mdir): top = capin_path(dropglob) if top is None: continue + keep = [capin_path(x) for x in CAPTURE_KEEP_FILES] for dropfile in glob.glob(top): + if dropfile in keep: + continue if (os.path.islink(dropfile) or not os.path.isdir(dropfile)) and droppable(dropfile): excludes.append(dropfile) return excludes From e65a4e960b0babfd88b288f0337d32d160bd3843 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 3 Nov 2026 17:41:31 +0100 Subject: [PATCH 4/8] Leave the source hostname out of captured images The image kept /etc/hostname of the source host, so a diskless node ran under that name until onboot set its own, and early journal and syslog entries carried the wrong name. With an empty /etc/hostname systemd keeps the node name the deployment initramfs already set. Install to disk writes /etc/hostname itself. --- imgutil/imgutil | 2 ++ 1 file changed, 2 insertions(+) diff --git a/imgutil/imgutil b/imgutil/imgutil index d117e42e..42a425bb 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -240,6 +240,8 @@ def capture_fs(args): masker.mask('/etc/pki/tls/private/*') masker.mask('/root/.ssh/id_*') masker.mask('/etc/netplan/*.yaml') + # an empty hostname keeps the node name the deployment initramfs set + masker.mask('/etc/hostname', '/run/imgutil/captmp/empty') mksquashargs = ['mksquashfs', '/run/imgutil/capin', fname + '.sfs', '-comp', 'xz'] excludes = get_capture_excludes(mdir) exfile = None From e8206bb9681361fbeed79ecf7407ab3e1f53da54 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 3 Nov 2026 19:34:23 +0100 Subject: [PATCH 5/8] Leave dhclient leases of Debian and Ubuntu out of captures dhclient keeps its leases in /var/lib/dhcp there, and on EL also as *.leases next to the *.lease files already excluded. Keep dhcpd.leases of a DHCP server, which lives in the same directory. --- imgutil/imgutil | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 42a425bb..37d96ec2 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -165,7 +165,9 @@ CAPTURE_DIRS_ONLY = ('/var/log', '/var/cache/apt') # writes the node's own (as with the netplan files masked in capture_fs) CAPTURE_DROP_FILES = ( '/var/lib/systemd/random-seed', '/var/lib/rsyslog/imjournal.state', - '/var/lib/NetworkManager/*.lease', '/var/lib/dhclient/*.lease', + '/var/lib/NetworkManager/*.lease', '/var/lib/dhclient/*.lease*', + # dhclient on Debian and Ubuntu, unlike dhcpd.leases of a DHCP server + '/var/lib/dhcp/dhclient*.leases', '/root/.*_history', '/root/.lesshst', '/root/.viminfo', '/etc/NetworkManager/system-connections/*', '/etc/sysconfig/network-scripts/ifcfg-*') From 80d55d2739c25eef2b13071029a9965c7cdd73a4 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 3 Nov 2026 19:35:39 +0100 Subject: [PATCH 6/8] Leave dhcpcd and NetworkManager host identity out of captures dhcpcd, present on Ubuntu 24.04 and later and on EL10, keeps its leases, DUID and secret in /var/lib/dhcpcd, and NetworkManager keeps a secret in secret_key. Copied into an image, every clone would use the same DHCPv6 DUID and derive the same stable privacy IPv6 addresses. Both regenerate them when missing. --- imgutil/imgutil | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/imgutil/imgutil b/imgutil/imgutil index 37d96ec2..814fc8aa 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -168,6 +168,11 @@ CAPTURE_DROP_FILES = ( '/var/lib/NetworkManager/*.lease', '/var/lib/dhclient/*.lease*', # dhclient on Debian and Ubuntu, unlike dhcpd.leases of a DHCP server '/var/lib/dhcp/dhclient*.leases', + # dhcpcd (Ubuntu 24.04 and later, EL10) and NetworkManager keep a per host + # DUID and secret, shared ones would give every clone the same DHCPv6 + # identity and stable privacy IPv6 addresses + '/var/lib/dhcpcd/*.lease*', '/var/lib/dhcpcd/duid', '/var/lib/dhcpcd/secret', + '/var/lib/NetworkManager/secret_key', '/root/.*_history', '/root/.lesshst', '/root/.viminfo', '/etc/NetworkManager/system-connections/*', '/etc/sysconfig/network-scripts/ifcfg-*') From ebd2d548a1e9dd5355d3e55abce07ca5218d68cd Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 3 Nov 2026 19:47:49 +0100 Subject: [PATCH 7/8] Keep source network profiles without static addresses Dropping every NetworkManager profile and ifcfg file also removed settings that confignet does not recreate from confluent attributes, such as bonds, DHCP profiles, MTU or the InfiniBand transport mode. Only drop the profiles that carry static addresses of the source. --- imgutil/imgutil | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 814fc8aa..e8187531 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -160,9 +160,7 @@ CAPTURE_EMPTY_DIRS = ( # Directories where daemons expect the subdirectories to exist, only the # files are dropped CAPTURE_DIRS_ONLY = ('/var/log', '/var/cache/apt') -# Individual files, may be globs. The network profiles of the source would -# keep its addresses on interfaces confignet does not configure, deployment -# writes the node's own (as with the netplan files masked in capture_fs) +# Individual files, may be globs CAPTURE_DROP_FILES = ( '/var/lib/systemd/random-seed', '/var/lib/rsyslog/imjournal.state', '/var/lib/NetworkManager/*.lease', '/var/lib/dhclient/*.lease*', @@ -173,9 +171,17 @@ CAPTURE_DROP_FILES = ( # identity and stable privacy IPv6 addresses '/var/lib/dhcpcd/*.lease*', '/var/lib/dhcpcd/duid', '/var/lib/dhcpcd/secret', '/var/lib/NetworkManager/secret_key', - '/root/.*_history', '/root/.lesshst', '/root/.viminfo', - '/etc/NetworkManager/system-connections/*', - '/etc/sysconfig/network-scripts/ifcfg-*') + '/root/.*_history', '/root/.lesshst', '/root/.viminfo') +# Network profiles with static addresses of the source, which would come up +# on interfaces confignet does not configure (deployment writes the node's +# own, as with the netplan files masked in capture_fs). Profiles without +# addresses, such as bonds, MTU or InfiniBand mode settings, are kept. +CAPTURE_DROP_STATIC = { + '/etc/NetworkManager/system-connections/*': + re.compile(r'^\s*(address\d*|addresses|method\s*=\s*manual)\b', re.M), + '/etc/sysconfig/network-scripts/ifcfg-*': + re.compile(r'^\s*(IPADDR\d*|IPV6ADDR(_SECONDARIES)?)\s*=', re.M), +} # Matches of the globs above to keep CAPTURE_KEEP_FILES = ('/etc/sysconfig/network-scripts/ifcfg-lo',) @@ -219,16 +225,24 @@ def get_capture_excludes(mdir): excludes.append(entry.path) elif entry.path not in emptied: pending.append(entry.path) - for dropglob in CAPTURE_DROP_FILES: + keep = [capin_path(x) for x in CAPTURE_KEEP_FILES] + for dropglob in CAPTURE_DROP_FILES + tuple(CAPTURE_DROP_STATIC): top = capin_path(dropglob) if top is None: continue - keep = [capin_path(x) for x in CAPTURE_KEEP_FILES] for dropfile in glob.glob(top): if dropfile in keep: continue - if (os.path.islink(dropfile) or not os.path.isdir(dropfile)) and droppable(dropfile): - excludes.append(dropfile) + if (os.path.isdir(dropfile) and not os.path.islink(dropfile)) or not droppable(dropfile): + continue + if dropglob in CAPTURE_DROP_STATIC: + try: + with open(dropfile, errors='replace') as profilein: + if not CAPTURE_DROP_STATIC[dropglob].search(profilein.read()): + continue + except OSError: + continue + excludes.append(dropfile) return excludes def capture_fs(args): From 8fd41b13150a8a6cb0f46051753e382d04ad4f81 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 3 Nov 2026 21:16:47 +0100 Subject: [PATCH 8/8] Clean up the capture initramfs config on Ubuntu sources Capture copies an initramfs-tools hook and conf.d/confluent.conf into the source to build the deployment initramfs, but only ever removed the EL dracut module afterwards. With MODULES=list left in place, the source's next initramfs update was limited to the hook's driver list, and a source captured again carried both files into the image, where rebuilding the initramfs on install to disk shrank the clones' initramfs the same way. Remove them after capture and leave them out of the image. --- imgutil/imgutil | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index e8187531..4d6e5f4e 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -150,6 +150,11 @@ class FileMasker(): secontext = secontext.split(b'\x00', 1)[0].decode('utf8') subprocess.check_call(['chcon', secontext, tmaskwith]) +# What capture adds to an Ubuntu source to build the deployment initramfs. +# MODULES=list in there would also shrink the initramfs of the source and of +# installed clones to the hook's driver list. +CAPTURE_BOOT_FILES = ('/etc/initramfs-tools/hooks/confluent', + '/etc/initramfs-tools/conf.d/confluent.conf') # Directories whose contents are host specific leftovers (scratch space, # crash dumps, logs, caches, cached credentials). The directories themselves # are kept so their mode, ownership and labels survive. @@ -171,7 +176,7 @@ CAPTURE_DROP_FILES = ( # identity and stable privacy IPv6 addresses '/var/lib/dhcpcd/*.lease*', '/var/lib/dhcpcd/duid', '/var/lib/dhcpcd/secret', '/var/lib/NetworkManager/secret_key', - '/root/.*_history', '/root/.lesshst', '/root/.viminfo') + '/root/.*_history', '/root/.lesshst', '/root/.viminfo') + CAPTURE_BOOT_FILES # Network profiles with static addresses of the source, which would come up # on interfaces confignet does not configure (deployment writes the node's # own, as with the netplan files masked in capture_fs). Profiles without @@ -291,6 +296,11 @@ def capture_local_cleanup(): shutil.rmtree('/usr/lib/dracut/modules.d/97confluent') except Exception: pass + for bootfile in CAPTURE_BOOT_FILES: + try: + os.remove(bootfile) + except FileNotFoundError: + pass subprocess.check_call(['umount', '/run/imgutil/capout']) def build_boot_tree(targpath):