From cfc4490fe116cf8486ee250d018996f228165201 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 16 Jul 2026 16:55:16 +0200 Subject: [PATCH] Use multi-threaded unsquashfs to extract untethered images `unsquashfs` can use multiple CPU cores during image extraction, significantly reducing boot time. For example the whole boot time from PXE to shell on a 8-core VM, from approximately 45 seconds to 20 seconds. This PR adds `squashfs-tools` as a dependency. Since the package is smaller than 1 MB, the additional image size is justified by the performance improvement. For backward compatibility, the existing `cp`-based extraction method is used when `unsquashfs` is unavailable, such as with images built before this change. The extraction logic has also been moved into the common functions and is now shared between EL9, EL10, and Ubuntu. Both untethered `squashfs` images and `confluent_multisquash` images are supported. Images must be rebuilt to include `unsquashfs` and benefit from the faster extraction path. --- .../common/profile/scripts/functions | 101 ++++++++++++++++++ .../profiles/default/scripts/imageboot.sh | 28 +---- .../profiles/default/scripts/imageboot.sh | 27 +---- .../profiles/default/scripts/imageboot.sh | 27 +---- imgutil/el10/dracut/install | 3 +- imgutil/el10/pkglist | 4 +- imgutil/el10/pkglist.aarch64 | 2 +- imgutil/el9/dracut/install | 3 +- imgutil/el9/pkglist | 4 +- imgutil/el9/pkglist.aarch64 | 2 +- imgutil/imgutil | 6 ++ .../ubuntu/initramfs-tools/hooks/confluent | 1 + 12 files changed, 128 insertions(+), 80 deletions(-) diff --git a/confluent_osdeploy/common/profile/scripts/functions b/confluent_osdeploy/common/profile/scripts/functions index a88ba210..1a5aebf8 100644 --- a/confluent_osdeploy/common/profile/scripts/functions +++ b/confluent_osdeploy/common/profile/scripts/functions @@ -212,5 +212,106 @@ run_remote_config() { echo '---------------------------------------------------------------------------' return } +startlegacyprogress() { + [ -n "$progresspid" ] && return + echo -en "Decrypting and extracting root filesystem: 0%\r" + srcsz=$(du -sk /mnt/remote | awk '{print $1}') + while [ -f /mnt/remoteimg/rootimg.sfs ]; do + dstsz=$(du -sk /sysroot | awk '{print $1}') + pct=$((dstsz * 100 / srcsz)) + if [ $pct -gt 99 ]; then + pct=99 + fi + echo -en "Decrypting and extracting root filesystem: $pct%\r" + sleep 0.25 + done & + progresspid=$! +} + +extractmultisquash() { + multisquashrc=1 + while read -r _ partsrc partmount; do + normalizedmount=${partmount%/} + [ "$normalizedmount" = /mnt/remote ] || continue + if unsquashfs -force -d /sysroot "$partsrc"; then + multisquashrc=0 + fi + break + done < /tmp/mountparts.sh + [ $multisquashrc -eq 0 ] || return 1 + while read -r _ partsrc partmount; do + normalizedmount=${partmount%/} + [ "$normalizedmount" = /mnt/remote ] && continue + partdest=/sysroot${normalizedmount#/mnt/remote} + mkdir -p "$partdest" + if ! unsquashfs -force -d "$partdest" "$partsrc"; then + return 1 + fi + done < /tmp/mountparts.sh +} + +cleanupremotemounts() { + if [ -f /tmp/mountparts.sh ]; then + for partmount in $(awk '{ mounts[NR] = $3 } END { for (idx = NR; idx > 0; idx--) print mounts[idx] }' /tmp/mountparts.sh); do + umount "$partmount" + done + for partdev in $(awk '{ devices[NR] = $NF } END { for (idx = NR; idx > 0; idx--) print devices[idx] }' /tmp/setupmount.sh); do + dmsetup remove "$partdev" + done + else + umount /mnt/remote + fi +} + +# Extract an untethered/uncompressed diskless root image into /sysroot and +# tear down the source devices; expects rootimgformat, mountsrc, and loopdev +# from imageboot.sh and the image content mounted under /mnt/remote. +extract_untethered_rootimg() { + extractrc=1 + progresspid= + if [ "$rootimgformat" = squashfs ] && command -v unsquashfs > /dev/null 2>&1; then + echo "Decrypting and extracting root filesystem in parallel" + if unsquashfs -force -d /sysroot "$mountsrc"; then + extractrc=0 + else + echo "Parallel root filesystem extraction failed, retrying with cp" + fi + elif [ "$rootimgformat" = confluent_multisquash ] && command -v unsquashfs > /dev/null 2>&1; then + echo "Decrypting and extracting multipart root filesystem in parallel" + if extractmultisquash; then + extractrc=0 + else + echo "Parallel multipart root filesystem extraction failed, retrying with cp" + fi + fi + if [ $extractrc -ne 0 ]; then + rm -rf /sysroot/* /sysroot/.[!.]* /sysroot/..?* + startlegacyprogress + if cp -a /mnt/remote/. /sysroot/; then + extractrc=0 + fi + fi + if [ $extractrc -ne 0 ]; then + if [ -n "$progresspid" ]; then + kill "$progresspid" 2> /dev/null || true + wait "$progresspid" 2> /dev/null || true + progresspid= + fi + echo "Failed to extract the root filesystem" + fi + cleanupremotemounts + if [ -e /dev/mapper/cryptimg ]; then + dmsetup remove cryptimg + fi + losetup -d $loopdev + rm /mnt/remoteimg/rootimg.sfs + umount /mnt/remoteimg + if [ -n "$progresspid" ]; then + wait "$progresspid" + echo -e "Decrypting and extracting root filesystem: 100%" + fi + return $extractrc +} + #If invoked as a command, use the arguments to actually run a function (return 0 2>/dev/null) || $1 "${@:2}" diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/imageboot.sh b/confluent_osdeploy/el10-diskless/profiles/default/scripts/imageboot.sh index df34883d..3e3c3c13 100644 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/imageboot.sh +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/imageboot.sh @@ -29,9 +29,10 @@ if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then mountsrc=/dev/mapper/cryptimg fi -if grep '^Format: squashfs' /tmp/rootimg.info > /dev/null; then +rootimgformat=$(awk -F': ' '/^Format:/ {print $2; exit}' /tmp/rootimg.info) +if [ "$rootimgformat" = squashfs ]; then mount -o ro $mountsrc /mnt/remote -elif grep '^Format: confluent_multisquash' /tmp/rootimg.info; then +elif [ "$rootimgformat" = confluent_multisquash ]; then tail -n +3 /tmp/rootimg.info | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh . /tmp/setupmount.sh cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh @@ -54,27 +55,7 @@ if [ "untethered" = "$(getarg confluent_imagemethod)" -o "uncompressed" = "$(get else mount -t tmpfs disklessroot /sysroot fi - echo -en "Decrypting and extracting root filesystem: 0%\r" - srcsz=$(du -sk /mnt/remote | awk '{print $1}') - while [ -f /mnt/remoteimg/rootimg.sfs ]; do - dstsz=$(du -sk /sysroot | awk '{print $1}') - pct=$((dstsz * 100 / srcsz)) - if [ $pct -gt 99 ]; then - pct=99 - fi - echo -en "Decrypting and extracting root filesystem: $pct%\r" - sleep 0.25 - done & - cp -ax /mnt/remote/* /sysroot/ - umount /mnt/remote - if [ -e /dev/mapper/cryptimg ]; then - dmsetup remove cryptimg - fi - losetup -d $loopdev - rm /mnt/remoteimg/rootimg.sfs - umount /mnt/remoteimg - wait - echo -e "Decrypting and extracting root filesystem: 100%" + extract_untethered_rootimg || return 1 else TETHERED=1 mount -o discard /dev/zram0 /mnt/overlay @@ -183,4 +164,3 @@ if grep debugssh /proc/cmdline >& /dev/null; then else exec /opt/confluent/bin/start_root -s # share mount namespace, keep kernel callbacks intact fi - diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh b/confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh index ec188a2b..1df5f015 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh @@ -29,9 +29,10 @@ if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then mountsrc=/dev/mapper/cryptimg fi -if grep '^Format: squashfs' /tmp/rootimg.info > /dev/null; then +rootimgformat=$(awk -F': ' '/^Format:/ {print $2; exit}' /tmp/rootimg.info) +if [ "$rootimgformat" = squashfs ]; then mount -o ro $mountsrc /mnt/remote -elif grep '^Format: confluent_multisquash' /tmp/rootimg.info; then +elif [ "$rootimgformat" = confluent_multisquash ]; then tail -n +3 /tmp/rootimg.info | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh . /tmp/setupmount.sh cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh @@ -54,27 +55,7 @@ if [ "untethered" = "$(getarg confluent_imagemethod)" -o "uncompressed" = "$(get else mount -t tmpfs disklessroot /sysroot fi - echo -en "Decrypting and extracting root filesystem: 0%\r" - srcsz=$(du -sk /mnt/remote | awk '{print $1}') - while [ -f /mnt/remoteimg/rootimg.sfs ]; do - dstsz=$(du -sk /sysroot | awk '{print $1}') - pct=$((dstsz * 100 / srcsz)) - if [ $pct -gt 99 ]; then - pct=99 - fi - echo -en "Decrypting and extracting root filesystem: $pct%\r" - sleep 0.25 - done & - cp -ax /mnt/remote/* /sysroot/ - umount /mnt/remote - if [ -e /dev/mapper/cryptimg ]; then - dmsetup remove cryptimg - fi - losetup -d $loopdev - rm /mnt/remoteimg/rootimg.sfs - umount /mnt/remoteimg - wait - echo -e "Decrypting and extracting root filesystem: 100%" + extract_untethered_rootimg || return 1 else TETHERED=1 mount -o discard /dev/zram0 /mnt/overlay diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh index 8f2f4701..64ca8c78 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh @@ -36,9 +36,10 @@ if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then mountsrc=/dev/mapper/cryptimg fi -if grep '^Format: squashfs' /tmp/rootimg.info > /dev/null; then +rootimgformat=$(awk -F': ' '/^Format:/ {print $2; exit}' /tmp/rootimg.info) +if [ "$rootimgformat" = squashfs ]; then mount -o ro $mountsrc /mnt/remote -elif grep '^Format: confluent_multisquash' /tmp/rootimg.info; then +elif [ "$rootimgformat" = confluent_multisquash ]; then tail -n +3 /tmp/rootimg.info | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh . /tmp/setupmount.sh cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh @@ -64,27 +65,7 @@ elif grep -q confluent_imagemethod=uncompressed /proc/cmdline; then mount -t tmpfs disklessroot /sysroot fi if [ "$TETHERED" = 0 ]; then - echo -en "Decrypting and extracting root filesystem: 0%\r" - srcsz=$(du -sk /mnt/remote | awk '{print $1}') - while [ -f /mnt/remoteimg/rootimg.sfs ]; do - dstsz=$(du -sk /sysroot | awk '{print $1}') - pct=$((dstsz * 100 / srcsz)) - if [ $pct -gt 99 ]; then - pct=99 - fi - echo -en "Decrypting and extracting root filesystem: $pct%\r" - sleep 0.25 - done & - cp -a /mnt/remote/* /sysroot/ - umount /mnt/remote - if [ -e /dev/mapper/cryptimg ]; then - dmsetup remove cryptimg - fi - losetup -d $loopdev - rm /mnt/remoteimg/rootimg.sfs - umount /mnt/remoteimg - wait - echo -e "Decrypting and extracting root filesystem: 100%" + extract_untethered_rootimg || return 1 elif [ ! -f /tmp/mountparts.sh ]; then mkdir -p /mnt/overlay/upper /mnt/overlay/work mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot diff --git a/imgutil/el10/dracut/install b/imgutil/el10/dracut/install index bface540..f597e39e 100644 --- a/imgutil/el10/dracut/install +++ b/imgutil/el10/dracut/install @@ -3,7 +3,7 @@ dracut_install /lib64/libtss2-tcti-device.so.0 dracut_install tpm2_create tpm2_pcrread tpm2_createpolicy tpm2_createprimary dracut_install tpm2_load tpm2_unseal tpm2_getcap tpm2_evictcontrol dracut_install tpm2_pcrextend tpm2_policypcr tpm2_flushcontext tpm2_startauthsession -dracut_install curl openssl tar cpio gzip lsmod ethtool xz lsmod ethtool +dracut_install curl openssl tar cpio gzip lsmod ethtool xz unsquashfs lsmod ethtool dracut_install modprobe touch echo cut wc bash uniq grep ip hostname dracut_install awk egrep dirname expr sort dracut_install ssh sshd reboot parted mkfs mkfs.ext4 mkfs.xfs xfs_db mkswap @@ -33,4 +33,3 @@ inst /usr/lib/dracut/modules.d/45net-lib/net-lib.sh /lib/net-lib.sh # network mount, and disk imaging helpers can come from a second stage # this is narrowly focused on getting network up and fetching images # and those images may opt to do something with cloning or whatever - diff --git a/imgutil/el10/pkglist b/imgutil/el10/pkglist index 0bee79dd..47114c51 100644 --- a/imgutil/el10/pkglist +++ b/imgutil/el10/pkglist @@ -19,8 +19,8 @@ fuse-libs libnl3 dhcpcd openssh-keysign -chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr attr +chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr attr squashfs-tools %onlyarch x86_64 shim-x64.x86_64 grub2-efi-x64 %onlyarch aarch64 -shim-aa64.aarch64 grub2-efi-aa64 \ No newline at end of file +shim-aa64.aarch64 grub2-efi-aa64 diff --git a/imgutil/el10/pkglist.aarch64 b/imgutil/el10/pkglist.aarch64 index 0d23e958..993e2be5 100644 --- a/imgutil/el10/pkglist.aarch64 +++ b/imgutil/el10/pkglist.aarch64 @@ -19,4 +19,4 @@ fuse-libs libnl3 dhcpcd openssh-keysign -chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr shim-aa64 grub2-efi-aa64 attr +chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr shim-aa64 grub2-efi-aa64 attr squashfs-tools diff --git a/imgutil/el9/dracut/install b/imgutil/el9/dracut/install index 27e68a2f..362e3bc6 100644 --- a/imgutil/el9/dracut/install +++ b/imgutil/el9/dracut/install @@ -3,7 +3,7 @@ dracut_install /lib64/libtss2-tcti-device.so.0 dracut_install tpm2_create tpm2_pcrread tpm2_createpolicy tpm2_createprimary dracut_install tpm2_load tpm2_unseal tpm2_getcap tpm2_evictcontrol dracut_install tpm2_pcrextend tpm2_policypcr tpm2_flushcontext tpm2_startauthsession -dracut_install curl openssl tar cpio gzip lsmod ethtool xz lsmod ethtool +dracut_install curl openssl tar cpio gzip lsmod ethtool xz unsquashfs lsmod ethtool dracut_install modprobe touch echo cut wc bash uniq grep ip hostname dracut_install awk egrep dirname expr sort dracut_install ssh sshd reboot parted mkfs mkfs.ext4 mkfs.xfs xfs_db mkswap @@ -32,4 +32,3 @@ inst /usr/lib/dracut/modules.d/40network/net-lib.sh /lib/net-lib.sh # network mount, and disk imaging helpers can come from a second stage # this is narrowly focused on getting network up and fetching images # and those images may opt to do something with cloning or whatever - diff --git a/imgutil/el9/pkglist b/imgutil/el9/pkglist index 44eaaf6d..511145b3 100644 --- a/imgutil/el9/pkglist +++ b/imgutil/el9/pkglist @@ -18,8 +18,8 @@ xfsprogs e2fsprogs fuse-libs libnl3 -chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr attr +chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr attr squashfs-tools %onlyarch x86_64 shim-x64.x86_64 grub2-efi-x64 %onlyarch aarch64 -shim-aa64.aarch64 grub2-efi-aa64 \ No newline at end of file +shim-aa64.aarch64 grub2-efi-aa64 diff --git a/imgutil/el9/pkglist.aarch64 b/imgutil/el9/pkglist.aarch64 index 2141634a..63ac2c51 100644 --- a/imgutil/el9/pkglist.aarch64 +++ b/imgutil/el9/pkglist.aarch64 @@ -17,4 +17,4 @@ xfsprogs e2fsprogs fuse-libs libnl3 -chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr shim-aa64 grub2-efi-aa64 attr +chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr shim-aa64 grub2-efi-aa64 attr squashfs-tools diff --git a/imgutil/imgutil b/imgutil/imgutil index 28f9e4c5..772285f1 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -683,6 +683,8 @@ class DebHandler(OsHandler): needpkgs = [] if not os.path.exists(os.path.join(hostpath, 'usr/bin/tpm2_getcap')): needpkgs.append('tpm2-tools') + if not os.path.exists(os.path.join(hostpath, 'usr/bin/unsquashfs')): + needpkgs.append('squashfs-tools') lfuses = glob.glob(os.path.join(hostpath, '/lib/*/libfuse.so.2')) if not lfuses: needpkgs.append('libfuse2') @@ -779,6 +781,10 @@ class ElHandler(OsHandler): needpkgs.append('dhcp-client') if not os.path.exists(os.path.join(hostpath, 'usr/sbin/mount.nfs')): needpkgs.append('nfs-utils') + if (self.oscategory in ('el9', 'el10') and + not os.path.exists(os.path.join(hostpath, 'usr/sbin/unsquashfs')) and + not os.path.exists(os.path.join(hostpath, 'usr/bin/unsquashfs'))): + needpkgs.append('squashfs-tools') if needpkgs: needapt = 'Missing packages needed in target for capture, to add required packages: dnf install ' + ' '.join(needpkgs) self.captureprereqs.append(needapt) diff --git a/imgutil/ubuntu/initramfs-tools/hooks/confluent b/imgutil/ubuntu/initramfs-tools/hooks/confluent index 611a1651..62e95c55 100644 --- a/imgutil/ubuntu/initramfs-tools/hooks/confluent +++ b/imgutil/ubuntu/initramfs-tools/hooks/confluent @@ -33,6 +33,7 @@ copy_exec /usr/bin/tpm2_pcrextend copy_exec /usr/bin/ssh-keygen copy_exec /usr/sbin/sshd copy_exec /usr/sbin/mkfs.xfs +copy_exec /usr/bin/unsquashfs [ -e /usr/lib/openssh/sshd-session ] && copy_exec /usr/lib/openssh/sshd-session [ -e /usr/lib/x86_64-linux-gnu/libfuse.so.2 ] && copy_exec /usr/lib/x86_64-linux-gnu/libfuse.so.2 [ -e /usr/lib/aarch64-linux-gnu/libfuse.so.2 ] && copy_exec /usr/lib/aarch64-linux-gnu/libfuse.so.2