diff --git a/confluent_osdeploy/common/profile/scripts/functions b/confluent_osdeploy/common/profile/scripts/functions index 6cfd5c3b..d5209eeb 100644 --- a/confluent_osdeploy/common/profile/scripts/functions +++ b/confluent_osdeploy/common/profile/scripts/functions @@ -216,5 +216,106 @@ run_remote_config() { echo '---------------------------------------------------------------------------' return } +startlegacyprogress() { + [ -n "$progresspid" ] && return + echo -en "Decrypting and extracting root filesystem: 0%\r" + srcsz=$(du -sk /mnt/remote | awk '{print $1}') + while [ -f /mnt/remoteimg/rootimg.sfs ]; do + dstsz=$(du -sk /sysroot | awk '{print $1}') + pct=$((dstsz * 100 / srcsz)) + if [ $pct -gt 99 ]; then + pct=99 + fi + echo -en "Decrypting and extracting root filesystem: $pct%\r" + sleep 0.25 + done & + progresspid=$! +} + +extractmultisquash() { + multisquashrc=1 + while read -r _ partsrc partmount; do + normalizedmount=${partmount%/} + [ "$normalizedmount" = /mnt/remote ] || continue + if unsquashfs -force -d /sysroot "$partsrc"; then + multisquashrc=0 + fi + break + done < /tmp/mountparts.sh + [ $multisquashrc -eq 0 ] || return 1 + while read -r _ partsrc partmount; do + normalizedmount=${partmount%/} + [ "$normalizedmount" = /mnt/remote ] && continue + partdest=/sysroot${normalizedmount#/mnt/remote} + mkdir -p "$partdest" + if ! unsquashfs -force -d "$partdest" "$partsrc"; then + return 1 + fi + done < /tmp/mountparts.sh +} + +cleanupremotemounts() { + if [ -f /tmp/mountparts.sh ]; then + for partmount in $(awk '{ mounts[NR] = $3 } END { for (idx = NR; idx > 0; idx--) print mounts[idx] }' /tmp/mountparts.sh); do + umount "$partmount" + done + for partdev in $(awk '{ devices[NR] = $NF } END { for (idx = NR; idx > 0; idx--) print devices[idx] }' /tmp/setupmount.sh); do + dmsetup remove "$partdev" + done + else + umount /mnt/remote + fi +} + +# Extract an untethered/uncompressed diskless root image into /sysroot and +# tear down the source devices; expects rootimgformat, mountsrc, and loopdev +# from imageboot.sh and the image content mounted under /mnt/remote. +extract_untethered_rootimg() { + extractrc=1 + progresspid= + if [ "$rootimgformat" = squashfs ] && command -v unsquashfs > /dev/null 2>&1; then + echo "Decrypting and extracting root filesystem in parallel" + if unsquashfs -force -d /sysroot "$mountsrc"; then + extractrc=0 + else + echo "Parallel root filesystem extraction failed, retrying with cp" + fi + elif [ "$rootimgformat" = confluent_multisquash ] && command -v unsquashfs > /dev/null 2>&1; then + echo "Decrypting and extracting multipart root filesystem in parallel" + if extractmultisquash; then + extractrc=0 + else + echo "Parallel multipart root filesystem extraction failed, retrying with cp" + fi + fi + if [ $extractrc -ne 0 ]; then + rm -rf /sysroot/* /sysroot/.[!.]* /sysroot/..?* + startlegacyprogress + if cp -a /mnt/remote/. /sysroot/; then + extractrc=0 + fi + fi + if [ $extractrc -ne 0 ]; then + if [ -n "$progresspid" ]; then + kill "$progresspid" 2> /dev/null || true + wait "$progresspid" 2> /dev/null || true + progresspid= + fi + echo "Failed to extract the root filesystem" + fi + cleanupremotemounts + if [ -e /dev/mapper/cryptimg ]; then + dmsetup remove cryptimg + fi + losetup -d $loopdev + rm /mnt/remoteimg/rootimg.sfs + umount /mnt/remoteimg + if [ -n "$progresspid" ]; then + wait "$progresspid" + echo -e "Decrypting and extracting root filesystem: 100%" + fi + return $extractrc +} + #If invoked as a command, use the arguments to actually run a function (return 0 2>/dev/null) || $1 "${@:2}" diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/imageboot.sh b/confluent_osdeploy/el10-diskless/profiles/default/scripts/imageboot.sh index bc7e5f9c..0bc7300b 100644 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/imageboot.sh +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/imageboot.sh @@ -30,9 +30,10 @@ if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then mountsrc=/dev/mapper/cryptimg fi -if grep '^Format: squashfs' /tmp/rootimg.info > /dev/null; then +rootimgformat=$(awk -F': ' '/^Format:/ {print $2; exit}' /tmp/rootimg.info) +if [ "$rootimgformat" = squashfs ]; then mount -o ro $mountsrc /mnt/remote -elif grep '^Format: confluent_multisquash' /tmp/rootimg.info; then +elif [ "$rootimgformat" = confluent_multisquash ]; then tail -n +3 /tmp/rootimg.info | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh . /tmp/setupmount.sh cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh @@ -55,27 +56,7 @@ if [ "untethered" = "$(getarg confluent_imagemethod)" -o "uncompressed" = "$(get else mount -t tmpfs disklessroot /sysroot fi - echo -en "Decrypting and extracting root filesystem: 0%\r" - srcsz=$(du -sk /mnt/remote | awk '{print $1}') - while [ -f /mnt/remoteimg/rootimg.sfs ]; do - dstsz=$(du -sk /sysroot | awk '{print $1}') - pct=$((dstsz * 100 / srcsz)) - if [ $pct -gt 99 ]; then - pct=99 - fi - echo -en "Decrypting and extracting root filesystem: $pct%\r" - sleep 0.25 - done & - cp -ax /mnt/remote/* /sysroot/ - umount /mnt/remote - if [ -e /dev/mapper/cryptimg ]; then - dmsetup remove cryptimg - fi - losetup -d $loopdev - rm /mnt/remoteimg/rootimg.sfs - umount /mnt/remoteimg - wait - echo -e "Decrypting and extracting root filesystem: 100%" + extract_untethered_rootimg else TETHERED=1 mount -o discard /dev/zram0 /mnt/overlay @@ -188,4 +169,3 @@ if grep debugssh /proc/cmdline >& /dev/null; then else exec /opt/confluent/bin/start_root -s # share mount namespace, keep kernel callbacks intact fi - diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh b/confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh index 0f870319..7dd29141 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh @@ -29,9 +29,10 @@ if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then mountsrc=/dev/mapper/cryptimg fi -if grep '^Format: squashfs' /tmp/rootimg.info > /dev/null; then +rootimgformat=$(awk -F': ' '/^Format:/ {print $2; exit}' /tmp/rootimg.info) +if [ "$rootimgformat" = squashfs ]; then mount -o ro $mountsrc /mnt/remote -elif grep '^Format: confluent_multisquash' /tmp/rootimg.info; then +elif [ "$rootimgformat" = confluent_multisquash ]; then tail -n +3 /tmp/rootimg.info | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh . /tmp/setupmount.sh cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh @@ -54,27 +55,7 @@ if [ "untethered" = "$(getarg confluent_imagemethod)" -o "uncompressed" = "$(get else mount -t tmpfs disklessroot /sysroot fi - echo -en "Decrypting and extracting root filesystem: 0%\r" - srcsz=$(du -sk /mnt/remote | awk '{print $1}') - while [ -f /mnt/remoteimg/rootimg.sfs ]; do - dstsz=$(du -sk /sysroot | awk '{print $1}') - pct=$((dstsz * 100 / srcsz)) - if [ $pct -gt 99 ]; then - pct=99 - fi - echo -en "Decrypting and extracting root filesystem: $pct%\r" - sleep 0.25 - done & - cp -ax /mnt/remote/* /sysroot/ - umount /mnt/remote - if [ -e /dev/mapper/cryptimg ]; then - dmsetup remove cryptimg - fi - losetup -d $loopdev - rm /mnt/remoteimg/rootimg.sfs - umount /mnt/remoteimg - wait - echo -e "Decrypting and extracting root filesystem: 100%" + extract_untethered_rootimg else TETHERED=1 mount -o discard /dev/zram0 /mnt/overlay diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh index 8f2f4701..df23007f 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh @@ -36,9 +36,10 @@ if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then mountsrc=/dev/mapper/cryptimg fi -if grep '^Format: squashfs' /tmp/rootimg.info > /dev/null; then +rootimgformat=$(awk -F': ' '/^Format:/ {print $2; exit}' /tmp/rootimg.info) +if [ "$rootimgformat" = squashfs ]; then mount -o ro $mountsrc /mnt/remote -elif grep '^Format: confluent_multisquash' /tmp/rootimg.info; then +elif [ "$rootimgformat" = confluent_multisquash ]; then tail -n +3 /tmp/rootimg.info | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh . /tmp/setupmount.sh cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh @@ -64,27 +65,7 @@ elif grep -q confluent_imagemethod=uncompressed /proc/cmdline; then mount -t tmpfs disklessroot /sysroot fi if [ "$TETHERED" = 0 ]; then - echo -en "Decrypting and extracting root filesystem: 0%\r" - srcsz=$(du -sk /mnt/remote | awk '{print $1}') - while [ -f /mnt/remoteimg/rootimg.sfs ]; do - dstsz=$(du -sk /sysroot | awk '{print $1}') - pct=$((dstsz * 100 / srcsz)) - if [ $pct -gt 99 ]; then - pct=99 - fi - echo -en "Decrypting and extracting root filesystem: $pct%\r" - sleep 0.25 - done & - cp -a /mnt/remote/* /sysroot/ - umount /mnt/remote - if [ -e /dev/mapper/cryptimg ]; then - dmsetup remove cryptimg - fi - losetup -d $loopdev - rm /mnt/remoteimg/rootimg.sfs - umount /mnt/remoteimg - wait - echo -e "Decrypting and extracting root filesystem: 100%" + extract_untethered_rootimg elif [ ! -f /tmp/mountparts.sh ]; then mkdir -p /mnt/overlay/upper /mnt/overlay/work mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot diff --git a/imgutil/el10/dracut/install b/imgutil/el10/dracut/install index 8578565c..e5a2d910 100644 --- a/imgutil/el10/dracut/install +++ b/imgutil/el10/dracut/install @@ -3,7 +3,8 @@ dracut_install /lib64/libtss2-tcti-device.so.0 dracut_install tpm2_create tpm2_pcrread tpm2_createpolicy tpm2_createprimary dracut_install tpm2_load tpm2_unseal tpm2_getcap tpm2_evictcontrol dracut_install tpm2_pcrextend tpm2_policypcr tpm2_flushcontext tpm2_startauthsession -dracut_install curl openssl tar cpio gzip lsmod ethtool xz lsmod ethtool +dracut_install curl openssl tar cpio gzip lsmod ethtool xz +dracut_install -o unsquashfs # optional, imageboot extracts with cp when absent dracut_install modprobe touch echo cut wc bash uniq grep ip hostname dracut_install awk egrep dirname expr sort dracut_install ssh sshd reboot parted mkfs mkfs.ext4 mkfs.xfs xfs_db mkswap @@ -33,4 +34,3 @@ inst /usr/lib/dracut/modules.d/45net-lib/net-lib.sh /lib/net-lib.sh # network mount, and disk imaging helpers can come from a second stage # this is narrowly focused on getting network up and fetching images # and those images may opt to do something with cloning or whatever - diff --git a/imgutil/el10/pkglist b/imgutil/el10/pkglist index 0bee79dd..47114c51 100644 --- a/imgutil/el10/pkglist +++ b/imgutil/el10/pkglist @@ -19,8 +19,8 @@ fuse-libs libnl3 dhcpcd openssh-keysign -chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr attr +chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr attr squashfs-tools %onlyarch x86_64 shim-x64.x86_64 grub2-efi-x64 %onlyarch aarch64 -shim-aa64.aarch64 grub2-efi-aa64 \ No newline at end of file +shim-aa64.aarch64 grub2-efi-aa64 diff --git a/imgutil/el10/pkglist.aarch64 b/imgutil/el10/pkglist.aarch64 index 0d23e958..993e2be5 100644 --- a/imgutil/el10/pkglist.aarch64 +++ b/imgutil/el10/pkglist.aarch64 @@ -19,4 +19,4 @@ fuse-libs libnl3 dhcpcd openssh-keysign -chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr shim-aa64 grub2-efi-aa64 attr +chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr shim-aa64 grub2-efi-aa64 attr squashfs-tools diff --git a/imgutil/el9/dracut/install b/imgutil/el9/dracut/install index 46bf6b75..2e334295 100644 --- a/imgutil/el9/dracut/install +++ b/imgutil/el9/dracut/install @@ -3,7 +3,8 @@ dracut_install /lib64/libtss2-tcti-device.so.0 dracut_install tpm2_create tpm2_pcrread tpm2_createpolicy tpm2_createprimary dracut_install tpm2_load tpm2_unseal tpm2_getcap tpm2_evictcontrol dracut_install tpm2_pcrextend tpm2_policypcr tpm2_flushcontext tpm2_startauthsession -dracut_install curl openssl tar cpio gzip lsmod ethtool xz lsmod ethtool +dracut_install curl openssl tar cpio gzip lsmod ethtool xz +dracut_install -o unsquashfs # optional, imageboot extracts with cp when absent dracut_install modprobe touch echo cut wc bash uniq grep ip hostname dracut_install awk egrep dirname expr sort dracut_install ssh sshd reboot parted mkfs mkfs.ext4 mkfs.xfs xfs_db mkswap @@ -32,4 +33,3 @@ inst /usr/lib/dracut/modules.d/40network/net-lib.sh /lib/net-lib.sh # network mount, and disk imaging helpers can come from a second stage # this is narrowly focused on getting network up and fetching images # and those images may opt to do something with cloning or whatever - diff --git a/imgutil/el9/pkglist b/imgutil/el9/pkglist index 44eaaf6d..511145b3 100644 --- a/imgutil/el9/pkglist +++ b/imgutil/el9/pkglist @@ -18,8 +18,8 @@ xfsprogs e2fsprogs fuse-libs libnl3 -chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr attr +chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr attr squashfs-tools %onlyarch x86_64 shim-x64.x86_64 grub2-efi-x64 %onlyarch aarch64 -shim-aa64.aarch64 grub2-efi-aa64 \ No newline at end of file +shim-aa64.aarch64 grub2-efi-aa64 diff --git a/imgutil/el9/pkglist.aarch64 b/imgutil/el9/pkglist.aarch64 index 2141634a..63ac2c51 100644 --- a/imgutil/el9/pkglist.aarch64 +++ b/imgutil/el9/pkglist.aarch64 @@ -17,4 +17,4 @@ xfsprogs e2fsprogs fuse-libs libnl3 -chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr shim-aa64 grub2-efi-aa64 attr +chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr shim-aa64 grub2-efi-aa64 attr squashfs-tools diff --git a/imgutil/imgutil b/imgutil/imgutil index 5f0ccfb2..58f022ab 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -234,6 +234,8 @@ def capture_remote(args): for cmd in unmet: sys.stderr.write(cmd + '\n') sys.exit(1) + for cmd in finfo.get('optionalprereqs', []): + sys.stderr.write(cmd + '\n') oscat = finfo['oscategory'] subprocess.check_call(['ssh', '-o', 'LogLevel=QUIET', '-t', targ, 'python3', '/run/imgutil/capenv/imgutil', 'capturelocal']) utillib = __file__.replace('bin/imgutil', 'lib/imgutil') @@ -488,6 +490,7 @@ class OsHandler(object): self.sourcepath = None self.osname = '{}-{}-{}'.format(name, version, arch) self.captureprereqs = [] + self.captureadvisories = [] try: pkglist = args.packagelist except AttributeError: @@ -524,7 +527,8 @@ class OsHandler(object): if not isinstance(odata[idx], str): odata[idx] = odata[idx].decode('utf8') info = {'oscategory': odata[0], - 'version': odata[1], 'arch': odata[2], 'name': odata[3], 'unmetprereqs': self.captureprereqs} + 'version': odata[1], 'arch': odata[2], 'name': odata[3], 'unmetprereqs': self.captureprereqs, + 'optionalprereqs': self.captureadvisories} return json.dumps(info) def prep_root_premount(self, args): @@ -698,14 +702,21 @@ class DebHandler(OsHandler): self.oscategory = name + version super().__init__(name, version, arch, args) needpkgs = [] + wantpkgs = [] if not os.path.exists(os.path.join(hostpath, 'usr/bin/tpm2_getcap')): needpkgs.append('tpm2-tools') lfuses = glob.glob(os.path.join(hostpath, '/lib/*/libfuse.so.2')) if not lfuses: needpkgs.append('libfuse2') + if not os.path.exists(os.path.join(hostpath, 'usr/bin/unsquashfs')): + wantpkgs.append('squashfs-tools') if needpkgs: needapt = 'Missing packages needed in target for capture, to add required packages: apt install ' + ' '.join(needpkgs) self.captureprereqs.append(needapt) + if wantpkgs: + wantapt = ('Missing optional packages in target, untethered boot will extract the image more slowly, ' + 'to add them: apt install ' + ' '.join(wantpkgs)) + self.captureadvisories.append(wantapt) def add_pkglists(self): self.includepkgs.extend(self.list_packages()) @@ -783,6 +794,7 @@ class ElHandler(OsHandler): self.yumargs = [] super().__init__(name, version, arch, args) needpkgs = [] + wantpkgs = [] if not hostpath: return if not os.path.exists(os.path.join(hostpath, 'usr/bin/tpm2_getcap')): @@ -801,9 +813,17 @@ class ElHandler(OsHandler): needpkgs.append('dhcp-client') if not os.path.exists(os.path.join(hostpath, 'usr/sbin/mount.nfs')): needpkgs.append('nfs-utils') + if (self.oscategory not in ('el7', 'el8') and + not os.path.exists(os.path.join(hostpath, 'usr/sbin/unsquashfs')) and + not os.path.exists(os.path.join(hostpath, 'usr/bin/unsquashfs'))): + wantpkgs.append('squashfs-tools') if needpkgs: needapt = 'Missing packages needed in target for capture, to add required packages: dnf install ' + ' '.join(needpkgs) self.captureprereqs.append(needapt) + if wantpkgs: + wantdnf = ('Missing optional packages in target, untethered boot will extract the image more slowly, ' + 'to add them: dnf install ' + ' '.join(wantpkgs)) + self.captureadvisories.append(wantdnf) def add_pkglists(self): self.yumargs.extend(self.list_packages()) diff --git a/imgutil/ubuntu/initramfs-tools/hooks/confluent b/imgutil/ubuntu/initramfs-tools/hooks/confluent index 611a1651..e352173b 100644 --- a/imgutil/ubuntu/initramfs-tools/hooks/confluent +++ b/imgutil/ubuntu/initramfs-tools/hooks/confluent @@ -33,6 +33,7 @@ copy_exec /usr/bin/tpm2_pcrextend copy_exec /usr/bin/ssh-keygen copy_exec /usr/sbin/sshd copy_exec /usr/sbin/mkfs.xfs +[ -e /usr/bin/unsquashfs ] && copy_exec /usr/bin/unsquashfs [ -e /usr/lib/openssh/sshd-session ] && copy_exec /usr/lib/openssh/sshd-session [ -e /usr/lib/x86_64-linux-gnu/libfuse.so.2 ] && copy_exec /usr/lib/x86_64-linux-gnu/libfuse.so.2 [ -e /usr/lib/aarch64-linux-gnu/libfuse.so.2 ] && copy_exec /usr/lib/aarch64-linux-gnu/libfuse.so.2