diff --git a/.github/workflows/xcat_test.yml b/.github/workflows/xcat_test.yml index 7b9bd84c4..68fb5b890 100644 --- a/.github/workflows/xcat_test.yml +++ b/.github/workflows/xcat_test.yml @@ -9,7 +9,7 @@ jobs: - name: Refresh package indexes run: sudo apt-get update - name: Install dependencies - run: sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --no-install-suggests bats bubblewrap gawk mawk rpm rsyslog build-essential fakeroot reprepro devscripts debhelper libcapture-tiny-perl libfile-slurper-perl libjson-perl libyaml-pp-perl libparallel-forkmanager-perl libsoap-lite-perl libdbi-perl libcgi-pm-perl quilt openssh-server dpkg looptools genometools software-properties-common grub-common grub-efi-amd64-bin + run: sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --no-install-suggests bats bubblewrap gawk mawk rpm rsyslog build-essential fakeroot reprepro devscripts debhelper libcapture-tiny-perl libfile-slurper-perl libjson-perl libyaml-pp-perl libparallel-forkmanager-perl libsoap-lite-perl libdbi-perl libcgi-pm-perl quilt openssh-server dpkg looptools genometools software-properties-common grub-common grub-efi-amd64-bin squashfs-tools pigz - name: Enable postscript test namespaces run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Install lifecycle test dependencies diff --git a/docs/source/guides/admin-guides/references/man1/packimage.1.rst b/docs/source/guides/admin-guides/references/man1/packimage.1.rst index 4c8568b15..29b625cb3 100644 --- a/docs/source/guides/admin-guides/references/man1/packimage.1.rst +++ b/docs/source/guides/admin-guides/references/man1/packimage.1.rst @@ -33,7 +33,7 @@ DESCRIPTION Packs the stateless image from the chroot file system into a file to be sent to the node for a diskless boot. -Note: For an osimage that is deployed on a cluster, running \ **packimage**\ will overwrite the existing rootimage file and be unavailable to the compute nodes while \ **packimage**\ is running. +Note: \ **packimage**\ writes the new file next to the existing one and replaces it only when the new file is complete. Compute nodes can download the existing file while \ **packimage**\ runs, and the existing file stays if \ **packimage**\ fails. The image directory needs space for both files while \ **packimage**\ runs. ********** diff --git a/xCAT-client/pods/man1/packimage.1.pod b/xCAT-client/pods/man1/packimage.1.pod index 32e91d9f7..45dd1da69 100644 --- a/xCAT-client/pods/man1/packimage.1.pod +++ b/xCAT-client/pods/man1/packimage.1.pod @@ -14,7 +14,7 @@ B [B<-m>|B<--method> B] [B<-c>|B<--compress> B will overwrite the existing rootimage file and be unavailable to the compute nodes while B is running. +Note: B writes the new file next to the existing one and replaces it only when the new file is complete. Compute nodes can download the existing file while B runs, and the existing file stays if B fails. The image directory needs space for both files while B runs. =head1 PARAMETERS diff --git a/xCAT-server/lib/xcat/plugins/packimage.pm b/xCAT-server/lib/xcat/plugins/packimage.pm index c3281e158..506482909 100644 --- a/xCAT-server/lib/xcat/plugins/packimage.pm +++ b/xCAT-server/lib/xcat/plugins/packimage.pm @@ -29,7 +29,9 @@ use Getopt::Long; use File::Path; use File::Copy; use Cwd; +use Errno qw(EEXIST); use File::Temp; +use Sys::Hostname (); use File::Basename; use File::Path; @@ -48,6 +50,9 @@ my $verbose = 0; #$verbose = 1; +# Publication takes milliseconds, so packimage waits no longer than this for the publication lock. +my $publish_lock_wait = 60; + #------------------------------------------------------- =head3 handled_commands @@ -268,8 +273,52 @@ sub process_request { $verbose && $callback->({ data => [ "rootimg_status = $rootimg_status at line " . __LINE__ ] }); + # Each StateLite change is recorded, so that every exit undoes exactly what was done. + my ($statelite_converted, @statelite_moved, @statelite_created, %statelite_saved); + # Every return after the conversion below must call this. + my $restore_statelite = sub { + return 0 unless $statelite_converted; + $statelite_converted = 0; + my @failed; + foreach my $filename (@statelite_moved) { + xCAT::Utils->runcmd("rm -rf $rootimg_dir$filename", 0, 1); + xCAT::Utils->runcmd("mv $rootimg_dir/.statebackup$filename $rootimg_dir$filename", 0, 1); + push @failed, $filename if $::RUNCMD_RC; + } + foreach my $filename (@statelite_created) { + xCAT::Utils->runcmd("rm -rf $rootimg_dir$filename", 0, 1); + } + foreach my $saved (sort keys %statelite_saved) { + xCAT::Utils->runcmd("mv $rootimg_dir/.statebackup/$saved $rootimg_dir$statelite_saved{$saved}", 0, 1); + push @failed, $statelite_saved{$saved} if $::RUNCMD_RC; + } + if (@failed) { + $callback->({ error => ["Cannot restore the StateLite files " . join(', ', @failed) + . " in $rootimg_dir. The originals stay in $rootimg_dir/.statebackup."], errorcode => [1] }); + return 1; + } + xCAT::Utils->runcmd("rm -rf $rootimg_dir/.statebackup", 0, 1); + return 0; + }; + my $statelite_failure = sub { + my $message = shift; + $callback->({ error => [$message], errorcode => [1] }); + $restore_statelite->(); + return 1; + }; + if ($rootimg_status) { - xCAT::Utils->runcmd("mkdir $rootimg_dir/.statebackup", 0, 1); + # A .statebackup that an earlier pack could not restore holds the only copy of the original files. + if (-e "$rootimg_dir/.statebackup" || -l "$rootimg_dir/.statebackup") { + $callback->({ error => ["$rootimg_dir/.statebackup holds StateLite files that an earlier packimage did not restore. " + . "Restore them and remove the directory, then run packimage again."], errorcode => [1] }); + return 1; + } + unless (mkdir("$rootimg_dir/.statebackup")) { + $callback->({ error => ["Cannot create $rootimg_dir/.statebackup: $!"], errorcode => [1] }); + return 1; + } + $statelite_converted = 1; # read through the litefile table to decide which file/directory should be restore my $defaultloc = "$rootimg_dir/.default"; @@ -292,10 +341,17 @@ sub process_request { unlink "$rootimg_dir/.statebackup$parent"; $verbose && $callback->({ data => ["mkdir -p $rootimg_dir/.statebackup$parent"] }); xCAT::Utils->runcmd("mkdir -p $rootimg_dir/.statebackup$parent", 0, 1); + return $statelite_failure->("Cannot create $rootimg_dir/.statebackup$parent") if $::RUNCMD_RC; } $verbose && $callback->({ data => [ "backing up the file $filename.. at line " . __LINE__ ] }); $verbose && print "++ $defaultloc$filename ++ $rootimg_dir$filename ++ at " . __LINE__ . "\n"; - xCAT::Utils->runcmd("mv $rootimg_dir$filename $rootimg_dir/.statebackup$filename", 0, 1); + if (-e "$rootimg_dir$filename" || -l "$rootimg_dir$filename") { + xCAT::Utils->runcmd("mv $rootimg_dir$filename $rootimg_dir/.statebackup$filename", 0, 1); + return $statelite_failure->("Cannot move $rootimg_dir$filename to $rootimg_dir/.statebackup") if $::RUNCMD_RC; + push @statelite_moved, $filename; + } else { + push @statelite_created, $filename; + } xCAT::Utils->runcmd("cp -r -a $defaultloc$filename $rootimg_dir$filename", 0, 1); } } @@ -304,7 +360,13 @@ sub process_request { unless ($is_openeuler) { # TODO: following the old genimage code, to update the stateles-only files/directories # # another file should be /opt/xcat/xcatdsklspost, but it seems not necessary - xCAT::Utils->runcmd("mv $rootimg_dir/etc/init.d/statelite $rootimg_dir/.statebackup/statelite ", 0, 1) if (-e "$rootimg_dir/etc/init.d/statelite"); + if (-e "$rootimg_dir/etc/init.d/statelite") { + xCAT::Utils->runcmd("mv $rootimg_dir/etc/init.d/statelite $rootimg_dir/.statebackup/statelite ", 0, 1); + if ($statelite_converted) { + return $statelite_failure->("Cannot move $rootimg_dir/etc/init.d/statelite to $rootimg_dir/.statebackup") if $::RUNCMD_RC; + $statelite_saved{statelite} = '/etc/init.d/statelite'; + } + } if (-e "$rootimg_dir/usr/share/dracut") { # currently only used for redhat families, not available for SuSE families @@ -316,6 +378,10 @@ sub process_request { #restore the install.netboot of xcat dracut module if (-e "$rootimg_dir/usr/lib/dracut/modules.d/97xcat/install") { xCAT::Utils->runcmd("mv $rootimg_dir/usr/lib/dracut/modules.d/97xcat/install $rootimg_dir/.statebackup/install", 0, 1); + if ($statelite_converted) { + return $statelite_failure->("Cannot move $rootimg_dir/usr/lib/dracut/modules.d/97xcat/install to $rootimg_dir/.statebackup") if $::RUNCMD_RC; + $statelite_saved{install} = '/usr/lib/dracut/modules.d/97xcat/install'; + } } my $dracut_install = "$::XCATROOT/share/xcat/netboot/$distname/dracut_033/install.netboot"; if (!-r $dracut_install) { @@ -417,6 +483,7 @@ sub process_request { # add the xCAT post scripts to the image unless (-d "$rootimg_dir") { $callback->({ error => ["$rootimg_dir does not exist, run genimage -o $osver -p $profile on a server with matching architecture"], errorcode => [1] }); + $restore_statelite->(); return 1; } @@ -478,11 +545,12 @@ sub process_request { chdir($oldpath); umask($oldmask) if defined($oldmask); rmtree($temppath) if defined($temppath) && -d $temppath; + unlink($xcat_packimg_tmpfile); + $restore_statelite->(); return 1; }; unless (-d $rootimg_dir) { - $callback->({ error => ["$rootimg_dir does not exist, run genimage -o $osver -p $profile on a server with matching architecture"], errorcode => [1] }); - return 1; + return $native_failure->("$rootimg_dir does not exist, run genimage -o $osver -p $profile on a server with matching architecture"); } my $suffix; @@ -490,27 +558,23 @@ sub process_request { if ($compress eq 'gzip') { my $isgzip = system("bash -c 'type -p gzip' >/dev/null 2>&1"); unless ($isgzip == 0) { - $callback->({ error => ["Command gzip does not exist, please make sure it is installed."], errorcode => [1] }); - return 1; + return $native_failure->("Command gzip does not exist, please make sure it is installed."); } $suffix = "gz"; } elsif ($compress eq 'pigz') { my $ispigz = system("bash -c 'type -p pigz' >/dev/null 2>&1"); unless ($ispigz == 0) { - $callback->({ error => ["Command pigz does not exist, please make sure it is installed."], errorcode => [1] }); - return 1; + return $native_failure->("Command pigz does not exist, please make sure it is installed."); } $suffix = "gz"; } elsif ($compress eq 'xz') { my $isxz = system("bash -c 'type -p xz' >/dev/null 2>&1"); unless ($isxz == 0) { - $callback->({ error => ["Command xz does not exist, please make sure it is installed."], errorcode => [1] }); - return 1; + return $native_failure->("Command xz does not exist, please make sure it is installed."); } $suffix = "xz"; } else { - $callback->({ error => ["Invalid compress method '$compress' requested"], errorcode => [1] }); - return 1; + return $native_failure->("Invalid compress method '$compress' requested"); } } else { my $ispigz = system("bash -c 'type -p pigz' >/dev/null 2>&1"); @@ -521,16 +585,14 @@ sub process_request { if ($isgzip == 0) { $compress = "gzip"; } else { - $callback->({ error => ["The default compress tool 'gzip' and 'pigz' does not exist, please specify an available compress method with '-c'."], errorcode => [1] }); - return 1; + return $native_failure->("The default compress tool 'gzip' and 'pigz' does not exist, please specify an available compress method with '-c'."); } } $suffix = "gz"; } unless (($method eq 'cpio') or ($method eq 'tar') or ($method eq 'squashfs')) { - $callback->({ error => ["Invalid archive method '$method' requested"], errorcode => [1] }); - return 1; + return $native_failure->("Invalid archive method '$method' requested"); } $callback->({ data => ["Packing contents of $rootimg_dir"] }); $callback->({ info => ["archive method:$method"] }); @@ -539,29 +601,62 @@ sub process_request { } $suffix = $method.".".$suffix; - unlink glob("$destdir/rootimg.*") unless $is_openeuler; - my $native_archive = $is_openeuler - ? File::Temp->new(DIR => $destdir, TEMPLATE => '.packimage-XXXXXXXX', UNLINK => 1) : undef; - my $archive_output = "../rootimg.$suffix"; - if ($is_openeuler) { - $archive_output = "$native_archive"; - $archive_output =~ s/'/'\\''/g; - $archive_output = "'$archive_output'"; - chdir($rootimg_dir) or return $native_failure->("Cannot enter $rootimg_dir: $!"); - my ($rc, $output) = native_pack_command("$excludestr > $xcat_packimg_tmpfile"); - return $native_failure->("Cannot enumerate $rootimg_dir: $output") if $rc; - if ($includestr) { - ($rc, $output) = native_pack_command("$includestr >> $xcat_packimg_tmpfile"); - return $native_failure->("Cannot enumerate included files: $output") if $rc; - } + my $image_file = $method eq 'squashfs' ? 'rootimg.sfs' : "rootimg.$suffix"; + # The image lock is per host, so only this host's .packimage files are known to be abandoned. + (my $host = Sys::Hostname::hostname()) =~ s/[^A-Za-z0-9.-]/_/g; + my @abandoned = glob("$destdir/.packimage-$host+????????"); + rmtree(\@abandoned) if @abandoned; + # ctorrent names the metainfo after the archive, so both are built under their final names in a private directory. + my $stage = eval { File::Temp->newdir(".packimage-$host+XXXXXXXX", DIR => $destdir) } + or return $native_failure->("Cannot create a temporary directory in $destdir: $@"); + my $archive_output = "$stage/$image_file"; + $archive_output =~ s/'/'\\''/g; + $archive_output = "'$archive_output'"; + chdir($rootimg_dir) or return $native_failure->("Cannot enter $rootimg_dir: $!"); + my ($list_rc, $list_output) = native_pack_command("$excludestr > $xcat_packimg_tmpfile"); + return $native_failure->("Cannot enumerate $rootimg_dir: $list_output") if $list_rc; + if ($includestr) { + ($list_rc, $list_output) = native_pack_command("$includestr >> $xcat_packimg_tmpfile"); + return $native_failure->("Cannot enumerate included files: $list_output") if $list_rc; } - my $publish_native_archive = sub { - my $target = $method eq 'squashfs' ? "$destdir/rootimg.sfs" : "$destdir/rootimg.$suffix"; - unless (-s "$native_archive" && chmod(0644, "$native_archive") - && rename("$native_archive", $target)) { - return $native_failure->("Cannot publish $target: $!"); + # Nodes keep downloading the previous archive until the rename replaces it. + my $publish_archive = sub { + # The image lock is per host, and mkdir is atomic for every host, also on NFS without locks. + my $publish_lock = "$destdir/.packimage-publish"; + for (my $waited = 0; !mkdir($publish_lock); $waited++) { + return $native_failure->("Cannot create $publish_lock: $!") unless $! == EEXIST; + my $age = time() - ((stat($publish_lock))[9] // time()); + if ($waited >= $publish_lock_wait || $age > $publish_lock_wait) { + my $holder = ''; + if (open(my $owner, '<', "$publish_lock/owner")) { + $holder = <$owner> // ''; + close($owner); + chomp($holder); + $holder = " by $holder" if $holder; + } + return $native_failure->("$publish_lock is held$holder. If no packimage of this image runs on any host, " + . "remove it and run packimage again."); + } + sleep 1; } - unlink grep { $_ ne $target } glob("$destdir/rootimg.*"); + if (open(my $owner, '>', "$publish_lock/owner")) { + print $owner "$host $$\n"; + close($owner); + } + my $target = "$destdir/$image_file"; + my ($new_archive, $new_metainfo) = ("$stage/$image_file", "$stage/$image_file.metainfo"); + my $metainfo = -e $new_metainfo; + + # The previous metainfo goes first, so no failure leaves a metainfo that describes another archive. + my $published = -s $new_archive && chmod(0644, $new_archive) && (!$metainfo || chmod(0644, $new_metainfo)) + && (!-e "$target.metainfo" || unlink("$target.metainfo")) + && rename($new_archive, $target) + && (!$metainfo || rename($new_metainfo, "$target.metainfo")); + my $error = $!; + unlink grep { $_ ne $target && $_ ne "$target.metainfo" } glob("$destdir/rootimg.*") if $published; + unlink("$publish_lock/owner"); + rmdir($publish_lock); + return $native_failure->("Cannot publish $target: $error") unless $published; return 0; }; @@ -569,11 +664,6 @@ sub process_request { if (!$excludestr) { $excludestr = "find . -xdev -print0 | cpio -H newc -o -0 | $compress -c - > $archive_output"; } else { - chdir("$rootimg_dir"); - system("$excludestr >> $xcat_packimg_tmpfile") unless $is_openeuler; - if ($includestr && !$is_openeuler) { - system("$includestr >> $xcat_packimg_tmpfile"); - } $excludestr = "cat $xcat_packimg_tmpfile|cpio -H newc -o | $compress -c - > $archive_output"; } $oldmask = umask 0077; @@ -590,58 +680,24 @@ sub process_request { if (!$excludestr) { $excludestr = "find . -xdev -print0 | tar $option --no-recursion --use-compress-program=$compress --null -T - -cf $archive_output"; } else { - chdir("$rootimg_dir"); - system("$excludestr >> $xcat_packimg_tmpfile") unless $is_openeuler; - if ($includestr && !$is_openeuler) { - system("$includestr >> $xcat_packimg_tmpfile"); - } $excludestr = "cat $xcat_packimg_tmpfile| tar $option --no-recursion --use-compress-program=$compress -T - -cf $archive_output"; } $oldmask = umask 0077; } elsif ($method =~ /squashfs/) { - $temppath = mkdtemp("/tmp/packimage.$$.XXXXXXXX"); + $temppath = eval { mkdtemp("/tmp/packimage.$$.XXXXXXXX") } + or return $native_failure->("Cannot create a staging directory: $@"); chmod 0755, $temppath; - chdir("$rootimg_dir"); - system("$excludestr >> $xcat_packimg_tmpfile") unless $is_openeuler; - if ($includestr && !$is_openeuler) { - system("$includestr >> $xcat_packimg_tmpfile"); - } $excludestr = "cat $xcat_packimg_tmpfile|cpio -dump $temppath"; } chdir("$rootimg_dir"); - my ($archive_rc, $outputmsg); - if ($is_openeuler) { - ($archive_rc, $outputmsg) = native_pack_command($excludestr); - } else { - $outputmsg = `$excludestr 2>&1`; - $archive_rc = $?; - } + my ($archive_rc, $outputmsg) = native_pack_command($excludestr); unless($archive_rc){ $callback->({ info => ["$outputmsg"] }); }else{ $callback->({ info => ["$outputmsg"] }); - return $native_failure->("packimage failed while running: \n $excludestr") if $is_openeuler; - $callback->({ error => ["packimage failed while running: \n $excludestr"], errorcode => [1] }); - system("rm -rf $xcat_packimg_tmpfile"); - return 1; + return $native_failure->("packimage failed while running: \n $excludestr"); } - return 1 if $is_openeuler && $method ne 'squashfs' && $publish_native_archive->(); - - if ($method =~ /cpio/) { - chmod 0644, "$destdir/rootimg.$suffix"; - if ($dotorrent) { - my $currdir = getcwd; - chdir($destdir); - unlink("rootimg.$suffix.metainfo"); - system("ctorrent -t -u $dotorrent -l 1048576 -s rootimg.$suffix.metainfo rootimg.$suffix"); - chmod 0644, "rootimg.$suffix.metainfo"; - chdir($currdir); - } - umask $oldmask; - } elsif ($method =~ /tar/) { - chmod 0644, "$destdir/rootimg.$suffix"; - umask $oldmask; - } elsif ($method =~ /squashfs/) { + if ($method =~ /squashfs/) { my $flags = ""; if ($osver =~ /rhels5/) { if ($arch =~ /x86/) { @@ -654,51 +710,35 @@ sub process_request { if (!-x "/sbin/mksquashfs" && !-x "/usr/bin/mksquashfs") { return $native_failure->("mksquashfs not found; install squashfs-tools") if $is_openeuler; if ($osver =~ /sle/) { - $callback->({ error => ["mksquashfs not found, squashfs rpm should be installed on the management node"], errorcode => [1] }); - } else { - $callback->({ error => ["mksquashfs not found, squashfs-tools rpm should be installed on the management node"], errorcode => [1] }); + return $native_failure->("mksquashfs not found, squashfs rpm should be installed on the management node"); } - return 1; + return $native_failure->("mksquashfs not found, squashfs-tools rpm should be installed on the management node"); } - my $squashfs_output = $is_openeuler ? $archive_output : '../rootimg.sfs'; - $flags .= ' -noappend' if $is_openeuler; - my $mksquashfs_command = "mksquashfs $temppath $squashfs_output $flags"; + my $mksquashfs_command = "mksquashfs $temppath $archive_output $flags"; xCAT::Utils->runcmd($mksquashfs_command, 0, 1); my $rc = $::RUNCMD_RC; if ($rc) { - return $native_failure->("Command \"$mksquashfs_command\" failed") if $is_openeuler; - $callback->({ error => ["Command \"$mksquashfs_command\" failed"], errorcode => [1] }); - return 1; + return $native_failure->("Command \"$mksquashfs_command\" failed"); } $rc = system("rm -rf $temppath"); if ($rc) { - return $native_failure->("Failed to clean up temp space") if $is_openeuler; - $callback->({ error => ["Failed to clean up temp space"], errorcode => [1] }); - return 1; + return $native_failure->("Failed to clean up temp space"); } - return 1 if $is_openeuler && $publish_native_archive->(); - chmod(0644, "../rootimg.sfs"); } + + if ($dotorrent && $method =~ /cpio/) { + my $made = chdir("$stage") && !system("ctorrent -t -u $dotorrent -l 1048576 -s $image_file.metainfo $image_file"); + return $native_failure->("ctorrent cannot create the metainfo of $destdir/$image_file") unless $made; + } + + # The StateLite files must be back in place before the new archive replaces the previous one. + return $native_failure->("$destdir keeps its previous archive because the StateLite files of $rootimg_dir were not restored") + if $restore_statelite->(); + return 1 if $publish_archive->(); + + umask($oldmask) if defined($oldmask); system("rm -f $xcat_packimg_tmpfile"); - # move the files in /.statebackup back to rootimg_dir - if ($rootimg_status) { # statelite mode - foreach my $entry (keys %liteHash) { - my @tmp = split /\s+/, $entry; - my $filename = $tmp[1]; - my $fileopt = $tmp[0]; - if ($fileopt =~ m/link/) { - chop $filename if ($filename =~ m/\/$/); - xCAT::Utils->runcmd("rm -rf $rootimg_dir$filename", 0, 1); - xCAT::Utils->runcmd("mv $rootimg_dir/.statebackup$filename $rootimg_dir$filename", 0, 1); - } - } - - xCAT::Utils->runcmd("mv $rootimg_dir/.statebackup/install $rootimg_dir/usr/lib/dracut/modules.d/97xcat/install", 0, 1); - xCAT::Utils->runcmd("mv $rootimg_dir/.statebackup/statelite $rootimg_dir/etc/init.d/statelite", 0, 1); - xCAT::Utils->runcmd("rm -rf $rootimg_dir/.statebackup", 0, 1); - } - my $restored = chdir($oldpath); return $is_openeuler ? 0 : $restored; diff --git a/xCAT-test/unit/packimage_atomic_archive.t b/xCAT-test/unit/packimage_atomic_archive.t new file mode 100644 index 000000000..d31dabec5 --- /dev/null +++ b/xCAT-test/unit/packimage_atomic_archive.t @@ -0,0 +1,367 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use FindBin; +use lib "$FindBin::Bin/../lib"; +use lib "$FindBin::Bin/../../perl-xCAT"; +use lib "$FindBin::Bin/../../xCAT-server/lib/perl"; +use Cwd qw(getcwd realpath); +use File::Path qw(make_path); +use File::Slurper qw(read_binary write_text); +use File::Temp qw(tempdir); +use POSIX (); +use Sys::Hostname qw(hostname); +use Test::More; +use XCAT::Test::File qw(repo_path); + +plan skip_all => 'packimage packs real archives with Linux tools' unless $^O eq 'linux'; +local $ENV{PATH} = "$ENV{PATH}:/usr/sbin:/sbin"; + +sub tool { + my ($name) = @_; + my ($path) = grep { -x $_ } map { "$_/$name" } split /:/, $ENV{PATH}; + return $path; +} +for my $name (qw(cpio gzip tar find)) { + plan skip_all => "$name is required" unless tool($name); +} + +my $fixture = realpath(tempdir(CLEANUP => 1)); +make_path("$fixture/db", "$fixture/root/lib/perl", "$fixture/root/share", "$fixture/install/postscripts"); +symlink(repo_path('perl-xCAT/xCAT'), "$fixture/root/lib/perl/xCAT") or die $!; +symlink(repo_path('xCAT-server/share/xcat'), "$fixture/root/share/xcat") or die $!; +write_text("$fixture/install/postscripts/xcatdsklspost", "#!/bin/sh\nexit 0\n"); +$ENV{XCATROOT} = "$fixture/root"; +$ENV{XCATCFG} = "SQLite:$fixture/db"; +require xCAT::Table; +require(repo_path('xCAT-server/lib/xcat/plugins/packimage.pm')); +xCAT::Table->new('site', -create => 1)->setAttribs({ key => 'installdir' }, { value => "$fixture/install" }); +xCAT::Table->new('passwd', -create => 1) + ->setAttribs({ key => 'system', username => 'root' }, { password => '*', cryptmethod => 'sha512' }); +my $osimage = xCAT::Table->new('osimage', -create => 1); +my $linuximage = xCAT::Table->new('linuximage', -create => 1); +{ + no warnings qw(redefine once); + *xCAT::Utils::acquire_lock_imageop = sub { return (0, undef); }; + *xCAT::Utils::runcmd = sub { + my ($class, $command) = @_; + if ($command =~ /^ilitefile /) { + $::RUNCMD_RC = 0; + return ([@main::LITEFILES]); + } + my @output = `$command 2>&1`; + $::RUNCMD_RC = $? >> 8; + return (\@output); + }; +} + +our @LITEFILES; +(my $host = hostname()) =~ s/[^A-Za-z0-9.-]/_/g; +my ($unsquashfs, $mksquashfs) = (tool('unsquashfs'), tool('mksquashfs')); +my $sequence = 0; + +# The wrapper records what nodes would download while the pack runs, then runs or fails the real tool. +sub image { + my ($method, $compress, $wrapped, $mode) = @_; + my $name = 'atomic-' . ++$sequence; + my $dest = "$fixture/images/$name"; + my $root = "$dest/rootimg"; + make_path("$root/etc", "$root/opt/xcat", "$root/usr/bin", "$root/usr/lib/dracut/modules.d/97xcat", "$dest/bin"); + write_text("$root/etc/shadow", "root:*:1:0:99999:7:::\n"); + write_text("$root/usr/bin/payload", "payload of $name\n"); + my $suffix = $method eq 'squashfs' ? 'sfs' : "$method." . ($compress eq 'xz' ? 'xz' : 'gz'); + my $archive = "$dest/rootimg.$suffix"; + write_text($archive, "previous archive\n"); + write_text("$archive.metainfo", "previous metainfo\n"); + write_text("$dest/rootimg.previous", "previous format\n"); + write_text("$dest/.packimage-$host+INTERRUP", "partial archive of an interrupted pack on this host\n"); + make_path("$dest/.packimage-$host+INTERDIR"); + write_text("$dest/.packimage-$host+INTERDIR/rootimg.$suffix", "partial archive of an interrupted pack on this host\n"); + write_text("$dest/.packimage-other.host+ACTIVE00", "archive that another host is writing\n"); + write_text("$dest/.packimage-$host-backup+ACTIVE00", "archive that a host with a longer name is writing\n"); + my $real = tool($wrapped); + my $fail = "printf 'injected $wrapped failure\\n' >&2; exit 29"; + my $run = $mode eq 'fail' ? $fail + : $mode eq 'fail-listing' ? "if [ \"\$1\" = . ] && [ \"\$2\" = -xdev ]; then $fail; fi\nexec '$real' \"\$\@\"" + : $mode eq 'fail-restore' ? "case \"\$1\" in */.statebackup/*) $fail;; esac\nexec '$real' \"\$\@\"" + : "exec '$real' \"\$\@\""; + write_text("$dest/bin/$wrapped", <<"SCRIPT"); +#!/bin/sh +if [ -f '$archive' ]; then cat '$archive' > '$dest/seen'; else echo missing > '$dest/seen'; fi +$run +SCRIPT + chmod 0755, "$dest/bin/$wrapped"; + $osimage->setAttribs({ imagename => $name }, + { osvers => 'rhels9.6', osarch => 'x86_64', profile => 'compute', provmethod => 'netboot' }); + $linuximage->setAttribs({ imagename => $name }, { rootimgdir => $dest }); + return ($name, $dest, $archive); +} + +sub packimage { + my ($dest, @args) = @_; + my ($cwd, $mask) = (getcwd(), umask()); + my @responses; + my $output = ''; + open(my $stdout, '>', \$output) or die $!; + my $result; + { + local $ENV{PATH} = "$dest/bin:$ENV{PATH}"; + local *STDOUT = $stdout; + $result = eval { + xCAT_plugin::packimage::process_request({ arg => [ '--nosyncfiles', @args ] }, sub { push @responses, @_ }); + }; + push @responses, { error => ["packimage died: $@"] } if $@; + } + chdir($cwd) or die $!; + umask($mask); + return ($result, join("\n", map { @{ $_->{error} || [] } } @responses)); +} + +sub content { + my ($path) = @_; + return -f $path ? read_binary($path) : undef; +} + +sub listing { + my ($method, $compress, $archive) = @_; + my @command = $method eq 'squashfs' ? ($unsquashfs, '-l', $archive) + : $method eq 'tar' ? (tool('tar'), '-tf', $archive) + : ('/bin/sh', '-c', '"$1" -dc "$2" | cpio -t 2>/dev/null', 'list', tool($compress), $archive); + open(my $pipe, '-|', @command) or die "list $archive: $!"; + my $text = do { local $/; <$pipe> }; + close($pipe); + return $? ? undef : $text; +} + +for my $case ([ 'cpio', 'gzip' ], [ 'cpio', 'xz' ], [ 'cpio', 'pigz' ], [ 'tar', 'gzip' ], [ 'tar', 'xz' ], + [ 'squashfs', 'gzip' ]) { + my ($method, $compress) = @$case; + my $wrapped = $method eq 'squashfs' ? 'mksquashfs' : $compress; + SKIP: { + skip "$wrapped is not installed", 18 unless tool($wrapped); + skip 'unsquashfs is not installed', 18 if $method eq 'squashfs' && !$unsquashfs; + my @args = $method eq 'squashfs' ? ('-m', 'squashfs') : ('-m', $method, '-c', $compress); + my $label = $method eq 'squashfs' ? 'squashfs' : "$method $compress"; + + my ($name, $dest, $archive) = image($method, $compress, $wrapped, 'pass'); + my ($result, $errors) = packimage($dest, @args, $name); + is($errors, '', "$label: the pack reports no error"); + is(content("$dest/seen"), "previous archive\n", "$label: nodes get the previous archive while the pack runs"); + isnt(content($archive), "previous archive\n", "$label: the new archive replaces the previous one"); + like(listing($method, $compress, $archive) // '', qr{(?:^|/)usr/bin/payload$}m, + "$label: the new archive holds the image"); + is((stat($archive))[2] & 07777, 0644, "$label: the new archive is world readable"); + ok(!-e "$archive.metainfo", "$label: the metainfo of the previous archive is removed"); + ok(!-e "$dest/rootimg.previous", "$label: other formats are removed after the replacement"); + is_deeply([ glob("$dest/.packimage-$host+*") ], [], "$label: no partial archive is left, including an interrupted one"); + ok(-e "$dest/.packimage-other.host+ACTIVE00", "$label: the archive that another host is writing stays"); + ok(-e "$dest/.packimage-$host-backup+ACTIVE00", "$label: the archive of a host whose name starts with this one stays"); + + ($name, $dest, $archive) = image($method, $compress, $wrapped, 'fail'); + ($result, $errors) = packimage($dest, @args, $name); + like($errors, $method eq 'squashfs' ? qr/Command "mksquashfs .*" failed/s : qr/packimage failed while running/, + "$label: the failure is reported"); + is(content("$dest/seen"), "previous archive\n", "$label: nodes get the previous archive while the failing pack runs"); + is(content($archive), "previous archive\n", "$label: a failed pack keeps the previous archive"); + is(content("$archive.metainfo"), "previous metainfo\n", "$label: a failed pack keeps the previous metainfo"); + is(content("$dest/rootimg.previous"), "previous format\n", "$label: a failed pack keeps other formats"); + is_deeply([ glob("$dest/.packimage-$host+*") ], [], "$label: a failed pack leaves no partial archive"); + my ($row) = $linuximage->getAttribs({ imagename => $name }, 'rootimgdir'); + is($row->{rootimgdir}, $dest, "$label: the image definition is unchanged"); + ok(-d "$dest/rootimg/usr/bin", "$label: the root image directory is unchanged"); + } +} + +# A failure before the compressor, while listing or reading the files, must also keep the previous archive. +for my $case ([ 'cpio', 'cpio', 'fail', qr/packimage failed while running/ ], + [ 'cpio', 'find', 'fail-listing', qr/Cannot enumerate/ ], [ 'tar', 'find', 'fail-listing', qr/Cannot enumerate/ ], + [ 'squashfs', 'find', 'fail-listing', qr/Cannot enumerate/ ]) { + my ($method, $wrapped, $mode, $error) = @$case; + SKIP: { + skip 'mksquashfs or unsquashfs is not installed', 5 if $method eq 'squashfs' && (!$mksquashfs || !$unsquashfs); + my @args = $method eq 'squashfs' ? ('-m', 'squashfs') : ('-m', $method, '-c', 'gzip'); + my $label = "$method, $wrapped failure"; + my ($name, $dest, $archive) = image($method, 'gzip', $wrapped, $mode); + my (undef, $errors) = packimage($dest, @args, $name); + like($errors, $error, "$label: the failure is reported"); + is(content($archive), "previous archive\n", "$label: the previous archive stays"); + is(content("$archive.metainfo"), "previous metainfo\n", "$label: the previous metainfo stays"); + is(content("$dest/rootimg.previous"), "previous format\n", "$label: other formats stay"); + is_deeply([ glob("$dest/.packimage-$host+*") ], [], "$label: no partial archive is left"); + } +} + +# packimage converts a StateLite tree to stateless files while it packs, and must convert it back on every exit. +my @statelite_files = qw(etc/hosts.lite etc/hosts.link etc/init.d/statelite usr/lib/dracut/modules.d/97xcat/install); +sub statelite_state { + my ($root) = @_; + my %state = map { + my $path = "$root/$_"; + ($_ => -l $path ? '-> ' . readlink($path) : -f $path ? content($path) : 'missing') + } @statelite_files; + $state{'.statebackup'} = -e "$root/.statebackup" ? 'present' : 'absent'; + return \%state; +} +sub statelite_tree { + my ($name, $root) = @_; + make_path("$root/.statelite", "$root/.default/etc", "$root/etc/init.d"); + write_text("$root/.statelite/litefile.save", "litefile table\n"); + write_text("$root/etc/hosts.lite", "original lite file\n"); + write_text("$root/.default/etc/hosts.lite", "default lite file\n"); + symlink('hosts.lite', "$root/etc/hosts.link") or die $!; + write_text("$root/.default/etc/hosts.link", "default for the link\n"); + write_text("$root/etc/init.d/statelite", "StateLite init script\n"); + write_text("$root/usr/lib/dracut/modules.d/97xcat/install", "StateLite dracut install\n"); + return ("$name link /etc/hosts.lite", "$name link /etc/hosts.link"); +} +for my $case ([ 'find', 'fail-listing', qr/Cannot enumerate/ ], [ 'gzip', 'fail', qr/packimage failed while running/ ]) { + my ($wrapped, $mode, $error) = @$case; + my ($name, $dest, $archive) = image('cpio', 'gzip', $wrapped, $mode); + my $root = "$dest/rootimg"; + local @LITEFILES = statelite_tree($name, $root); + my $before = statelite_state($root); + my $label = "StateLite, $wrapped failure"; + my (undef, $errors) = packimage($dest, '-m', 'cpio', '-c', 'gzip', $name); + like($errors, $error, "$label: the failure is reported"); + is_deeply(statelite_state($root), $before, "$label: the files moved for packing are restored"); + unlink("$dest/bin/$wrapped") or die $!; + (undef, $errors) = packimage($dest, '-m', 'cpio', '-c', 'gzip', $name); + is($errors, '', "$label: a retry succeeds"); + is_deeply(statelite_state($root), $before, "$label: the retry keeps the original files"); + isnt(content($archive), "previous archive\n", "$label: the retry replaces the archive"); +} + +SKIP: { + skip 'root can create the temporary archive in a read-only directory', 5 if $> == 0; + my ($name, $dest, $archive) = image('cpio', 'gzip', 'gzip', 'pass'); + my $root = "$dest/rootimg"; + local @LITEFILES = statelite_tree($name, $root); + my $before = statelite_state($root); + my $label = 'StateLite, temporary archive failure'; + chmod 0555, $dest or die $!; + my (undef, $errors) = packimage($dest, '-m', 'cpio', '-c', 'gzip', $name); + chmod 0755, $dest or die $!; + like($errors, qr/Cannot create a temporary directory in \Q$dest\E/, "$label: the failure is reported"); + is_deeply(statelite_state($root), $before, "$label: the files moved for packing are restored"); + is(content($archive), "previous archive\n", "$label: the previous archive stays"); + (undef, $errors) = packimage($dest, '-m', 'cpio', '-c', 'gzip', $name); + is($errors, '', "$label: a retry succeeds"); + is_deeply(statelite_state($root), $before, "$label: the retry keeps the original files"); +} + +{ + my ($name, $dest, $archive) = image('cpio', 'gzip', 'mv', 'fail-restore'); + my $root = "$dest/rootimg"; + local @LITEFILES = statelite_tree($name, $root); + my $label = 'StateLite, failed restore'; + my (undef, $errors) = packimage($dest, '-m', 'cpio', '-c', 'gzip', $name); + like($errors, qr/Cannot restore the StateLite files .* The originals stay in \Q$root\E\/\.statebackup\./s, + "$label: the failure is reported"); + is(content("$root/.statebackup/etc/hosts.lite"), "original lite file\n", "$label: the original file stays in .statebackup"); + is(readlink("$root/.statebackup/etc/hosts.link"), 'hosts.lite', "$label: the original link stays in .statebackup"); + is(content("$root/.statebackup/install"), "StateLite dracut install\n", "$label: the original dracut install stays in .statebackup"); + is(content($archive), "previous archive\n", "$label: the previous archive stays"); + is(content("$archive.metainfo"), "previous metainfo\n", "$label: the previous metainfo stays"); + unlink("$dest/bin/mv") or die $!; + (undef, $errors) = packimage($dest, '-m', 'cpio', '-c', 'gzip', $name); + like($errors, qr/\Q$root\E\/\.statebackup holds StateLite files that an earlier packimage did not restore/, + "$label: the next pack refuses to run"); + is(content("$root/.statebackup/etc/hosts.lite"), "original lite file\n", "$label: the next pack keeps the original file"); + is(content("$root/.statebackup/install"), "StateLite dracut install\n", "$label: the next pack keeps the original dracut install"); +} + +{ + my ($name, $dest, $archive) = image('cpio', 'gzip', 'gzip', 'pass'); + my $label = 'another host publishing'; + mkdir("$dest/.packimage-publish") or die $!; + write_text("$dest/.packimage-publish/owner", "other.host 4242\n"); + my $pid = fork(); + die "fork: $!" unless defined $pid; + if ($pid == 0) { + my (undef, $errors) = packimage($dest, '-m', 'cpio', '-c', 'gzip', $name); + POSIX::_exit($errors eq '' ? 0 : 1); + } + my $deadline = time() + 30; + sleep 1 until (!-e "$dest/.packimage-$host+INTERRUP" && (() = glob("$dest/.packimage-$host+????????"))) + || time() > $deadline; + sleep 2; + is(waitpid($pid, POSIX::WNOHANG()), 0, "$label: the pack waits while another host publishes"); + is(content($archive), "previous archive\n", "$label: the previous archive stays while another host publishes"); + unlink("$dest/.packimage-publish/owner"); + rmdir("$dest/.packimage-publish") or die $!; + waitpid($pid, 0); + is($? >> 8, 0, "$label: the pack succeeds after the other host finishes"); + isnt(content($archive), "previous archive\n", "$label: the new archive is published after the other host finishes"); + ok(!-e "$dest/.packimage-publish", "$label: the pack releases the publication lock"); +} + +{ + my ($name, $dest, $archive) = image('cpio', 'gzip', 'gzip', 'pass'); + my $label = 'publication lock left by another pack'; + mkdir("$dest/.packimage-publish") or die $!; + write_text("$dest/.packimage-publish/owner", "other.host 4242\n"); + utime(time() - 120, time() - 120, "$dest/.packimage-publish") or die $!; + my ($result, $errors) = packimage($dest, '-m', 'cpio', '-c', 'gzip', $name); + like($errors, qr/\Q$dest\E\/\.packimage-publish is held by other\.host 4242\. If no packimage of this image runs on any host, remove it/, + "$label: the failure names the holder and the recovery"); + is(content($archive), "previous archive\n", "$label: the previous archive stays"); + is(content("$dest/rootimg.previous"), "previous format\n", "$label: other formats stay"); + ok(-d "$dest/.packimage-publish", "$label: the lock of the other pack is not taken over"); + is_deeply([ glob("$dest/.packimage-$host+*") ], [], "$label: no partial archive is left"); +} + +{ + my ($name, $dest, $archive) = image('cpio', 'gzip', 'gzip', 'pass'); + my $label = 'tracker'; + write_text("$dest/bin/ctorrent", <<"SCRIPT"); +#!/bin/sh +while [ \$# -gt 1 ]; do [ "\$1" = -s ] && out=\$2; shift; done +{ echo "\$1 \$(stat -c %s "\$1")"; cat '$archive'; } > "\$out" +SCRIPT + chmod 0755, "$dest/bin/ctorrent"; + my (undef, $errors) = packimage($dest, '-m', 'cpio', '-c', 'gzip', '--tracker', 'http://tracker.example:6969/announce', $name); + is($errors, '', "$label: the pack reports no error"); + is(content("$archive.metainfo"), 'rootimg.cpio.gz ' . (-s $archive) . "\nprevious archive\n", + "$label: the metainfo describes the new archive under its final name before the archive is replaced"); + is((stat("$archive.metainfo"))[2] & 07777, 0644, "$label: nodes can read the metainfo"); + is_deeply([ glob("$dest/.packimage-$host+*") ], [], "$label: no temporary file is left"); +} + +{ + my ($name, $dest, $archive) = image('cpio', 'gzip', 'gzip', 'pass'); + my $label = 'failed tracker'; + write_text("$dest/bin/ctorrent", <<"SCRIPT"); +#!/bin/sh +while [ \$# -gt 1 ]; do [ "\$1" = -s ] && out=\$2; shift; done +echo partial > "\$out" +exit 3 +SCRIPT + chmod 0755, "$dest/bin/ctorrent"; + my (undef, $errors) = packimage($dest, '-m', 'cpio', '-c', 'gzip', '--tracker', 'http://tracker.example:6969/announce', $name); + like($errors, qr/ctorrent cannot create the metainfo of \Q$archive\E/, "$label: the failure is reported"); + is(content($archive), "previous archive\n", "$label: the previous archive stays"); + is(content("$archive.metainfo"), "previous metainfo\n", "$label: the previous metainfo stays"); + is(content("$dest/rootimg.previous"), "previous format\n", "$label: other formats stay"); + is_deeply([ glob("$dest/.packimage-$host+*") ], [], "$label: no temporary file is left"); +} + +{ + my ($name, $dest, $archive) = image('cpio', 'gzip', 'gzip', 'pass'); + my $label = 'previous metainfo that cannot be removed'; + write_text("$dest/bin/ctorrent", <<"SCRIPT"); +#!/bin/sh +while [ \$# -gt 1 ]; do [ "\$1" = -s ] && out=\$2; shift; done +echo new metainfo > "\$out" +SCRIPT + chmod 0755, "$dest/bin/ctorrent"; + unlink("$archive.metainfo") or die $!; + make_path("$archive.metainfo/held"); + my (undef, $errors) = packimage($dest, '-m', 'cpio', '-c', 'gzip', '--tracker', 'http://tracker.example:6969/announce', $name); + like($errors, qr/Cannot publish \Q$archive\E: /, "$label: the failure is reported"); + is(content($archive), "previous archive\n", "$label: the previous archive stays"); + ok(-d "$archive.metainfo/held", "$label: the previous metainfo stays"); + is(content("$dest/rootimg.previous"), "previous format\n", "$label: other formats stay"); + is_deeply([ glob("$dest/.packimage-$host+*") ], [], "$label: no temporary file is left"); +} + +done_testing();