diff --git a/xCAT-server/lib/perl/xCAT/Postage.pm b/xCAT-server/lib/perl/xCAT/Postage.pm index ddf949b67..3e009b684 100644 --- a/xCAT-server/lib/perl/xCAT/Postage.pm +++ b/xCAT-server/lib/perl/xCAT/Postage.pm @@ -173,6 +173,58 @@ my $mn; %::GLOBAL_SN_HASH; %::GLOBAL_TABDUMP_HASH; +#------------------------------------------------------------------------------- + +=head3 defer_syncfiles_to_postboot + + On Ubuntu/Debian the DISKFUL install runs a node's postscripts inside the installer's + in-target chroot -- before the node has booted as itself. The syncfiles postscript works + by asking the management node to scp files INTO the running node, which cannot happen in + that phase: the not-yet-booted node has no sshd for the MN to reach, so the push times + out, syncfiles exits 1, and the node reports status=failed even though the OS installed + perfectly. Defer it to the postbootscripts, which run on the booted node where ssh is + already listening and the MN's push succeeds. + + Scoped to the diskful install path only: netboot and statelite already run their + postscripts on the booted node, so moving syncfiles there would change behaviour that + works. EL/SLES are unaffected either way -- their postscripts run on the booted node. + syncfiles is PREPENDED so it still runs before any postbootscript that consumes the + files it synchronises. + + Arguments: + $os nodetype.os for the node + $provmethod the effective provmethod (the osimage's, when the node names one) + $nodesetstate the nodeset state, when the caller knows it + $postscripts the rendered postscripts list + $postbootscripts the rendered postbootscripts list + Returns: + ($postscripts, $postbootscripts), unchanged unless the deferral applies + +=cut + +#------------------------------------------------------------------------------- +sub defer_syncfiles_to_postboot { + my ($os, $provmethod, $nodesetstate, $postscripts, $postbootscripts) = @_; + + return ($postscripts, $postbootscripts) + unless defined($os) && $os =~ /^(?:ubuntu|debian)/i; + + my $diskful_install = + ($nodesetstate && $nodesetstate eq 'install') + || (!$nodesetstate && defined($provmethod) && $provmethod eq 'install'); + return ($postscripts, $postbootscripts) unless $diskful_install; + + return ($postscripts, $postbootscripts) + unless defined($postscripts) && $postscripts =~ s/^[ \t]*syncfiles[ \t]*\n//m; + + $postbootscripts = "" unless defined $postbootscripts; + $postbootscripts = + "# ubuntu-deferred-postbootscripts-start-here\nsyncfiles\n# ubuntu-deferred-postbootscripts-end-here\n" + . $postbootscripts; + + return ($postscripts, $postbootscripts); +} + sub makescript { my $nodes = shift; my $nodesetstate = shift; @@ -546,38 +598,14 @@ sub makescript { my $postbootscripts; $postbootscripts = getPostbootScripts($node, $osimgname, $script_hash); - # On Ubuntu/Debian the DISKFUL install runs a node's postscripts inside the installer's - # in-target chroot -- before the node has booted as itself. The syncfiles postscript - # works by asking the management node to scp files INTO the running node, which cannot - # happen in that phase: the not-yet-booted node has no sshd for the MN to reach, so the - # push times out, syncfiles exits 1, and the node reports status=failed even though the - # OS installed perfectly. Defer it to the postbootscripts, which run on the booted node - # where ssh is already listening and the MN's push succeeds. - # - # Scope this to the diskful install path only: netboot and statelite already run their - # postscripts on the booted node, so moving syncfiles there would change behaviour that - # works. EL/SLES are unaffected either way -- their postscripts run on the booted node. - # syncfiles is PREPENDED so it still runs before any postbootscript that consumes the - # files it synchronises. - if (defined($os) && $os =~ /^(?:ubuntu|debian)/i) { - my $effective_provmethod = $provmethod; - if ($osimgname && defined($image_hash{$osimgname}{'provmethod'})) { - $effective_provmethod = $image_hash{$osimgname}{'provmethod'}; - } - my $diskful_install = - ($nodesetstate && $nodesetstate eq 'install') - || (!$nodesetstate - && defined($effective_provmethod) - && $effective_provmethod eq 'install'); - if ($diskful_install - && defined($postscripts) - && $postscripts =~ s/^[ \t]*syncfiles[ \t]*\n//m) { - $postbootscripts = "" unless defined $postbootscripts; - $postbootscripts = - "# ubuntu-deferred-postbootscripts-start-here\nsyncfiles\n# ubuntu-deferred-postbootscripts-end-here\n" - . $postbootscripts; - } + # See defer_syncfiles_to_postboot(): on the Ubuntu/Debian diskful install path syncfiles + # has to run on the booted node, not in the installer's in-target chroot. + my $effective_provmethod = $provmethod; + if ($osimgname && defined($image_hash{$osimgname}{'provmethod'})) { + $effective_provmethod = $image_hash{$osimgname}{'provmethod'}; } + ($postscripts, $postbootscripts) = defer_syncfiles_to_postboot( + $os, $effective_provmethod, $nodesetstate, $postscripts, $postbootscripts); # if using zones then must go to the zone.sshbetweennodes # else go to site.sshbetweennodes diff --git a/xCAT-server/lib/xcat/plugins/debian.pm b/xCAT-server/lib/xcat/plugins/debian.pm index 04b470bf4..4c7987195 100644 --- a/xCAT-server/lib/xcat/plugins/debian.pm +++ b/xCAT-server/lib/xcat/plugins/debian.pm @@ -507,6 +507,56 @@ sub copycd } } +#------------------------------------------------------------------------------- + +=head3 subiquity_kcmdline + + Build the kernel command line for a Subiquity (Ubuntu live installer) diskful install. + + boot=casper is required: without it casper never processes netboot=nfs, it scans the local + disks, finds no live media and panics "Unable to find a medium containing a live file + system" (initramfs emergency shell -> PXE loop). + + nfsroot MUST be a literal IP: casper mounts the live filesystem with klibc's nfsmount, which + cannot resolve hostnames ("nfsmount: can't parse IP address ''"). The ds= URL is + fetched later by cloud-init in the booted live system where normal DNS works, so it keeps + the install server's name. + + 'toram' makes casper copy the live squashfs into RAM and UNMOUNT the NFS source (casper + scripts/casper: copy_to_ram then umount of the copy source), so the installer runs from RAM + with NO network root. This is what lets the node reboot at all: with the NFS root still + mounted, a process doing I/O to it during systemd-shutdown (lvm, netplan, udev) blocks in + uninterruptible D state -- it cannot be SIGKILLed, so systemd-shutdown waits forever + ("Waiting for process: (lvm)") and the node never power-cycles into the disk it just + installed. casper has no cmdline knob for NFS mount options -- it parses only nfsroot=, + taking the whole value as the path, so appending ,soft breaks the mount; toram is casper's + supported way to avoid the network root. The 24.04 layers total ~1.5G, well within the CN's + RAM. + + Arguments: + $base the command line built so far + $nfsip the install server as a literal IP, for casper's klibc nfsmount + $pkgdir the install media path exported over NFS + $instserver the install server name, for the cloud-init seed URL + $httpport the xCAT HTTP port + $node the node being installed + Returns: + the completed command line + +=cut + +#------------------------------------------------------------------------------- +sub subiquity_kcmdline { + my ($base, $nfsip, $pkgdir, $instserver, $httpport, $node) = @_; + + my $kcmdline = $base; + $kcmdline .= " autoinstall ip=dhcp boot=casper netboot=nfs nfsroot=${nfsip}:${pkgdir} toram"; + $kcmdline .= " ds=nocloud-net;s=http://${instserver}:${httpport}/install/autoinst/${node}/"; + $kcmdline .= " ---"; + + return $kcmdline; +} + sub mkinstall { xCAT::MsgUtils->message("S", "Doing debian mkinstall"); my $request = shift; @@ -986,29 +1036,8 @@ sub mkinstall { my $kcmdline = "nofb utf8 auto xcatd=" . $instserver; if (using_subiquity($os,$tmplfile)) { - # boot=casper is required: without it casper never processes netboot=nfs, it - # scans the local disks, finds no live media and panics "Unable to find a - # medium containing a live file system" (initramfs emergency shell -> PXE loop). - # nfsroot MUST be a literal IP: casper mounts the live filesystem with klibc's - # nfsmount, which cannot resolve hostnames ("nfsmount: can't parse IP address - # ''"), so resolve instserver to its IP. The ds= URL is fetched later by - # cloud-init in the booted live system where normal DNS works, so it may stay a - # hostname. - # 'toram' makes casper copy the live squashfs into RAM and UNMOUNT the NFS source - # (casper scripts/casper: copy_to_ram then `umount ${copyfrom}`), so the installer runs - # from RAM with NO network root. This fixes the end-of-install reboot hang: with the NFS - # root still mounted, a process doing I/O to it during systemd-shutdown (lvm, netplan, - # udev) blocks in uninterruptible D state -- it cannot be SIGKILLed, so systemd-shutdown - # waits forever ("Waiting for process: (lvm)") and the node never power-cycles into - # the installed disk. With the root in RAM there is nothing to wait on and the reboot - # completes on its own. (casper has no cmdline knob for NFS mount options -- it parses - # only nfsroot=, taking the whole value as the path, so appending ,soft breaks the mount; - # toram is casper's supported way to avoid the network root. The 24.04 layers total - # ~1.5G, well within the CN's RAM.) my $nfsip = xCAT::NetworkUtils->getipaddr($instserver) || $instserver; - $kcmdline .= " autoinstall ip=dhcp boot=casper netboot=nfs nfsroot=${nfsip}:${pkgdir} toram"; - $kcmdline .= " ds=nocloud-net;s=http://${instserver}:${httpport}/install/autoinst/${node}/"; - $kcmdline .= " ---"; + $kcmdline = subiquity_kcmdline($kcmdline, $nfsip, $pkgdir, $instserver, $httpport, $node); } else { $kcmdline .= " url=http://${instserver}:$httpport/install/autoinst/$node"; $kcmdline .= " mirror/http/hostname=${instserver}:$httpport"; diff --git a/xCAT-test/unit/debian_subiquity_netboot.t b/xCAT-test/unit/debian_subiquity_netboot.t index 5b3b3029c..0f512cd0d 100644 --- a/xCAT-test/unit/debian_subiquity_netboot.t +++ b/xCAT-test/unit/debian_subiquity_netboot.t @@ -1,47 +1,74 @@ #!/usr/bin/env perl use strict; use warnings; + +use FindBin; use Test::More; -# Regression: the Ubuntu (subiquity) diskful install netboots the live installer over NFS. -# The kernel command line built in debian.pm must satisfy TWO casper requirements or the -# installer never boots and the node PXE-loops forever (surfacing to the test as -# "ssh: connect ... port 22: Connection refused"): +# The Ubuntu live installer (Subiquity) is booted over NFS. Three things on the kernel command +# line decide whether it boots at all and whether the node can reboot into the disk afterwards: # -# 1. boot=casper -- without it casper does not process netboot=nfs at all; it scans the -# local disks, finds nothing, and panics "Unable to find a medium containing a live -# file system", dropping to the initramfs emergency shell. +# boot=casper without it casper never processes netboot=nfs -- it scans local disks, finds no +# live media and panics into the initramfs shell, which PXE-loops. +# nfsroot=IP casper mounts the live filesystem with klibc's nfsmount, which has no resolver, +# so a hostname there fails with "can't parse IP address". +# toram casper copies the squashfs to RAM and unmounts the NFS source. Without it the +# NFS root stays mounted, systemd-shutdown blocks forever on I/O to it and the +# node never power-cycles into the disk it just installed. # -# 2. nfsroot must be a literal IP, NOT a hostname. casper mounts the live filesystem with -# klibc's nfsmount, which does NOT resolve hostnames -- it fails with -# "nfsmount: can't parse IP address ''". (busybox/util-linux `mount -t nfs` -# resolves names, which is why manual mounts work while casper's does not.) The instserver -# must therefore be passed through xCAT::NetworkUtils->getipaddr() before it is put in -# nfsroot=. The ds=...http URL is fetched later by cloud-init in the booted live system, -# where normal DNS works, so only nfsroot needs the IP. +# Build the command line for real and inspect it, rather than reading the source that builds it. -sub slurp { my ($p) = @_; local $/; open my $fh, '<', $p or return undef; <$fh> } +use lib "$FindBin::Bin/../../perl-xCAT"; +use lib "$FindBin::Bin/../../xCAT-server/lib/perl"; +my $plugin = "$FindBin::Bin/../../xCAT-server/lib/xcat/plugins/debian.pm"; +plan skip_all => 'debian.pm not found' unless -r $plugin; +eval { require $plugin; 1 } or plan skip_all => "could not load debian.pm: $@"; -my $deb = slurp('xCAT-server/lib/xcat/plugins/debian.pm'); -plan skip_all => 'debian.pm not found' unless defined $deb; +my $cmdline = xCAT_plugin::debian::subiquity_kcmdline( + 'nofb utf8 auto xcatd=xcatmn', # what mkinstall has built so far + '10.0.0.1', # instserver resolved to a literal IP + '/install/ubuntu24.04/x86_64', # pkgdir exported over NFS + 'xcatmn', # instserver by name + '80', # httpport + 'node01', # node +); -# The exact netboot fragment is unique to the subiquity install cmdline, so match it directly -# instead of trying to slice one of the several using_subiquity() blocks. -like($deb, qr/\bboot=casper\b/, - 'subiquity netboot kcmdline includes boot=casper (else casper scans local disks and panics)'); -like($deb, qr/autoinstall ip=dhcp boot=casper netboot=nfs nfsroot=\$\{nfsip\}:/, - 'subiquity kcmdline is: autoinstall ip=dhcp boot=casper netboot=nfs nfsroot='); -like($deb, qr/getipaddr\(\$instserver\)/, - 'instserver is resolved to an IP via getipaddr for nfsroot (klibc nfsmount cannot resolve hostnames)'); -unlike($deb, qr/nfsroot=\$\{instserver\}:/, - 'nfsroot does NOT use the bare instserver hostname (would break klibc nfsmount)'); +# --- the three settings the installer cannot boot without ------------------ +like($cmdline, qr/(?:^| )boot=casper(?: |$)/, + 'casper is told to boot, so it processes netboot=nfs instead of scanning disks'); +like($cmdline, qr{(?:^| )nfsroot=10\.0\.0\.1:/install/ubuntu24\.04/x86_64(?: |$)}, + 'nfsroot names the install server by IP, which klibc nfsmount can parse'); +like($cmdline, qr/(?:^| )toram(?: |$)/, + 'toram copies the live filesystem to RAM so the NFS root is unmounted before shutdown'); -# The subiquity netboot cmdline must include 'toram' so casper copies the squashfs into RAM and -# unmounts the NFS source -- otherwise the end-of-install reboot wedges in systemd-shutdown on a -# D-state process doing I/O to the still-mounted NFS root, and the node never boots the installed disk. -like($deb, qr/\btoram\b/, - 'subiquity netboot cmdline includes toram (run from RAM, no NFS root at shutdown -> no reboot hang)'); -unlike($deb, qr/nfsroot=[^ ]*,soft/, - 'nfsroot does NOT append ,soft (casper takes the whole nfsroot value as the path, which breaks the mount)'); +unlike($cmdline, qr/nfsroot=xcatmn:/, + 'nfsroot never carries a hostname, which klibc nfsmount cannot resolve'); +unlike($cmdline, qr/nfsroot=[^ ]*,/, + 'no mount options are appended to nfsroot -- casper takes the whole value as the path'); + +# --- the rest of the line --------------------------------------------------- +like($cmdline, qr/(?:^| )autoinstall(?: |$)/, 'the installer runs unattended'); +like($cmdline, qr/(?:^| )ip=dhcp(?: |$)/, 'the live system configures its NIC by DHCP'); +like($cmdline, qr/(?:^| )netboot=nfs(?: |$)/, 'the live filesystem is fetched over NFS'); + +# cloud-init fetches the seed later, in the booted live system, where DNS works -- so this one +# keeps the install server's name rather than its address. +like($cmdline, qr{(?:^| )ds=nocloud-net;s=http://xcatmn:80/install/autoinst/node01/(?: |$)}, + 'the cloud-init seed URL addresses the install server by name'); + +is((split / /, $cmdline)[-1], '---', + 'the line ends with the separator that divides installer arguments from kernel arguments'); + +# What mkinstall had already built is preserved, not replaced. +like($cmdline, qr/^nofb utf8 auto xcatd=xcatmn /, + 'the command line built so far is kept ahead of the installer arguments'); + +# A non-default HTTP port reaches the seed URL. +{ + my $alt = xCAT_plugin::debian::subiquity_kcmdline( + 'base', '10.0.0.1', '/pkgdir', 'xcatmn', '8080', 'node02'); + like($alt, qr{ds=nocloud-net;s=http://xcatmn:8080/install/autoinst/node02/}, + 'the seed URL carries the configured HTTP port and node'); +} done_testing(); diff --git a/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t b/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t index 00cde9e77..e0e935ffb 100644 --- a/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t +++ b/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t @@ -2,58 +2,94 @@ use strict; use warnings; -use File::Spec; use FindBin; use Test::More; -# Regression: on Ubuntu/Debian the DISKFUL install runs a node's postscripts inside the -# installer's in-target chroot -- before the node has booted as itself. The syncfiles -# postscript works by asking the management node to scp files INTO the running node, which -# cannot happen in that phase: the not-yet-booted node has no sshd for the MN to reach, so the -# push times out, syncfiles exits 1, and the node reports status=failed even though the OS -# installed perfectly. -# -# syncfiles must therefore move to the postbootscripts set, which runs on the booted node -# where ssh is already listening. Two things bound that move: -# -# * It applies to the DISKFUL install path only. netboot and statelite already run their -# postscripts on the booted node, so moving syncfiles there changes working behaviour for -# no reason. EL/SLES are unaffected either way -- their postscripts already run on the -# booted node. -# * syncfiles must run BEFORE the existing postbootscripts, since a postbootscript may -# consume the files it synchronises. Appending it would invert that. +# On the Ubuntu/Debian DISKFUL install path a node's postscripts run inside the installer's +# in-target chroot, before the node has booted as itself. syncfiles asks the management node to +# scp files INTO the running node, which cannot work there -- no sshd yet -- so it times out and +# the node reports status=failed even though the OS installed fine. Postage defers it to the +# postbootscripts, which run on the booted node. Drive that decision directly. -my $repo_root = File::Spec->rel2abs( - File::Spec->catdir( $FindBin::Bin, '..', '..' ) +my $repo = "$FindBin::Bin/../.."; +plan skip_all => 'Postage.pm not found' unless -r "$repo/xCAT-server/lib/perl/xCAT/Postage.pm"; + +use lib "$FindBin::Bin/../../perl-xCAT"; +use lib "$FindBin::Bin/../../xCAT-server/lib/perl"; +eval { require xCAT::Postage; 1 } or plan skip_all => "could not load xCAT::Postage: $@"; + +my $DEFERRED = "# ubuntu-deferred-postbootscripts-start-here\nsyncfiles\n" + . "# ubuntu-deferred-postbootscripts-end-here\n"; + +sub defer { return xCAT::Postage::defer_syncfiles_to_postboot(@_) } + +# --- the case the fix exists for ------------------------------------------- +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', 'install', "syncfiles\notherpkgs\n", "setupntp\n"); + + is($post, "otherpkgs\n", 'syncfiles is removed from the postscripts'); + is($postboot, $DEFERRED . "setupntp\n", + 'syncfiles is prepended to the postbootscripts, ahead of what consumes its files'); +} + +# The nodeset state is not always known; the osimage provmethod decides then. +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', undef, "syncfiles\n", "setupntp\n"); + is($post, '', 'provmethod=install defers when no nodeset state is given'); + is($postboot, $DEFERRED . "setupntp\n", 'and the postbootscripts receive it'); +} + +# --- paths that must NOT change -------------------------------------------- +my @untouched = ( + [ 'netboot keeps its postscripts on the booted node already', + 'ubuntu24.04', 'netboot', 'netboot' ], + [ 'statelite is unaffected', + 'ubuntu22.04', 'statelite', 'statelite' ], + [ 'EL runs postscripts on the booted node, so nothing moves', + 'rhels9.4', 'install', 'install' ], + [ 'SLES is unaffected', + 'sles15.6', 'install', 'install' ], + [ 'an undefined os is left alone', + undef, 'install', 'install' ], ); -my $path = File::Spec->catfile( $repo_root, 'xCAT-server', 'lib', 'perl', 'xCAT', 'Postage.pm' ); -plan skip_all => "Postage.pm not found" unless -f $path; +foreach my $case (@untouched) { + my ($name, $os, $provmethod, $state) = @$case; + my ($post, $postboot) = defer($os, $provmethod, $state, "syncfiles\notherpkgs\n", "setupntp\n"); + is($post, "syncfiles\notherpkgs\n", "$name: the postscripts are unchanged"); + is($postboot, "setupntp\n", "$name: the postbootscripts are unchanged"); +} -my $src = do { local $/; open my $fh, '<', $path or die $!; <$fh> }; +# A node that does not run syncfiles must not gain an empty deferral block. +{ + my ($post, $postboot) = defer('ubuntu24.04', 'install', 'install', "otherpkgs\n", "setupntp\n"); + is($post, "otherpkgs\n", 'a node without syncfiles keeps its postscripts'); + is($postboot, "setupntp\n", 'and gains no deferral block'); +} -# Isolate the deferral block so the assertions below cannot accidentally match code elsewhere. -my ($block) = $src =~ /(\n[^\n]*ubuntu\|debian[^\n]*\n(?:.*?\n)*?[^\n]*syncfiles(?:.*?\n)*?\s*\}\n\s*\}\n)/; -ok( defined $block, 'found the syncfiles deferral block in makescript' ) - or do { done_testing(); exit }; +# The name is matched whole: a postscript whose name merely contains "syncfiles" stays put. +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', 'install', "syncfiles2\nmysyncfiles\n", "setupntp\n"); + is($post, "syncfiles2\nmysyncfiles\n", 'a lookalike postscript name is not deferred'); + is($postboot, "setupntp\n", 'and nothing is added to the postbootscripts'); +} -like( $block, qr/\$os\s*=~\s*.\^\(\?:ubuntu\|debian\)/, - 'the move is guarded to ubuntu/debian nodes only' ); +# Indented entries are still the syncfiles postscript. +{ + my ($post, $postboot) = defer('ubuntu24.04', 'install', 'install', " syncfiles \nfoo\n", undef); + is($post, "foo\n", 'an indented syncfiles entry is deferred'); + is($postboot, $DEFERRED, 'an undefined postbootscripts list becomes the deferral block'); +} -like( $block, qr/\$diskful_install|\bnodesetstate\b|\bprovmethod\b/, - 'the move is guarded to the diskful install path (not netboot/statelite)' ); - -like( $block, qr/\$postscripts\s*=~\s*s\/\^\[[^\]]*\]\*syncfiles/, - 'syncfiles is stripped from the in-target postscripts list' ); - -like( $block, qr/\$postbootscripts\s*=\s*"[^"]*syncfiles[^"]*"\s*\.\s*\$postbootscripts/, - 'syncfiles is PREPENDED to postbootscripts, so it runs before scripts that consume its files' ); - -unlike( $block, qr/\$postbootscripts\s*\.=\s*"syncfiles/, - 'syncfiles is not appended after the existing postbootscripts' ); - -# The strip and the re-add must be paired, so a node with no syncfiles postscript never -# acquires a spurious one. -like( $block, qr/if\s*\(.*?\$postscripts.*?syncfiles.*?\)\s*\{\s*.*?\$postbootscripts/s, - 'syncfiles is only added to postbootscripts when it was present in postscripts' ); +# Rendering the same node twice must not stack a second copy. +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', 'install', "syncfiles\nfoo\n", "setupntp\n"); + my ($post2, $postboot2) = defer('ubuntu24.04', 'install', 'install', $post, $postboot); + is($post2, $post, 'a second pass leaves the postscripts alone'); + is($postboot2, $postboot, 'a second pass does not duplicate syncfiles'); +} done_testing(); diff --git a/xCAT-test/unit/ubuntu_resolvconf_ip.t b/xCAT-test/unit/ubuntu_resolvconf_ip.t index d35600d8b..4a1f1ed8f 100644 --- a/xCAT-test/unit/ubuntu_resolvconf_ip.t +++ b/xCAT-test/unit/ubuntu_resolvconf_ip.t @@ -1,42 +1,79 @@ #!/usr/bin/env perl use strict; use warnings; + +use FindBin; +use File::Temp qw(tempdir); use Test::More; -# Regression: the Ubuntu (subiquity) diskful install writes the installer's /etc/resolv.conf -# from compute.subiquity.tmpl's "DNS setup" early-command. A nameserver line in -# /etc/resolv.conf MUST be an IP address -- glibc's resolver does NOT resolve a hostname -# given on a nameserver line, it simply discards the entry. The template used to write the -# xcatmaster *name* there: +# A nameserver line in /etc/resolv.conf must hold an IP address: glibc's resolver discards an +# entry naming a host. Writing the xcatmaster *name* left the installer -- and the in-target +# apt-get that inherits the file -- with no usable DNS, so the install hung resolving +# archive.ubuntu.com. The template resolves the name to an address first. # -# echo "nameserver #TABLE:noderes:$NODE:xcatmaster#" >/etc/resolv.conf -# -# which left the installer (and the in-target apt-get update, which inherits this resolv.conf) -# with no usable DNS. The install then hangs resolving archive.ubuntu.com -- the node never -# finishes, and the test sees "ssh: connect ... port 22: Connection refused". (Everything the -# early-command did BEFORE this point still worked because the live installer's DHCP-provided -# resolv.conf can resolve the xcatmaster name -- so the failure only surfaces at the very next -# DNS-dependent step, the in-target apt.) -# -# The fix resolves the xcatmaster hostname to an IPv4 address (via getent, while DNS still -# works) and writes that IP into resolv.conf. +# Run the template's own shell for that step and inspect the file it writes. -sub slurp { my ($p) = @_; local $/; open my $fh, '<', $p or return undef; <$fh> } +my $tmpl = "$FindBin::Bin/../../xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl"; +plan skip_all => 'compute.subiquity.tmpl not found' unless -r $tmpl; -my $tmpl = slurp('xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl'); -plan skip_all => 'compute.subiquity.tmpl not found' unless defined $tmpl; +open(my $fh, '<', $tmpl) or die "open $tmpl: $!"; +my $source = do { local $/; <$fh> }; +close $fh; -# The old, broken form -- a bare hostname token written straight into the nameserver line -- -# must be gone. -unlike($tmpl, qr/nameserver \s+ \#TABLE:noderes:\$NODE:xcatmaster\# /x, - 'resolv.conf nameserver is NOT the bare xcatmaster hostname token (glibc cannot resolve a name there)'); +my ($fragment) = $source =~ m{^(\s*xcatmaster_host=.*?echo "nameserver [^\n]*\n)}ms; +BAIL_OUT('the template does not build /etc/resolv.conf from the xcatmaster') unless $fragment; -# The xcatmaster name must be resolved to an IPv4 address before it is used as a nameserver. -like($tmpl, qr/getent \s+ ahostsv4 /x, - 'DNS setup resolves the xcatmaster to an IPv4 address via getent ahostsv4'); +# $NODE and the xcatmaster come from the xCAT template renderer; stand in for both. +sub write_resolv_conf { + my (%opt) = @_; + my $root = tempdir(CLEANUP => 1); -# And the nameserver line must be written from that resolved IP variable. -like($tmpl, qr/nameserver \s+ \$xcatmaster_ip/x, - 'resolv.conf nameserver is written from the resolved xcatmaster IP'); + my $script = $fragment; + $script =~ s/\#TABLE:noderes:\$NODE:xcatmaster\#/$opt{xcatmaster}/; + $script =~ s{/etc/resolv\.conf}{$root/resolv.conf}g; + + # getent is the resolver the fragment uses; make it answer as the test wants. + my $getent = $opt{resolves} + ? "getent() { printf '%s\\n' '$opt{resolves} $opt{xcatmaster}'; }\n" + : "getent() { return 2; }\n"; + + system('bash', '-c', $getent . $script) == 0 or return { rc => $? }; + + open my $rh, '<', "$root/resolv.conf" or return { rc => 0, content => '' }; + my $content = do { local $/; <$rh> }; + close $rh; + return { rc => 0, content => $content }; +} + +# --- the case the fix exists for ------------------------------------------- +{ + my $r = write_resolv_conf(xcatmaster => 'xcatmn', resolves => '10.0.0.1'); + is($r->{content}, "nameserver 10.0.0.1\n", + 'the nameserver line holds the address, which glibc will actually use'); + unlike($r->{content}, qr/nameserver \s+ xcatmn/x, + 'the nameserver line never holds a host name, which glibc discards'); +} + +# --- more than one address: the first is taken ----------------------------- +{ + my $r = write_resolv_conf(xcatmaster => 'xcatmn', resolves => '10.0.0.1'); + like($r->{content}, qr/^nameserver \d+\.\d+\.\d+\.\d+$/m, + 'exactly one IPv4 address is written'); +} + +# --- resolution fails: fall back rather than write an empty entry ---------- +{ + my $r = write_resolv_conf(xcatmaster => 'xcatmn'); + is($r->{content}, "nameserver xcatmn\n", + 'an unresolvable xcatmaster falls back to its name rather than an empty nameserver'); + unlike($r->{content}, qr/^nameserver\s*$/m, + 'no empty nameserver line is written'); +} + +# --- an xcatmaster already given as an address is left alone --------------- +{ + my $r = write_resolv_conf(xcatmaster => '10.0.0.1', resolves => '10.0.0.1'); + is($r->{content}, "nameserver 10.0.0.1\n", 'an address-valued xcatmaster is written as is'); +} done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t index 6d57d5e30..a916de89f 100644 --- a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t +++ b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t @@ -2,55 +2,124 @@ use strict; use warnings; -use File::Spec; use FindBin; +use File::Temp qw(tempdir); +use IO::Socket::INET; use Test::More; -# Regression: after Subiquity installs the OS the node MUST be flipped to local-disk boot, or it -# PXE-loops straight back into the installer and the installed OS -- with its sshd -- never -# boots. The diskful case then only ever reports -# "ssh: connect to host port 22: Connection refused". +# When Subiquity finishes it reboots, and unless the node has been flipped to local-disk boot it +# PXEs straight back into the installer. The in-target post-script does that through +# updateflag.awk, which needs gawk's |& coprocess -- Ubuntu's /usr/bin/awk is mawk, so the flip +# silently failed and the node reinstalled forever. The template now does the exchange from the +# live installer over bash's built-in /dev/tcp. # -# xCAT performs that flip when the node reports "next" to xcatd on the install-monitor port, -# which makes xcatd run "nodeset next" and rewrite the node's xNBA script to fall through -# to the local disk. The in-target post-script tries this through updateflag.awk, but that -# depends on gawk's |& / /inet coprocess and on /usr/bin/awk being gawk -- on Ubuntu -# /usr/bin/awk is normally mawk, which has neither, so the flip fails silently. -# -# The template must therefore trigger the flip itself from the live installer, with no gawk -# dependency, and must NOT report success when it did not happen. +# Run that command against a stand-in for xcatd and check the exchange, rather than reading the +# template text. -my $repo_root = File::Spec->rel2abs( - File::Spec->catdir( $FindBin::Bin, '..', '..' ) -); -my $path = File::Spec->catfile( - $repo_root, 'xCAT-server', 'share', 'xcat', 'install', 'ubuntu', 'compute.subiquity.tmpl' -); -plan skip_all => 'compute.subiquity.tmpl not found' unless -f $path; +my $tmpl = "$FindBin::Bin/../../xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl"; +plan skip_all => 'compute.subiquity.tmpl not found' unless -r $tmpl; -my $tmpl = do { local $/; open my $fh, '<', $path or die $!; <$fh> }; +my $XCATD_PORT = 3002; # the install-monitor port the template addresses -like( $tmpl, qr{/dev/tcp/\$xm/3002}, - 'the flip contacts xcatd on the install-monitor port from the installer' ); -like( $tmpl, qr{printf "next}, - 'it sends the "next" token that makes xcatd run "nodeset next"' ); -like( $tmpl, qr{xm=#XCATVAR:XCATMASTER#}, - "the flip targets the node's own xcatmaster" ); -like( $tmpl, qr{^\s*-\s*\['bash',\s*'-c',}m, - 'the flip runs under bash (dash has no /dev/tcp) via an argv list command' ); +my $probe = IO::Socket::INET->new( + LocalAddr => '127.0.0.1', LocalPort => $XCATD_PORT, Proto => 'tcp', + Listen => 5, ReuseAddr => 1); +plan skip_all => "port $XCATD_PORT is not available on the loopback interface" unless $probe; +close $probe; # each case below opens its own listener, or none at all -# A failed flip must be visible. The original form broke out of its retry loop as soon as the -# socket connected -- never on a confirmed exchange -- and ended in `true`, so a total failure -# looked exactly like success and the node silently reinstalled forever. -like( $tmpl, qr{ok=1}, - 'success is recorded only after the exchange completes, not merely on connect' ); -like( $tmpl, qr{FAILED to flip}, - 'a failed flip is reported into the install log rather than passing silently' ); +open(my $fh, '<', $tmpl) or die "open $tmpl: $!"; +my $source = do { local $/; <$fh> }; +close $fh; -# The magic-SysRq forced reboot must be gone: toram already unmounts the NFS live root, so the -# shutdown hang it worked around cannot occur, and an unconditional timed reboot would race a -# slow install. -unlike( $tmpl, qr{sysrq-trigger}, - 'no unconditional magic-SysRq reboot (toram removes the hang it worked around)' ); +# The boot flip is the late-command that talks to the install-monitor port. +my ($command) = $source =~ m{- \['bash', '-c', '(.*?/dev/tcp/.*?)'\]}; +BAIL_OUT('no late-command in the template performs the boot flip over /dev/tcp') unless $command; + +# Run the command with the install server pointed at our stand-in, and its log inside a scratch +# tree. Everything else is the template's own text. +sub run_flip { + my (%opt) = @_; + my $root = tempdir(CLEANUP => 1); + mkdir "$root/target"; mkdir "$root/target/var"; mkdir "$root/target/var/log"; + mkdir "$root/target/var/log/xcat"; + + my $script = $command; + $script =~ s/\#XCATVAR:XCATMASTER\#/127.0.0.1/; + $script =~ s{/target/var/log/xcat/xcat\.log}{$root/target/var/log/xcat/xcat.log}; + $script =~ s/sleep 5/sleep 1/; # shorten the retry pause, keep the retry + + my $pid; + if ($opt{listen}) { + $pid = fork(); + die "fork: $!" unless defined $pid; + if (!$pid) { # the stand-in xcatd + sleep $opt{delay} if $opt{delay}; # appear only after the first attempts fail + my $srv = IO::Socket::INET->new(LocalAddr => '127.0.0.1', LocalPort => $XCATD_PORT, + Proto => 'tcp', Listen => 5, ReuseAddr => 1) or exit 1; + open my $seen, '>', "$root/received" or exit 1; + $seen->autoflush(1); + for (1 .. $opt{listen}) { + my $c = $srv->accept() or last; + $c->autoflush(1); + print {$c} "ready\n"; + my $line = <$c>; + print {$seen} $line if defined $line; + print {$c} "ok\n" unless $opt{no_ack}; + close $c; + } + close $seen; + close $srv; + exit 0; + } + } + + # The installer would hang here if the exchange ever blocked, so bound it. + # the no-listener case prints "Connection refused" by design + my $rc = system("timeout 25 bash -c \Q$script\E 2>/dev/null"); + my $timed_out = (($rc >> 8) == 124); + if ($pid) { kill 'TERM', $pid; waitpid($pid, 0) } + + my $received = ''; + if (open my $rh, '<', "$root/received") { local $/; $received = <$rh>; close $rh } + + my $log = ''; + if (open my $lh, '<', "$root/target/var/log/xcat/xcat.log") { local $/; $log = <$lh>; close $lh } + return { rc => $rc, timed_out => $timed_out, log => $log, received => $received }; +} + +# --- xcatd answers: the node is flipped ------------------------------------ +{ + my $r = run_flip(listen => 1); + is($r->{rc}, 0, 'the boot flip exits cleanly so the install is not failed by it'); + is($r->{log}, '', 'nothing is written to the install log when the flip succeeds'); + is($r->{received}, "next\n", + 'the node sends the token that makes xcatd run "nodeset next"'); + ok(!$r->{timed_out}, 'the exchange completes rather than hanging the late-command'); +} + +# --- xcatd never answers: the failure is recorded, not swallowed ----------- +{ + my $r = run_flip(listen => 0); + is($r->{rc}, 0, 'a failed flip still exits 0 rather than aborting the install'); + like($r->{log}, qr/FAILED to flip/, + 'a failed flip is recorded in the install log instead of PXE-looping silently'); + like($r->{log}, qr/127\.0\.0\.1:3002/, + 'the log names the install server and port that could not be reached'); +} + +# --- xcatd accepts but never acknowledges ---------------------------------- +{ + my $r = run_flip(listen => 1, no_ack => 1); + like($r->{log}, qr/FAILED to flip/, + 'a connection without an acknowledgement counts as a failure, not a success'); +} + +# --- the command retries rather than giving up on the first refusal -------- +{ + # Answer only on a later connection: the flip must still succeed. + my $r = run_flip(listen => 1, delay => 2); + is($r->{log}, '', 'the flip retries until the install monitor answers'); + is($r->{received}, "next\n", 'and the token still reaches it on the later attempt'); +} done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_bootparams.t b/xCAT-test/unit/ubuntu_subiquity_bootparams.t deleted file mode 100644 index fccc9502f..000000000 --- a/xCAT-test/unit/ubuntu_subiquity_bootparams.t +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env perl -use strict; -use warnings; -use Test::More; - -my $plugin_path = defined $ENV{XCATROOT} ? "$ENV{XCATROOT}/lib/perl/xCAT_plugin/debian.pm" : ''; -$plugin_path = "xCAT-server/lib/xcat/plugins/debian.pm" - unless -f $plugin_path; - -plan skip_all => "debian.pm not found" unless -f $plugin_path; - -my $src = do { local $/; open my $fh, '<', $plugin_path or die $!; <$fh> }; - -like($src, qr/autoinstall ip=dhcp boot=casper netboot=nfs/, - 'subiquity bootparams enable autoinstall (boot=casper is required for casper to process netboot=nfs)'); -like($src, qr/ds=nocloud-net;s=http:\/\//, 'subiquity bootparams point at NoCloud seed'); -like($src, qr/\$kcmdline\s*\.=\s*" ---";/, 'subiquity bootparams end with installer argument separator'); - -done_testing();