diff --git a/docs/source/troubleshooting/os_installation/index.rst b/docs/source/troubleshooting/os_installation/index.rst index 6a263c5f3..ac8ed52cf 100644 --- a/docs/source/troubleshooting/os_installation/index.rst +++ b/docs/source/troubleshooting/os_installation/index.rst @@ -41,4 +41,5 @@ Next chapter introduces the procedures on how to troubleshoot operating system i log_to_mn_cn.rst debug_port.rst ssh_enable.rst + ubuntu_subiquity_memory.rst diff --git a/docs/source/troubleshooting/os_installation/ubuntu_subiquity_memory.rst b/docs/source/troubleshooting/os_installation/ubuntu_subiquity_memory.rst new file mode 100644 index 000000000..c31079284 --- /dev/null +++ b/docs/source/troubleshooting/os_installation/ubuntu_subiquity_memory.rst @@ -0,0 +1,28 @@ +Ubuntu diskful install needs memory for the live filesystem +=========================================================== + +An Ubuntu diskful (Subiquity) install boots the live installer over NFS and then copies it +into RAM, because the kernel command line xCAT generates carries ``toram``. This is +deliberate: with the NFS root still mounted at the end of the install, ``systemd-shutdown`` +blocks on I/O to it and the node never reboots into the disk it just installed. + +The consequence is that the compute node must have enough memory for the live filesystem on +top of whatever the installer itself needs. The squashfs layers are roughly 1.5 GB on 24.04 +and grow with each release, so **4 GB is a practical floor and 8 GB is comfortable**. + +A node with too little memory fails part-way through the install, and the symptom resembles +an unrelated problem: the node reboots back into the installer, so it looks like a PXE loop +rather than an out-of-memory condition. Check the console before assuming a boot-flip +failure. :: + + rcons + +To confirm, look for the OOM killer in the installer's kernel messages, or watch the node's +memory from the hypervisor while it installs. For a libvirt-hosted node the memory is +``vmm``/``vmmemory`` in the **vm** table. :: + + lsdef -i vmmemory + chdef vmmemory=8192 + +Diskless (netboot) nodes have a related but separate requirement: the rootimg is unpacked +into a tmpfs, and on ppc64le the 64 KB page size inflates it considerably. diff --git a/perl-xCAT/xCAT/NetworkUtils.pm b/perl-xCAT/xCAT/NetworkUtils.pm index f8b363ca0..2d9088f6c 100644 --- a/perl-xCAT/xCAT/NetworkUtils.pm +++ b/perl-xCAT/xCAT/NetworkUtils.pm @@ -288,8 +288,20 @@ sub getipaddr #print "============================\n"; #cache, do not lookup DNS each time + # + # An unrestricted lookup asks for AF_UNSPEC and caches whatever came back, which + # on a dual-stack host is the AAAA record. Serving that to an OnlyV4 caller gives + # it an IPv6 address it cannot use: debian.pm renders nfsroot=
:, + # and nfsroot=2001:db8::1:/install does not parse. + my $cached_v6 = + defined($::hostiphash{$iporhost}) + && $::hostiphash{$iporhost} + && $::hostiphash{$iporhost}{hostip} + && $::hostiphash{$iporhost}{hostip} =~ /:/; if ( - ((not $extraarguments{OnlyV6}) and (not $extraarguments{GetAllAddresses})) and defined($::hostiphash{$iporhost}) and $::hostiphash{$iporhost}) + ((not $extraarguments{OnlyV6}) and (not $extraarguments{GetAllAddresses})) + and (not($extraarguments{OnlyV4} and $cached_v6)) + and defined($::hostiphash{$iporhost}) and $::hostiphash{$iporhost}) { if($extraarguments{GetNumber} ) { diff --git a/xCAT-server/lib/perl/xCAT/Postage.pm b/xCAT-server/lib/perl/xCAT/Postage.pm index b7d44b9d3..e9d40ba8c 100644 --- a/xCAT-server/lib/perl/xCAT/Postage.pm +++ b/xCAT-server/lib/perl/xCAT/Postage.pm @@ -173,6 +173,59 @@ my $mn; %::GLOBAL_SN_HASH; %::GLOBAL_TABDUMP_HASH; +#------------------------------------------------------------------------------- + +=head3 defer_syncfiles_to_postboot + + On the Ubuntu/Debian diskful install path a node's postscripts run inside the installer's + in-target chroot, before the node has booted as itself. syncfiles asks the management node + to scp files INTO the running node, which cannot work there -- there is no sshd yet -- so it + times out and the node reports status=failed although the OS installed. Defer it to the + postbootscripts, which run on the booted node. + + Scoped to the diskful install path: netboot and statelite already run their postscripts on + the booted node, and EL/SLES are unaffected either way. syncfiles is prepended so it still + runs before any postbootscript that consumes the files it synchronises. + + Arguments: + $os nodetype.os for the node + $provmethod the effective provmethod (the osimage's, when the node names one) + $nodesetstate the nodeset state, when the caller knows it + $postscripts the rendered postscripts list + $postbootscripts the rendered postbootscripts list + Returns: + ($postscripts, $postbootscripts), unchanged unless the deferral applies + +=cut + +#------------------------------------------------------------------------------- +sub defer_syncfiles_to_postboot { + my ($os, $provmethod, $nodesetstate, $postscripts, $postbootscripts) = @_; + + return ($postscripts, $postbootscripts) + unless defined($os) && $os =~ /^(?:ubuntu|debian)/i; + + my $diskful_install = + ($nodesetstate && $nodesetstate eq 'install') + || (!$nodesetstate && defined($provmethod) && $provmethod eq 'install'); + return ($postscripts, $postbootscripts) unless $diskful_install; + + return ($postscripts, $postbootscripts) + unless defined($postscripts) && $postscripts =~ s/^[ \t]*syncfiles[ \t]*\n//m; + + $postbootscripts = "" unless defined $postbootscripts; + + # A node may already list syncfiles as a postbootscript of its own; do not add a second copy. + return ($postscripts, $postbootscripts) + if $postbootscripts =~ /^[ \t]*syncfiles[ \t]*$/m; + + $postbootscripts = + "# ubuntu-deferred-postbootscripts-start-here\nsyncfiles\n# ubuntu-deferred-postbootscripts-end-here\n" + . $postbootscripts; + + return ($postscripts, $postbootscripts); +} + sub makescript { my $nodes = shift; my $nodesetstate = shift; @@ -546,6 +599,19 @@ sub makescript { my $postbootscripts; $postbootscripts = getPostbootScripts($node, $osimgname, $script_hash); + # See defer_syncfiles_to_postboot(): on the Ubuntu/Debian diskful install path syncfiles + # has to run on the booted node, not in the installer's in-target chroot. + # + # nodetype.provmethod is often an osimage NAME rather than 'install'. getScripts() above + # was handed this same %image_hash and filled it from the osimage table, provmethod + # included, so resolve through it -- getDisklessNet() reads the same key the same way. + my $effective_provmethod = $provmethod; + if ($osimgname && defined($image_hash{$osimgname}{'provmethod'})) { + $effective_provmethod = $image_hash{$osimgname}{'provmethod'}; + } + ($postscripts, $postbootscripts) = defer_syncfiles_to_postboot( + $os, $effective_provmethod, $nodesetstate, $postscripts, $postbootscripts); + # if using zones then must go to the zone.sshbetweennodes # else go to site.sshbetweennodes my $enablesshbetweennodes; diff --git a/xCAT-server/lib/perl/xCAT/Template.pm b/xCAT-server/lib/perl/xCAT/Template.pm index 8eba50a8f..177cf48ec 100644 --- a/xCAT-server/lib/perl/xCAT/Template.pm +++ b/xCAT-server/lib/perl/xCAT/Template.pm @@ -1796,8 +1796,21 @@ sub ubuntu_subiquity_apt_config ' primary:', " - uri: $online_mirror", ); - if (@otherpkg_sources) { + # On 20.04/22.04 Subiquity renders the target's sources.list from the install media + # alone, so in-target apt cannot find packages the ISO does not carry (chrony). Add the + # online archive through `sources:` -- `sources_list:` is a curtin key Subiquity's + # schema ignores. Excluded on Deb822 releases, where the primary mirror already lands + # in ubuntu.sources and these legacy .list files would duplicate the same suites. + my $need_sources_block = !$use_deb822; + if ($need_sources_block) { push @lines, ' sources:'; + push @lines, ' xcat-ubuntu-archive.list:'; + push @lines, qq( source: "deb $online_mirror \$RELEASE main restricted universe multiverse"); + push @lines, ' xcat-ubuntu-updates.list:'; + push @lines, qq( source: "deb $online_mirror \$RELEASE-updates main restricted universe multiverse"); + } + if (@otherpkg_sources) { + push @lines, ' sources:' unless $need_sources_block; my $index = 0; foreach my $source (@otherpkg_sources) { push @lines, " xcat-otherpkgs-$index.list:"; diff --git a/xCAT-server/lib/xcat/plugins/debian.pm b/xCAT-server/lib/xcat/plugins/debian.pm index 48a3241a1..a54a8a3b5 100644 --- a/xCAT-server/lib/xcat/plugins/debian.pm +++ b/xCAT-server/lib/xcat/plugins/debian.pm @@ -507,6 +507,121 @@ sub copycd } } +#------------------------------------------------------------------------------- + +=head3 subiquity_nfsroot_server + + Resolve the install server to the address casper's klibc nfsmount needs. + + '!myipfn!' is a placeholder, not a name: pxe.pm and grub2.pm substitute it with + my_ip_facing($node) when they write the boot config, so it is already an address by the + time klibc sees it. Resolving it here would only ever fail, and failing would drop a node + whose noderes.xcatmaster is simply unset -- which noderes.5.rst documents as supported. + anaconda.pm and sles.pm guard the same placeholder the same way. + + Arguments: + $instserver the install server name, address, or the '!myipfn!' placeholder + $resolver optional coderef, for tests; defaults to NetworkUtils::getipaddr, IPv4 only + Returns: + the value to put in nfsroot, or undef when a real name does not resolve + +=cut + +#------------------------------------------------------------------------------- +sub subiquity_nfsroot_server { + my ($instserver, $resolver) = @_; + + return undef unless defined($instserver) && length($instserver); + return $instserver if $instserver eq '!myipfn!'; + + # A dual-stack management node also has an AAAA record. casper takes everything after the + # first colon in nfsroot= as the path, so an IPv6 address there cannot be parsed. + $resolver ||= sub { xCAT::NetworkUtils->getipaddr($_[0], OnlyV4 => 1) }; + return $resolver->($instserver); +} + +#------------------------------------------------------------------------------- + +=head3 subiquity_kcmdline + + Build the kernel command line for a Subiquity (Ubuntu live installer) diskful install. + + boot=casper: without it casper never processes netboot=nfs -- it scans the local disks, + finds no live media and panics into the initramfs shell, which PXE-loops. + + nfsroot must be a literal IP: casper mounts the live filesystem with klibc's nfsmount, + which has no resolver. The ds= URL is fetched later by cloud-init, where DNS works, so it + keeps the install server's name. + + toram: casper copies the squashfs to RAM and unmounts the NFS source, so nothing holds the + network root at shutdown. Without it a process doing I/O to it blocks uninterruptibly, + systemd-shutdown waits forever and the node never reboots into the disk it just installed. + casper parses only nfsroot= and takes the whole value as the path, so mount options cannot + be appended; toram is its supported alternative. It does mean the node needs memory for the + live filesystem on top of the installer -- roughly 1.5G on 24.04 -- or it dies part-way + through and reboots into the installer again, which looks like a boot-flip failure. See + docs/source/troubleshooting/os_installation/ubuntu_subiquity_memory.rst. + + Arguments: + $base the command line built so far + $nfsip the install server as a literal IP, for casper's klibc nfsmount + $pkgdir the install media path exported over NFS + $instserver the install server name, for the cloud-init seed URL + $httpport the xCAT HTTP port + $node the node being installed + Returns: + the completed command line + +=cut + +#------------------------------------------------------------------------------- +sub subiquity_kcmdline { + my ($base, $nfsip, $pkgdir, $instserver, $httpport, $node) = @_; + + my $kcmdline = $base; + $kcmdline .= " autoinstall ip=dhcp boot=casper netboot=nfs nfsroot=${nfsip}:${pkgdir} toram"; + $kcmdline .= " ds=nocloud-net;s=http://${instserver}:${httpport}/install/autoinst/${node}/"; + $kcmdline .= " ---"; + + return $kcmdline; +} + +#------------------------------------------------------------------------------- + +=head3 subiquity_boot_params + + Resolve the install server and build the Subiquity command line, or say why not. + + The two steps are composed here rather than in mkinstall so the composition can be driven: + mkinstall needs a management node, and the decision that matters -- which install server + ends up in nfsroot -- is exactly what a regression would change. The caller keeps the side + effects: reporting the error and skipping the node. + + Arguments: + $base the command line built so far + $instserver the install server name, address, or the '!myipfn!' placeholder + $pkgdir the install media path exported over NFS + $httpport the xCAT HTTP port + $node the node being installed + $resolver optional coderef, for tests; passed through to subiquity_nfsroot_server + Returns: + ($kcmdline, undef) on success, or (undef, $message) when the server does not resolve + +=cut + +#------------------------------------------------------------------------------- +sub subiquity_boot_params { + my ($base, $instserver, $pkgdir, $httpport, $node, $resolver) = @_; + + my $nfsip = subiquity_nfsroot_server($instserver, $resolver); + return (undef, "Could not resolve the install server '$instserver' to an address. " + . "The Ubuntu live installer mounts its root with klibc nfsmount, which cannot " + . "resolve names, so nfsroot must be an address.") + unless $nfsip; + + return (subiquity_kcmdline($base, $nfsip, $pkgdir, $instserver, $httpport, $node), undef); +} + sub mkinstall { xCAT::MsgUtils->message("S", "Doing debian mkinstall"); my $request = shift; @@ -986,9 +1101,16 @@ sub mkinstall { my $kcmdline = "nofb utf8 auto xcatd=" . $instserver; if (using_subiquity($os,$tmplfile)) { - $kcmdline .= " autoinstall ip=dhcp netboot=nfs nfsroot=${instserver}:${pkgdir}"; - $kcmdline .= " ds=nocloud-net;s=http://${instserver}:${httpport}/install/autoinst/${node}/"; - $kcmdline .= " ---"; + # Fail rather than hand casper a name: klibc's nfsmount cannot resolve one, so + # the node would panic "can't parse IP address" at boot, on the node, with + # nothing said on the management node. + my ($subiquity_cmdline, $subiquity_error) = + subiquity_boot_params($kcmdline, $instserver, $pkgdir, $httpport, $node); + if ($subiquity_error) { + xCAT::MsgUtils->report_node_error($callback, $node, $subiquity_error); + next; + } + $kcmdline = $subiquity_cmdline; } else { $kcmdline .= " url=http://${instserver}:$httpport/install/autoinst/$node"; $kcmdline .= " mirror/http/hostname=${instserver}:$httpport"; diff --git a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl index 8399cef6c..5918a61e3 100644 --- a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl +++ b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl @@ -66,8 +66,19 @@ autoinstall: sed -i '/^\.\.\.$/d' /autoinstall.yaml cat /tmp/partitionfile >> /autoinstall.yaml echo "=== DNS setup ===" - rm -f /etc/resolv.conf - echo "nameserver #TABLE:noderes:$NODE:xcatmaster#" >/etc/resolv.conf + # glibc's resolver discards a nameserver line naming a host, so writing the xcatmaster + # *name* leaves the installer -- and the in-target apt that inherits this file -- with no + # DNS. Resolve it here, while the live installer's DHCP resolv.conf still works, and keep + # that file when the name does not resolve. A name written back is the case this step + # exists to prevent. + xcatmaster_host="#TABLE:noderes:$NODE:xcatmaster#" + xcatmaster_ip="$(getent ahostsv4 "$xcatmaster_host" | awk '{print $1; exit}')" + if [ -n "$xcatmaster_ip" ]; then + rm -f /etc/resolv.conf + echo "nameserver $xcatmaster_ip" >/etc/resolv.conf + else + echo "xcat: $xcatmaster_host has no IPv4 address; keeping the resolver DHCP supplied" + fi echo "domain #TABLE:site:key=domain:value#" >>/etc/resolv.conf echo "=== early-commands complete ===" late-commands: @@ -100,5 +111,14 @@ autoinstall: cp ./#HOSTNAME#.post /target/root/post.script; curtin in-target --target /target /root/post.script; } >>/target/var/log/xcat/xcat.log 2>&1' + # Flip the node to local-disk boot, or it PXE-loops back into the installer on reboot. + # xcatd's install monitor greets with "ready", then answers "next" with "done" and runs + # "nodeset next". Require both tokens: another service on that port is not a flipped + # node. The monitor listens on site.xcatiport; TABLEBLANKOKAY because an absent key fails a + # plain TABLE lookup, and with it the whole template. + # Retry, and log a failure rather than reinstall silently. Both reads are bounded: a monitor + # that accepts and never answers would block a bare read and hang the install here. A + # monitor that died and never came back is #7759. + - ['bash', '-c', 'xm=#XCATVAR:XCATMASTER#; port="#TABLEBLANKOKAY:site:key=xcatiport:value#"; [ -n "$port" ] || port=3002; ok=0; for i in 1 2 3 4 5; do if exec 3<>/dev/tcp/$xm/$port; then if read -r -t 10 hello <&3 && [ "$hello" = "ready" ]; then printf "next\n" >&3; if read -r -t 10 ack <&3 && [ "$ack" = "done" ]; then ok=1; fi; fi; exec 3>&- 3<&-; [ "$ok" = 1 ] && break; fi; sleep 5; done; if [ "$ok" != 1 ]; then echo "xcat: FAILED to flip $(hostname) to local-disk boot via $xm:$port; the node will PXE back into the installer" >>/target/var/log/xcat/xcat.log; fi; exit 0'] error-commands: - tar -c --ignore-failed-read --transform='s/^/#HOSTNAME#-logs\//' /var/crash /var/log/installer /tmp/pre-install.log /autoinstall.yaml 2>/dev/null |nc -l 8080 diff --git a/xCAT-test/autotest/testcase/commoncmd/retry_install.sh b/xCAT-test/autotest/testcase/commoncmd/retry_install.sh index 9f4c9e24e..d4589e673 100755 --- a/xCAT-test/autotest/testcase/commoncmd/retry_install.sh +++ b/xCAT-test/autotest/testcase/commoncmd/retry_install.sh @@ -7,7 +7,7 @@ node=$1 osimage=$2 vmhost=`lsdef $node -i vmhost -c | cut -d '=' -f 2` times=2 -wait_for_provision=20 #Min to wait for node to provision +wait_for_provision=${WAIT_FOR_PROVISION:-20} #Min to wait for node to provision (overridable via WAIT_FOR_PROVISION; kept short so a boot-loop fails fast, raised only where the happy path is slow -- e.g. Ubuntu subiquity diskful) check_status=10 #Sec to keep checking status iterations=$wait_for_provision*60/$check_status #Iterations to check for "booted" status diff --git a/xCAT-test/unit/debian_mkinstall_subiquity_branch.t b/xCAT-test/unit/debian_mkinstall_subiquity_branch.t new file mode 100644 index 000000000..a40ee8297 --- /dev/null +++ b/xCAT-test/unit/debian_mkinstall_subiquity_branch.t @@ -0,0 +1,130 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use FindBin; +use Test::More; + +# Regression: the helpers were covered, the CALL SITE was not. +# +# subiquity_boot_params and subiquity_nfsroot_server each have tests, but putting mkinstall's +# pre-fix branch back -- getipaddr plus the inline command line -- left the entire unit suite +# green. Argument order, the error branch, and the `next` that skips the node were all +# unobservable, which is exactly where the bug this PR fixes lived. +# +# mkinstall needs a management node and a database, so the branch is lifted out and eval'd into +# a scratch package with report_node_error stubbed, and driven inside a real loop so the `next` +# it performs is the `next` under test. + +my $repo_root = File::Spec->rel2abs( + File::Spec->catdir( $FindBin::Bin, '..', '..' ) +); +my $plugin = File::Spec->catfile( + $repo_root, 'xCAT-server', 'lib', 'xcat', 'plugins', 'debian.pm' +); +plan skip_all => "debian.pm not found" unless -f $plugin; + +my $src = do { local $/; open my $fh, '<', $plugin or die $!; <$fh> }; + +# The helpers the branch calls, plus the branch itself. BAIL_OUT rather than skip, so a rename +# fails loudly instead of silently covering nothing. +my $helpers = ''; +for my $name (qw(subiquity_nfsroot_server subiquity_kcmdline subiquity_boot_params)) { + my ($sub) = $src =~ /\n(sub \Q$name\E \{.*?\n\})\n/s; + BAIL_OUT("could not extract $name from debian.pm") unless defined $sub; + $helpers .= "$sub\n"; +} + +# There are several `if (using_subiquity(...))` in debian.pm; take the one that actually builds +# the boot parameters, selected by what it contains rather than by where it sits, so adding +# another above it does not silently swap which branch is under test. +my @candidates = $src =~ /\n[ ]+if \(using_subiquity\([^)]*\)\) \{\n(.*?)\n[ ]+\} else \{\n/gs; +my @wanted = grep { /subiquity_boot_params\(/ } @candidates; +BAIL_OUT('could not find the mkinstall branch that builds the subiquity boot parameters') + unless @wanted == 1; +my $branch = $wanted[0]; +BAIL_OUT('the extracted branch does not skip the node on error') + unless $branch =~ /\bnext\b/; +BAIL_OUT('the extracted branch is implausibly large -- the match ran past its block') + if ($branch =~ tr/\n//) > 20; + +my @reported; + +# The branch calls subiquity_boot_params with no resolver, exactly as production does, so the +# fallback to xCAT::NetworkUtils->getipaddr is the seam to stand in at. That keeps the call path +# under test identical to the real one -- nothing is injected into it. +{ + package xCAT::NetworkUtils; + sub getipaddr { + my (undef, $name) = @_; + return $name if defined($name) && $name =~ /^\d+\.\d+\.\d+\.\d+$/; + return '10.0.0.1' if defined($name) && $name eq 'mn.cluster'; + return undef; + } +} + +{ + package xCAT::MsgUtils; + sub report_node_error { shift; my ($cb, $node, $msg) = @_; push @reported, [ $node, $msg ]; } +} + +my $driver = <<"CODE"; +$helpers +sub drive { + my (\$kcmdline, \$instserver, \$pkgdir, \$httpport, \$node) = \@_; + my \$callback; + my \$result; + NODE: foreach my \$n (\$node) { +$branch + \$result = \$kcmdline; + } + return \$result; +} +CODE + +# `next` inside the branch belongs to the loop the driver wraps around it. +$driver =~ s/\bnext;/next NODE;/g; + +{ + package T; + eval "$driver; 1" or main::BAIL_OUT("could not eval the mkinstall branch: $@"); +} + +# A resolvable install server: the branch must produce a live command line. +{ + @reported = (); + my $out = T::drive( 'nofb utf8 auto xcatd=10.0.0.1', '10.0.0.1', + '/install/ubuntu24.04/x86_64', '80', 'cn1' ); + ok( defined $out, 'a resolvable install server yields a command line' ); + like( $out, qr/boot=casper/, 'the branch puts boot=casper on the command line' ); + like( $out, qr/\btoram\b/, 'and toram' ); + like( $out, qr{nfsroot=10\.0\.0\.1:/install/ubuntu24\.04/x86_64}, + 'and nfsroot as a literal address, in the media path' ); + is( scalar @reported, 0, 'and nothing is reported as an error' ); +} + +# The placeholder: this is the case the fix restored, driven through the call site rather than +# through the helper. +{ + @reported = (); + my $out = T::drive( 'nofb utf8 auto xcatd=!myipfn!', '!myipfn!', + '/install/ubuntu24.04/x86_64', '80', 'cn1' ); + ok( defined $out, 'a node with no xcatmaster is not skipped' ); + like( $out, qr/nfsroot=!myipfn!:/, + 'and the placeholder reaches the boot config for pxe.pm/grub2.pm to substitute' ); + is( scalar @reported, 0, 'and no error is reported for it' ); +} + +# An install server that does not resolve must skip the node, not emit a command line naming it. +{ + @reported = (); + my $out = T::drive( 'nofb utf8 auto xcatd=nosuchhost', 'nosuchhost', + '/install/ubuntu24.04/x86_64', '80', 'cn1' ); + ok( !defined $out, 'an unresolvable install server skips the node' ); + is( scalar @reported, 1, 'and reports exactly one error' ); + like( $reported[0][1], qr/nosuchhost/, 'naming the server that failed' ); + is( $reported[0][0], 'cn1', 'and the node it happened on' ); +} + +done_testing(); diff --git a/xCAT-test/unit/debian_subiquity_boot_params.t b/xCAT-test/unit/debian_subiquity_boot_params.t new file mode 100644 index 000000000..c22430565 --- /dev/null +++ b/xCAT-test/unit/debian_subiquity_boot_params.t @@ -0,0 +1,114 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use FindBin; +use Test::More; + +# Regression: the helpers were covered and the call site was not. +# +# subiquity_nfsroot_server and subiquity_kcmdline each had tests, but reverting the CALL SITE in +# mkinstall -- putting `xCAT::NetworkUtils->getipaddr($instserver)` back in place of +# subiquity_nfsroot_server, i.e. the exact regression the fix removes -- left the whole unit +# suite green. A helper can be perfectly covered while nothing connects it to production. +# +# mkinstall needs a management node, so the composition it performs (resolve the install server, +# then build the command line, or explain why not) lives in subiquity_boot_params, which takes +# its inputs and returns an answer. The caller keeps report_node_error and the loop's `next`. + +my $repo_root = File::Spec->rel2abs( + File::Spec->catdir( $FindBin::Bin, '..', '..' ) +); +my $plugin = File::Spec->catfile( + $repo_root, 'xCAT-server', 'lib', 'xcat', 'plugins', 'debian.pm' +); +plan skip_all => "debian.pm not found" unless -f $plugin; + +my $src = do { local $/; open my $fh, '<', $plugin or die $!; <$fh> }; + +# BAIL_OUT rather than skip, so a rename fails loudly instead of silently covering nothing. +my $body = ''; +for my $name (qw(subiquity_nfsroot_server subiquity_kcmdline subiquity_boot_params)) { + my ($sub) = $src =~ /\n(sub \Q$name\E \{.*?\n\})\n/s; + BAIL_OUT("could not extract $name from debian.pm") unless defined $sub; + $body .= "$sub\n"; +} + +# Stand in for the module the routine falls back to when no resolver is injected. Without this +# a call site that bypassed the injected resolver would die on a missing module -- a red, but +# for the wrong reason. With it, bypassing the resolver produces the WRONG ANSWER instead of an +# exception, which is what the assertions below are meant to catch. +{ + package xCAT::NetworkUtils; + sub getipaddr { return undef } +} + +{ + package T; + eval "$body; 1" or main::BAIL_OUT("could not eval the subiquity helpers: $@"); +} + +my $resolver = sub { $_[0] eq 'mn.cluster' ? '10.0.0.1' : undef }; + +# The regression this exists to catch: an unset noderes.xcatmaster gives '!myipfn!', and the +# boot config must still be produced with the placeholder intact, because pxe.pm and grub2.pm +# substitute it with an address when they write the config. +{ + my ($kcmdline, $err) = T::subiquity_boot_params( + 'nofb utf8 auto xcatd=!myipfn!', '!myipfn!', + '/install/ubuntu24.04/x86_64', '80', 'cn1', $resolver ); + ok( !defined $err, 'a node with no xcatmaster still gets a boot config' ) + or diag("error was: $err"); + like( $kcmdline, qr/nfsroot=!myipfn!:/, + 'and nfsroot keeps the placeholder for pxe.pm/grub2.pm to substitute' ); + like( $kcmdline, qr/boot=casper/, 'the casper boot flag survives the composition' ); +} + +# A resolvable name still becomes an address, because klibc's nfsmount has no resolver. +{ + my ($kcmdline, $err) = T::subiquity_boot_params( + 'nofb utf8 auto xcatd=mn.cluster', 'mn.cluster', + '/install/ubuntu24.04/x86_64', '80', 'cn1', $resolver ); + ok( !defined $err, 'a resolvable install server produces a boot config' ); + like( $kcmdline, qr{nfsroot=10\.0\.0\.1:/install/ubuntu24\.04/x86_64}, + 'and nfsroot carries the address, not the name' ); + like( $kcmdline, qr{ds=nocloud-net;s=http://mn\.cluster:80/}, + 'while the cloud-init seed URL keeps the name, where DNS works' ); +} + +# A dual-stack management node has both an A and an AAAA record, and getipaddr answers with +# whichever the resolver returns first. casper mounts the live filesystem with klibc's nfsmount, +# which takes an IPv4 address, and it takes everything after the first colon in nfsroot= as the +# path -- so an IPv6 address there is not a slower path, it is an unparseable one. This install +# asks for ip=dhcp, so the address must be IPv4. +{ + no warnings 'redefine'; + local *xCAT::NetworkUtils::getipaddr = sub { + my ($class, $host, %opt) = @_; + return $opt{OnlyV4} ? '10.0.0.1' : '2001:db8::1'; + }; + + my ($kcmdline, $err) = T::subiquity_boot_params( + 'nofb utf8 auto xcatd=mn.cluster', 'mn.cluster', + '/install/ubuntu24.04/x86_64', '80', 'cn1' ); + ok( !defined $err, 'a dual-stack install server produces a boot config' ) + or diag("error was: $err"); + like( $kcmdline, qr{nfsroot=10\.0\.0\.1:/install/ubuntu24\.04/x86_64}, + 'and nfsroot carries the IPv4 address of the install server' ); + unlike( $kcmdline, qr/nfsroot=[^ ]*::/, + 'never the IPv6 address, which klibc nfsmount cannot use' ); +} + +# The guard the original commit added must survive: a name that does not resolve is an error, +# not a command line with a name in nfsroot. +{ + my ($kcmdline, $err) = T::subiquity_boot_params( + 'nofb utf8 auto xcatd=nosuchhost', 'nosuchhost', + '/install/ubuntu24.04/x86_64', '80', 'cn1', $resolver ); + ok( !defined $kcmdline, 'an unresolvable install server yields no boot config' ); + like( $err, qr/nosuchhost/, 'and the error names the server that failed' ); + like( $err, qr/klibc|nfsmount|address/i, 'and says why it matters' ); +} + +done_testing(); diff --git a/xCAT-test/unit/debian_subiquity_netboot.t b/xCAT-test/unit/debian_subiquity_netboot.t new file mode 100644 index 000000000..0f512cd0d --- /dev/null +++ b/xCAT-test/unit/debian_subiquity_netboot.t @@ -0,0 +1,74 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use FindBin; +use Test::More; + +# The Ubuntu live installer (Subiquity) is booted over NFS. Three things on the kernel command +# line decide whether it boots at all and whether the node can reboot into the disk afterwards: +# +# boot=casper without it casper never processes netboot=nfs -- it scans local disks, finds no +# live media and panics into the initramfs shell, which PXE-loops. +# nfsroot=IP casper mounts the live filesystem with klibc's nfsmount, which has no resolver, +# so a hostname there fails with "can't parse IP address". +# toram casper copies the squashfs to RAM and unmounts the NFS source. Without it the +# NFS root stays mounted, systemd-shutdown blocks forever on I/O to it and the +# node never power-cycles into the disk it just installed. +# +# Build the command line for real and inspect it, rather than reading the source that builds it. + +use lib "$FindBin::Bin/../../perl-xCAT"; +use lib "$FindBin::Bin/../../xCAT-server/lib/perl"; +my $plugin = "$FindBin::Bin/../../xCAT-server/lib/xcat/plugins/debian.pm"; +plan skip_all => 'debian.pm not found' unless -r $plugin; +eval { require $plugin; 1 } or plan skip_all => "could not load debian.pm: $@"; + +my $cmdline = xCAT_plugin::debian::subiquity_kcmdline( + 'nofb utf8 auto xcatd=xcatmn', # what mkinstall has built so far + '10.0.0.1', # instserver resolved to a literal IP + '/install/ubuntu24.04/x86_64', # pkgdir exported over NFS + 'xcatmn', # instserver by name + '80', # httpport + 'node01', # node +); + +# --- the three settings the installer cannot boot without ------------------ +like($cmdline, qr/(?:^| )boot=casper(?: |$)/, + 'casper is told to boot, so it processes netboot=nfs instead of scanning disks'); +like($cmdline, qr{(?:^| )nfsroot=10\.0\.0\.1:/install/ubuntu24\.04/x86_64(?: |$)}, + 'nfsroot names the install server by IP, which klibc nfsmount can parse'); +like($cmdline, qr/(?:^| )toram(?: |$)/, + 'toram copies the live filesystem to RAM so the NFS root is unmounted before shutdown'); + +unlike($cmdline, qr/nfsroot=xcatmn:/, + 'nfsroot never carries a hostname, which klibc nfsmount cannot resolve'); +unlike($cmdline, qr/nfsroot=[^ ]*,/, + 'no mount options are appended to nfsroot -- casper takes the whole value as the path'); + +# --- the rest of the line --------------------------------------------------- +like($cmdline, qr/(?:^| )autoinstall(?: |$)/, 'the installer runs unattended'); +like($cmdline, qr/(?:^| )ip=dhcp(?: |$)/, 'the live system configures its NIC by DHCP'); +like($cmdline, qr/(?:^| )netboot=nfs(?: |$)/, 'the live filesystem is fetched over NFS'); + +# cloud-init fetches the seed later, in the booted live system, where DNS works -- so this one +# keeps the install server's name rather than its address. +like($cmdline, qr{(?:^| )ds=nocloud-net;s=http://xcatmn:80/install/autoinst/node01/(?: |$)}, + 'the cloud-init seed URL addresses the install server by name'); + +is((split / /, $cmdline)[-1], '---', + 'the line ends with the separator that divides installer arguments from kernel arguments'); + +# What mkinstall had already built is preserved, not replaced. +like($cmdline, qr/^nofb utf8 auto xcatd=xcatmn /, + 'the command line built so far is kept ahead of the installer arguments'); + +# A non-default HTTP port reaches the seed URL. +{ + my $alt = xCAT_plugin::debian::subiquity_kcmdline( + 'base', '10.0.0.1', '/pkgdir', 'xcatmn', '8080', 'node02'); + like($alt, qr{ds=nocloud-net;s=http://xcatmn:8080/install/autoinst/node02/}, + 'the seed URL carries the configured HTTP port and node'); +} + +done_testing(); diff --git a/xCAT-test/unit/debian_subiquity_nfsroot_server.t b/xCAT-test/unit/debian_subiquity_nfsroot_server.t new file mode 100644 index 000000000..23d93a3ad --- /dev/null +++ b/xCAT-test/unit/debian_subiquity_nfsroot_server.t @@ -0,0 +1,81 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use FindBin; +use Test::More; + +# Regression: nodes whose noderes.xcatmaster is unset stopped getting a boot config at all. +# +# When xcatmaster is unset, debian.pm sets $instserver to the literal '!myipfn!'. That is a +# placeholder, not a hostname: pxe.pm:176 and grub2.pm:129 substitute it with +# my_ip_facing($node) -- an address -- when they write the boot config. Resolving it as a name +# returns undef, so the subiquity path reported "Could not resolve the install server" and +# `next`ed past the node, where before it had produced exactly the numeric nfsroot the fix +# wants. anaconda.pm and sles.pm both guard the same placeholder with +# `unless ($instserver eq '!myipfn!')`; the Ubuntu path did not. +# +# noderes.5.rst:125 documents an unset xcatmaster as supported, and the CI confs all set it +# (reg_linux_diskfull_installation_flat chdefs xcatmaster=$$MN), which is why no pipeline +# would catch this. +# +# The decision lives in its own routine so it can be driven with an injected resolver instead +# of a management node. + +my $repo_root = File::Spec->rel2abs( + File::Spec->catdir( $FindBin::Bin, '..', '..' ) +); +my $plugin = File::Spec->catfile( + $repo_root, 'xCAT-server', 'lib', 'xcat', 'plugins', 'debian.pm' +); +plan skip_all => "debian.pm not found" unless -f $plugin; + +my $src = do { local $/; open my $fh, '<', $plugin or die $!; <$fh> }; + +# Lift the routine into a scratch package: debian.pm itself needs a management node to load. +# BAIL_OUT rather than skip, so a rename fails loudly instead of silently covering nothing. +my ($body) = $src =~ /\n(sub subiquity_nfsroot_server \{.*?\n\})\n/s; +BAIL_OUT('could not extract subiquity_nfsroot_server from debian.pm') + unless defined $body; + +{ + package T; + eval "$body; 1" or main::BAIL_OUT("could not eval subiquity_nfsroot_server: $@"); +} + +# A resolver that records what it was asked, so "never asked" is checkable. +my @asked; +my $resolver = sub { push @asked, $_[0]; return $_[0] eq 'mn.cluster' ? '10.0.0.1' : undef }; + +# The bug: the placeholder must survive to the boot config, not be resolved. +@asked = (); +is( T::subiquity_nfsroot_server( '!myipfn!', $resolver ), '!myipfn!', + 'the !myipfn! placeholder is passed through untouched' ); +is_deeply( \@asked, [], + 'and the resolver is never asked to resolve it' ); + +# A name that resolves still resolves. +@asked = (); +is( T::subiquity_nfsroot_server( 'mn.cluster', $resolver ), '10.0.0.1', + 'a resolvable install server name becomes its address' ); +is_deeply( \@asked, ['mn.cluster'], 'by asking the resolver' ); + +# An address passes through the resolver unchanged, as getipaddr does. +is( T::subiquity_nfsroot_server( '10.0.0.1', sub { $_[0] } ), '10.0.0.1', + 'an address is returned as itself' ); + +# The guard that motivated the original commit must survive: a name that does not resolve is +# still a failure, because klibc's nfsmount cannot resolve it either. +@asked = (); +ok( !defined T::subiquity_nfsroot_server( 'nosuchhost', $resolver ), + 'a name that does not resolve is still rejected' ); +is_deeply( \@asked, ['nosuchhost'], 'after actually trying to resolve it' ); + +# Degenerate inputs are rejected rather than passed to the resolver. +ok( !defined T::subiquity_nfsroot_server( undef, $resolver ), + 'an undefined install server is rejected' ); +ok( !defined T::subiquity_nfsroot_server( '', $resolver ), + 'an empty install server is rejected' ); + +done_testing(); diff --git a/xCAT-test/unit/networkutils_getipaddr_onlyv4.t b/xCAT-test/unit/networkutils_getipaddr_onlyv4.t new file mode 100644 index 000000000..4e64315c6 --- /dev/null +++ b/xCAT-test/unit/networkutils_getipaddr_onlyv4.t @@ -0,0 +1,47 @@ +#!/usr/bin/env perl +# +# getipaddr caches a resolved address in %::hostiphash and answers later calls +# from it. The cache bypass tests OnlyV6 and GetAllAddresses, and does not test +# OnlyV4, so a caller that asks for IPv4 can be handed a cached IPv6 address. +# +# The cache is filled by whichever lookup ran first. An unrestricted lookup +# passes AF_UNSPEC to getaddrinfo, and on a dual-stack management node with an +# AAAA record that answers with the IPv6 address, which is then stored. xcatd is +# long-lived and %::hostiphash is a global, so any earlier caller in the process +# poisons every OnlyV4 caller that follows. +# +# What it costs: debian.pm builds the Subiquity install command line with +# getipaddr($host, OnlyV4 => 1) and writes nfsroot=
:/install. With an +# IPv6 address that renders as nfsroot=2001:db8::1:/install, which is not a +# parseable nfsroot, so the installer never mounts and the node never installs. +# dhcp.pm and mknb.pm carry four more OnlyV4 callers with the same exposure. + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::Bin/../../perl-xCAT"; +use Test::More; + +use xCAT::NetworkUtils; + +# The address an unrestricted lookup left behind on a dual-stack host. +$::hostiphash{'mn.cluster'}{hostip} = '2001:db8::1'; + +my $only_v4 = xCAT::NetworkUtils->getipaddr('mn.cluster', OnlyV4 => 1); + +ok(!defined($only_v4) || $only_v4 !~ /:/, + 'OnlyV4 does not return the IPv6 address an earlier lookup cached') + or diag("getipaddr returned '$only_v4', which renders as nfsroot=$only_v4:/install"); + +# An IPv4 entry must still be served from the cache: the bypass is about the +# family of the cached answer, not about disabling the cache for OnlyV4. +$::hostiphash{'v4.cluster'}{hostip} = '10.1.2.3'; +is(xCAT::NetworkUtils->getipaddr('v4.cluster', OnlyV4 => 1), '10.1.2.3', + 'an IPv4 cache entry is still served to an OnlyV4 caller'); + +# An unrestricted caller keeps its cache hit, whatever family it holds. +is(xCAT::NetworkUtils->getipaddr('mn.cluster'), '2001:db8::1', + 'a caller that did not ask for IPv4 still gets the cached address'); + +done_testing(); diff --git a/xCAT-test/unit/postage_makescript_defer_call_site.t b/xCAT-test/unit/postage_makescript_defer_call_site.t new file mode 100644 index 000000000..902585ded --- /dev/null +++ b/xCAT-test/unit/postage_makescript_defer_call_site.t @@ -0,0 +1,122 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use FindBin; +use Test::More; + +# Regression: makescript's call to defer_syncfiles_to_postboot was covered by nothing. +# +# The helper itself has tests, but deleting the whole block out of makescript -- the +# provmethod override AND the call -- left the entire unit suite green. That is how a wrong +# deletion shipped once already: the override was removed on the mistaken grounds that +# %image_hash never carries a provmethod, and no test noticed. +# +# makescript needs a management node and a database, so the block is lifted out and eval'd into +# a scratch package, driven with the hash makescript actually builds. What is under test is the +# resolution -- an osimage NAME becoming the osimage's real provmethod -- and that the result is +# what reaches the helper. + +my $repo_root = File::Spec->rel2abs( + File::Spec->catdir( $FindBin::Bin, '..', '..' ) +); +my $postage = File::Spec->catfile( + $repo_root, 'xCAT-server', 'lib', 'perl', 'xCAT', 'Postage.pm' +); +plan skip_all => "Postage.pm not found" unless -f $postage; + +my $src = do { local $/; open my $fh, '<', $postage or die $!; <$fh> }; + +# The helper, and the block in makescript that calls it. BAIL_OUT rather than skip, so a rename +# fails loudly instead of silently covering nothing. +my ($helper) = $src =~ /\n(sub defer_syncfiles_to_postboot \{.*?\n\})\n/s; +BAIL_OUT('could not extract defer_syncfiles_to_postboot from Postage.pm') unless defined $helper; + +my ($block) = $src =~ /\n([ ]+my \$effective_provmethod = \$provmethod;\n.*?defer_syncfiles_to_postboot\(\n.*?\);)\n/s; +BAIL_OUT('could not extract the makescript call site from Postage.pm') unless defined $block; +BAIL_OUT('the extracted call site does not consult the image hash') + unless $block =~ /\$image_hash\{\$osimgname\}\{'provmethod'\}/; + +my $driver = <<"CODE"; +$helper + +sub drive { + my (\%a) = \@_; + my \$provmethod = \$a{provmethod}; + my \$osimgname = \$a{osimgname}; + my \$os = \$a{os}; + my \$nodesetstate = \$a{nodesetstate}; + my \$postscripts = \$a{postscripts}; + my \$postbootscripts = \$a{postbootscripts}; + my \%image_hash = \%{ \$a{image_hash} || {} }; +$block + return (\$postscripts, \$postbootscripts, \$effective_provmethod); +} +CODE + +{ + package T; + eval "$driver; 1" or main::BAIL_OUT("could not eval the makescript call site: $@"); +} + +my $POST = "otherpkgs\nsyncfiles\nremoteshell\n"; + +# nodetype.provmethod naming an osimage: getScripts() fills %image_hash from the osimage table, +# so the override is what turns that name into the real provmethod. +{ + my ($post, $postboot, $eff) = T::drive( + os => 'ubuntu24.04', nodesetstate => undef, + provmethod => 'ubuntu24.04-x86_64-install-compute', + osimgname => 'ubuntu24.04-x86_64-install-compute', + image_hash => { 'ubuntu24.04-x86_64-install-compute' => { provmethod => 'install' } }, + postscripts => $POST, postbootscripts => '' ); + is( $eff, 'install', 'an osimage name resolves to the osimage provmethod' ); + unlike( $post, qr/^syncfiles$/m, 'so syncfiles leaves the in-target postscripts' ); + like( $postboot, qr/^syncfiles$/m, 'and is deferred to the postboot scripts' ); +} + +# Without the resolution there is nothing to key off when nodesetstate is absent. +{ + my ($post, $postboot, $eff) = T::drive( + os => 'ubuntu24.04', nodesetstate => undef, + provmethod => 'ubuntu24.04-x86_64-install-compute', + osimgname => 'ubuntu24.04-x86_64-install-compute', + image_hash => {}, + postscripts => $POST, postbootscripts => '' ); + is( $eff, 'ubuntu24.04-x86_64-install-compute', + 'an osimage with no row in the hash leaves the provmethod alone' ); + like( $post, qr/^syncfiles$/m, 'and nothing is deferred' ); +} + +# A literal provmethod is passed straight through -- $osimgname is undef in that case. +{ + my ($post, $postboot, $eff) = T::drive( + os => 'ubuntu24.04', nodesetstate => undef, provmethod => 'install', + osimgname => undef, image_hash => {}, + postscripts => $POST, postbootscripts => '' ); + is( $eff, 'install', 'a literal provmethod is used as-is' ); + unlike( $post, qr/^syncfiles$/m, 'and still defers' ); +} + +# nodesetstate wins when it is set, which is the ordinary nodeset path. +{ + my ($post, $postboot) = T::drive( + os => 'ubuntu24.04', nodesetstate => 'install', + provmethod => 'ubuntu24.04-x86_64-install-compute', + osimgname => 'ubuntu24.04-x86_64-install-compute', + image_hash => {}, postscripts => $POST, postbootscripts => '' ); + unlike( $post, qr/^syncfiles$/m, 'nodesetstate install defers regardless of provmethod' ); + like( $postboot, qr/^syncfiles$/m, 'and syncfiles lands in the postboot scripts' ); +} + +# Not Ubuntu: the block must not touch anything. +{ + my ($post, $postboot) = T::drive( + os => 'rhels9', nodesetstate => 'install', provmethod => 'install', + osimgname => undef, image_hash => {}, postscripts => $POST, postbootscripts => '' ); + like( $post, qr/^syncfiles$/m, 'a non-Debian OS keeps syncfiles where it was' ); + is( $postboot, '', 'and gains no postboot scripts' ); +} + +done_testing(); diff --git a/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t b/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t new file mode 100644 index 000000000..7603c97aa --- /dev/null +++ b/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t @@ -0,0 +1,127 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use FindBin; +use Test::More; + +# On the Ubuntu/Debian DISKFUL install path a node's postscripts run inside the installer's +# in-target chroot, before the node has booted as itself. syncfiles asks the management node to +# scp files INTO the running node, which cannot work there -- no sshd yet -- so it times out and +# the node reports status=failed even though the OS installed fine. Postage defers it to the +# postbootscripts, which run on the booted node. Drive that decision directly. + +my $repo = "$FindBin::Bin/../.."; +plan skip_all => 'Postage.pm not found' unless -r "$repo/xCAT-server/lib/perl/xCAT/Postage.pm"; + +use lib "$FindBin::Bin/../../perl-xCAT"; +use lib "$FindBin::Bin/../../xCAT-server/lib/perl"; +eval { require xCAT::Postage; 1 } or plan skip_all => "could not load xCAT::Postage: $@"; + +my $DEFERRED = "# ubuntu-deferred-postbootscripts-start-here\nsyncfiles\n" + . "# ubuntu-deferred-postbootscripts-end-here\n"; + +sub defer { return xCAT::Postage::defer_syncfiles_to_postboot(@_) } + +# --- the case the fix exists for ------------------------------------------- +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', 'install', "syncfiles\notherpkgs\n", "setupntp\n"); + + is($post, "otherpkgs\n", 'syncfiles is removed from the postscripts'); + is($postboot, $DEFERRED . "setupntp\n", + 'syncfiles is prepended to the postbootscripts, ahead of what consumes its files'); +} + +# The nodeset state is not always known; the osimage provmethod decides then. +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', undef, "syncfiles\n", "setupntp\n"); + is($post, '', 'provmethod=install defers when no nodeset state is given'); + is($postboot, $DEFERRED . "setupntp\n", 'and the postbootscripts receive it'); +} + +# --- paths that must NOT change -------------------------------------------- +my @untouched = ( + [ 'netboot keeps its postscripts on the booted node already', + 'ubuntu24.04', 'netboot', 'netboot' ], + [ 'statelite is unaffected', + 'ubuntu22.04', 'statelite', 'statelite' ], + [ 'EL runs postscripts on the booted node, so nothing moves', + 'rhels9.4', 'install', 'install' ], + [ 'SLES is unaffected', + 'sles15.6', 'install', 'install' ], + [ 'an undefined os is left alone', + undef, 'install', 'install' ], +); +foreach my $case (@untouched) { + my ($name, $os, $provmethod, $state) = @$case; + my ($post, $postboot) = defer($os, $provmethod, $state, "syncfiles\notherpkgs\n", "setupntp\n"); + is($post, "syncfiles\notherpkgs\n", "$name: the postscripts are unchanged"); + is($postboot, "setupntp\n", "$name: the postbootscripts are unchanged"); +} + +# A node that does not run syncfiles must not gain an empty deferral block. +{ + my ($post, $postboot) = defer('ubuntu24.04', 'install', 'install', "otherpkgs\n", "setupntp\n"); + is($post, "otherpkgs\n", 'a node without syncfiles keeps its postscripts'); + is($postboot, "setupntp\n", 'and gains no deferral block'); +} + +# The name is matched whole: a postscript whose name merely contains "syncfiles" stays put. +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', 'install', "syncfiles2\nmysyncfiles\n", "setupntp\n"); + is($post, "syncfiles2\nmysyncfiles\n", 'a lookalike postscript name is not deferred'); + is($postboot, "setupntp\n", 'and nothing is added to the postbootscripts'); +} + +# Indented entries are still the syncfiles postscript. +{ + my ($post, $postboot) = defer('ubuntu24.04', 'install', 'install', " syncfiles \nfoo\n", undef); + is($post, "foo\n", 'an indented syncfiles entry is deferred'); + is($postboot, $DEFERRED, 'an undefined postbootscripts list becomes the deferral block'); +} + +# A node that already lists syncfiles as its own postbootscript keeps that entry, and does not +# gain a second one -- the install-time copy is still removed, since that is the one that cannot work. +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', 'install', "syncfiles\nfoo\n", "syncfiles\nsetupntp\n"); + is($post, "foo\n", 'the postscripts entry is still removed'); + is($postboot, "syncfiles\nsetupntp\n", + "an admin's own syncfiles postbootscript is left as it was, not duplicated"); +} + +# Rendering the same node twice must not stack a second copy. +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', 'install', "syncfiles\nfoo\n", "setupntp\n"); + my ($post2, $postboot2) = defer('ubuntu24.04', 'install', 'install', $post, $postboot); + is($post2, $post, 'a second pass leaves the postscripts alone'); + is($postboot2, $postboot, 'a second pass does not duplicate syncfiles'); +} + +# The node's provmethod is frequently an osimage NAME rather than the literal 'install' -- that +# is what makescript passes, and resolving it to a real provmethod is not possible there +# (getImage does not store one). nodesetstate is what carries the install signal on that path, +# so the deferral must key off it and not require provmethod to say 'install'. +{ + my ($post, $postboot) = defer( + 'ubuntu24.04', 'ubuntu24.04-x86_64-install-compute', 'install', + "otherpkgs\nsyncfiles\nremoteshell\n", "" ); + unlike( $post, qr/^syncfiles$/m, + 'an osimage-named provmethod still defers when nodesetstate says install' ); + like( $postboot, qr/^syncfiles$/m, + 'and syncfiles lands in the postboot scripts' ); +} + +{ + my ($post, $postboot) = defer( + 'ubuntu24.04', 'ubuntu24.04-x86_64-install-compute', undef, + "otherpkgs\nsyncfiles\nremoteshell\n", "" ); + like( $post, qr/^syncfiles$/m, + 'with no nodesetstate an osimage name is not mistaken for a diskful install' ); +} + +done_testing(); diff --git a/xCAT-test/unit/ubuntu_resolvconf_ip.t b/xCAT-test/unit/ubuntu_resolvconf_ip.t new file mode 100644 index 000000000..a82340add --- /dev/null +++ b/xCAT-test/unit/ubuntu_resolvconf_ip.t @@ -0,0 +1,104 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use FindBin; +use File::Temp qw(tempdir); +use Test::More; + +# A nameserver line in /etc/resolv.conf must hold an IP address: glibc's resolver discards an +# entry naming a host. Writing the xcatmaster *name* left the installer -- and the in-target +# apt-get that inherits the file -- with no usable DNS, so the install hung resolving +# archive.ubuntu.com. The template resolves the name to an address first, and when the name does +# not resolve it keeps the resolver the live installer already got from DHCP, because a name +# written into resolv.conf resolves nothing. +# +# Run the template's own shell for that step and inspect the file it writes. + +my $tmpl = "$FindBin::Bin/../../xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl"; +plan skip_all => 'compute.subiquity.tmpl not found' unless -r $tmpl; + +open(my $fh, '<', $tmpl) or die "open $tmpl: $!"; +my $source = do { local $/; <$fh> }; +close $fh; + +my ($fragment) = $source =~ m{^(\s*xcatmaster_host=.*?\n)\s*echo "=== early-commands complete}ms; +BAIL_OUT('the template does not build /etc/resolv.conf from the xcatmaster') unless $fragment; + +# $NODE, the xcatmaster and the domain come from the xCAT template renderer; stand in for them. +sub write_resolv_conf { + my (%opt) = @_; + my $root = tempdir(CLEANUP => 1); + + # What DHCP left behind in the live installer, which the step either replaces or keeps. + open my $seed, '>', "$root/resolv.conf" or die $!; + print {$seed} "nameserver 192.168.0.53\n"; + close $seed; + + my $script = $fragment; + $script =~ s/\#TABLE:noderes:\$NODE:xcatmaster\#/$opt{xcatmaster}/; + $script =~ s/\#TABLE:site:key=domain:value\#/cluster/; + $script =~ s{/etc/resolv\.conf}{$root/resolv.conf}g; + + # The fragment contains `rm -f /etc/resolv.conf` and this suite runs as root in CI, so a + # rewrite that stops matching would delete the runner's resolver configuration rather than + # fail a test. Sandboxing by rewriting paths is fragile by nature -- respelling the path in + # the template as, say, `etcdir=/etc; rm -f "$etcdir/resolv.conf"` slips straight past the + # substitution above. Refuse to execute anything that still points outside the scratch tree. + # \b not "/etc/": the respelling this guard exists to catch -- `etcdir=/etc; rm -f + # "$etcdir/resolv.conf"` -- has no slash after /etc, so requiring one let it straight past. + if ($script =~ m{(? $? }; + + open my $rh, '<', "$root/resolv.conf" or return { rc => 0, content => '' }; + my $content = do { local $/; <$rh> }; + close $rh; + return { rc => 0, content => $content }; +} + +# --- the case the fix exists for ------------------------------------------- +{ + my $r = write_resolv_conf(xcatmaster => 'xcatmn', resolves => '10.0.0.1'); + like($r->{content}, qr/^nameserver 10\.0\.0\.1$/m, + 'the nameserver line holds the address, which glibc will actually use'); + unlike($r->{content}, qr/nameserver \s+ xcatmn/x, + 'the nameserver line never holds a host name, which glibc discards'); + like($r->{content}, qr/^domain cluster$/m, 'and the search domain is written'); +} + +# --- more than one address: the first is taken ----------------------------- +{ + my $r = write_resolv_conf(xcatmaster => 'xcatmn', resolves => '10.0.0.1'); + my @ns = ($r->{content} =~ /^nameserver (\S+)$/mg); + is_deeply(\@ns, ['10.0.0.1'], 'exactly one IPv4 address is written'); +} + +# --- resolution fails: keep the resolver DHCP gave the live installer ------ +# Writing the name back was the original defect. It leaves the installer with no DNS, and the +# in-target apt inherits the same file. +{ + my $r = write_resolv_conf(xcatmaster => 'xcatmn'); + unlike($r->{content}, qr/^nameserver \s* xcatmn/xm, + 'an unresolvable xcatmaster is never written as a nameserver'); + unlike($r->{content}, qr/^nameserver\s*$/m, + 'no empty nameserver line is written'); + like($r->{content}, qr/^nameserver 192\.168\.0\.53$/m, + 'the resolver DHCP gave the live installer is kept instead'); +} + +# --- an xcatmaster already given as an address is left alone --------------- +{ + my $r = write_resolv_conf(xcatmaster => '10.0.0.1', resolves => '10.0.0.1'); + like($r->{content}, qr/^nameserver 10\.0\.0\.1$/m, + 'an address-valued xcatmaster is written as is'); +} + +done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_apt_sources.t b/xCAT-test/unit/ubuntu_subiquity_apt_sources.t new file mode 100644 index 000000000..0c55d0e8c --- /dev/null +++ b/xCAT-test/unit/ubuntu_subiquity_apt_sources.t @@ -0,0 +1,60 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::Bin/../../perl-xCAT"; +use lib "$FindBin::Bin/../../xCAT-server/lib/perl"; + +use Test::More; + +# An online Subiquity install of a classic-sources release needs the online archive added, or +# in-target apt cannot find chrony and the install crashes. A Deb822 release must NOT get it, +# or the same suites are configured twice. See the comment in Template.pm. + +require xCAT::Template; + +my $MIRROR = 'http://br.archive.ubuntu.com/ubuntu'; + +sub apt_config_for { + my (%opt) = @_; + no warnings 'redefine'; + local *xCAT::Template::ubuntu_subiquity_apt_mirror = sub { $opt{mirror} }; + local *xCAT::Template::ubuntu_subiquity_uses_deb822_sources = sub { $opt{deb822} }; + local *xCAT::Template::ubuntu_subiquity_otherpkg_sources = sub { () }; + local *xCAT::Template::ubuntu_subiquity_uses_generated_cdrom_source = sub { 0 }; + return xCAT::Template::ubuntu_subiquity_apt_config('/some/media/dir'); +} + +# --- online, classic sources (20.04 / 22.04): the archive must be added via sources: --- +my $classic = apt_config_for( mirror => $MIRROR, deb822 => 0 ); + +like( $classic, qr/^\s*mirror-selection:/m, + 'classic online install still sets the primary mirror' ); +like( $classic, qr/^\s*sources:/m, + 'classic online install adds apt sources (sources_list is ignored by Subiquity)' ); +like( $classic, qr/xcat-ubuntu-archive\.list:/, + 'classic online install writes an xCAT-owned archive source' ); +like( $classic, qr/xcat-ubuntu-updates\.list:/, + 'classic online install writes an xCAT-owned updates source' ); +like( $classic, qr/deb \Q$MIRROR\E \$RELEASE main restricted universe multiverse/, + 'the archive source uses the configured mirror and curtin\'s $RELEASE token' ); + +# --- online, Deb822 (24.04 / 26.04): no legacy .list files on top of ubuntu.sources --- +my $deb822 = apt_config_for( mirror => $MIRROR, deb822 => 1 ); + +like( $deb822, qr/^\s*mirror-selection:/m, + 'Deb822 online install still sets the primary mirror' ); +unlike( $deb822, qr/xcat-ubuntu-archive\.list:/, + 'Deb822 online install does NOT add a legacy archive .list (it would duplicate ubuntu.sources)' ); +unlike( $deb822, qr/xcat-ubuntu-updates\.list:/, + 'Deb822 online install does NOT add a legacy updates .list' ); + +# --- offline is untouched by any of this --- +my $offline = apt_config_for( mirror => '', deb822 => 0 ); +like( $offline, qr/fallback: offline-install/, + 'the offline path is unchanged' ); +unlike( $offline, qr/xcat-ubuntu-archive\.list:/, + 'the offline path adds no online archive source' ); + +done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t new file mode 100644 index 000000000..cf98f2e8e --- /dev/null +++ b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t @@ -0,0 +1,174 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use FindBin; +use File::Temp qw(tempdir); +use IO::Socket::INET; +use Test::More; + +# When Subiquity finishes it reboots, and unless the node has been flipped to local-disk boot it +# PXEs straight back into the installer. The in-target post-script does that through +# updateflag.awk, which needs gawk's |& coprocess -- Ubuntu's /usr/bin/awk is mawk, so the flip +# silently failed and the node reinstalled forever. The template now does the exchange from the +# live installer over bash's built-in /dev/tcp. +# +# Run that command against a stand-in for xcatd and check the exchange, rather than reading the +# template text. + +my $tmpl = "$FindBin::Bin/../../xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl"; +plan skip_all => 'compute.subiquity.tmpl not found' unless -r $tmpl; + +# The install monitor listens on site.xcatiport, and on any management node xcatd is already +# there -- so binding it here made the whole file skip_all exactly where the suite runs. Take an +# ephemeral port from the kernel instead and hand it to the template as the site value. +my $probe = IO::Socket::INET->new( + LocalAddr => '127.0.0.1', LocalPort => 0, Proto => 'tcp', + Listen => 5, ReuseAddr => 1) + or BAIL_OUT("could not take an ephemeral port on the loopback interface: $!"); +my $XCATD_PORT = $probe->sockport; +close $probe; # each case below opens its own listener, or none at all + +open(my $fh, '<', $tmpl) or die "open $tmpl: $!"; +my $source = do { local $/; <$fh> }; +close $fh; + +# The boot flip is the late-command that talks to the install-monitor port. +my ($command) = $source =~ m{- \['bash', '-c', '(.*?/dev/tcp/.*?)'\]}; +BAIL_OUT('no late-command in the template performs the boot flip over /dev/tcp') unless $command; + +# Run the command with the install server pointed at our stand-in, and its log inside a scratch +# tree. site.xcatiport is substituted the way the template renderer substitutes it. Everything +# else is the template's own text. +sub run_flip { + my (%opt) = @_; + my $root = tempdir(CLEANUP => 1); + mkdir "$root/target"; mkdir "$root/target/var"; mkdir "$root/target/var/log"; + mkdir "$root/target/var/log/xcat"; + + my $site_port = exists $opt{site_port} ? $opt{site_port} : $XCATD_PORT; + + my $script = $command; + $script =~ s/\#XCATVAR:XCATMASTER\#/127.0.0.1/; + $script =~ s/\#TABLEBLANKOKAY:site:key=xcatiport:value\#/$site_port/; + $script =~ s{/target/var/log/xcat/xcat\.log}{$root/target/var/log/xcat/xcat.log}; + $script =~ s/sleep 5/sleep 1/; # shorten the retry pause, keep the retry + + my $pid; + if ($opt{listen}) { + $pid = fork(); + die "fork: $!" unless defined $pid; + if (!$pid) { # the stand-in xcatd + sleep $opt{delay} if $opt{delay}; # appear only after the first attempts fail + my $srv = IO::Socket::INET->new(LocalAddr => '127.0.0.1', LocalPort => $XCATD_PORT, + Proto => 'tcp', Listen => 5, ReuseAddr => 1) or exit 1; + open my $seen, '>', "$root/received" or exit 1; + $seen->autoflush(1); + for (1 .. $opt{listen}) { + my $c = $srv->accept() or last; + $c->autoflush(1); + if ($opt{mute}) { sleep 600; close $c; next } # accept and hold, never answer + # xcatd greets with "ready", then answers every request with "done". + print {$c} ($opt{greeting} || "ready\n"); + my $line = <$c>; + print {$seen} $line if defined $line; + print {$c} ($opt{ack} || "done\n") unless $opt{no_ack}; + close $c; + } + close $seen; + close $srv; + exit 0; + } + } + + # The installer would hang here if the exchange ever blocked, so bound it. + # the no-listener case prints "Connection refused" by design + my $cap = $opt{cap} || 25; + my $rc = system("timeout $cap bash -c \Q$script\E 2>/dev/null"); + my $timed_out = (($rc >> 8) == 124); + if ($pid) { kill 'TERM', $pid; waitpid($pid, 0) } + + my $received = ''; + if (open my $rh, '<', "$root/received") { local $/; $received = <$rh> || ''; close $rh } + + my $log = ''; + if (open my $lh, '<', "$root/target/var/log/xcat/xcat.log") { local $/; $log = <$lh> || ''; close $lh } + return { rc => $rc, timed_out => $timed_out, log => $log, received => $received }; +} + +# --- xcatd answers: the node is flipped ------------------------------------ +{ + my $r = run_flip(listen => 1); + is($r->{rc}, 0, 'the boot flip exits cleanly so the install is not failed by it'); + is($r->{log}, '', 'nothing is written to the install log when the flip succeeds'); + is($r->{received}, "next\n", + 'the node sends the token that makes xcatd run "nodeset next"'); + ok(!$r->{timed_out}, 'the exchange completes rather than hanging the late-command'); +} + +# --- the port comes from site.xcatiport ------------------------------------ +# The case above already proves it: the stand-in listens on an ephemeral port, not on 3002, and +# the exchange only completes because the template asks site for the port. What is left is the +# default, for a site table that does not carry the key. +{ + my $r = run_flip(site_port => '', listen => 0, cap => 90); + like($r->{log}, qr/:3002\b/, + 'an unset site.xcatiport falls back to the port xcatd listens on by default'); +} + +# --- xcatd never answers: the failure is recorded, not swallowed ----------- +{ + my $r = run_flip(listen => 0); + is($r->{rc}, 0, 'a failed flip still exits 0 rather than aborting the install'); + like($r->{log}, qr/FAILED to flip/, + 'a failed flip is recorded in the install log instead of PXE-looping silently'); + like($r->{log}, qr/127\.0\.0\.1:\Q$XCATD_PORT\E\b/, + 'the log names the install server and port that could not be reached'); +} + +# --- something else is listening on the port ------------------------------- +# Only xcatd's install monitor answers "nodeset next". A service that accepts the +# connection and talks its own protocol must not be counted as a flipped node. +{ + my $r = run_flip(listen => 5, greeting => "220 smtp\n"); + like($r->{log}, qr/FAILED to flip/, + 'a peer that does not greet with "ready" is not treated as the install monitor'); + is($r->{received}, '', + 'and the flip token is never sent to it'); +} + +# --- the peer greets but does not acknowledge the request ------------------ +{ + my $r = run_flip(listen => 5, ack => "?\n"); + like($r->{log}, qr/FAILED to flip/, + 'a reply other than "done" is not counted as an accepted request'); +} + +# --- xcatd accepts but never acknowledges ---------------------------------- +{ + my $r = run_flip(listen => 5, no_ack => 1); + like($r->{log}, qr/FAILED to flip/, + 'a connection without an acknowledgement counts as a failure, not a success'); +} + +# --- a monitor that accepts and never answers must not hang the install ---- +# The late-command runs inside Subiquity: a bare read on a socket that is open but silent blocks +# forever and the install never finishes. #7759 fixes the monitor dying; this bounds the wait. +{ + # Five attempts, each bounded by two 10s reads plus the retry pause: ~2 minutes worst case. + my $r = run_flip(listen => 1, mute => 1, cap => 200); + ok(!$r->{timed_out}, + 'a monitor that accepts but never replies does not hang the late-command'); + like($r->{log}, qr/FAILED to flip/, + 'it is recorded as a failed flip rather than waiting indefinitely'); +} + +# --- the command retries rather than giving up on the first refusal -------- +{ + # Answer only on a later connection: the flip must still succeed. + my $r = run_flip(listen => 1, delay => 2); + is($r->{log}, '', 'the flip retries until the install monitor answers'); + is($r->{received}, "next\n", 'and the token still reaches it on the later attempt'); +} + +done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_bootparams.t b/xCAT-test/unit/ubuntu_subiquity_bootparams.t deleted file mode 100644 index bea2de236..000000000 --- a/xCAT-test/unit/ubuntu_subiquity_bootparams.t +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env perl -use strict; -use warnings; -use Test::More; - -my $plugin_path = defined $ENV{XCATROOT} ? "$ENV{XCATROOT}/lib/perl/xCAT_plugin/debian.pm" : ''; -$plugin_path = "xCAT-server/lib/xcat/plugins/debian.pm" - unless -f $plugin_path; - -plan skip_all => "debian.pm not found" unless -f $plugin_path; - -my $src = do { local $/; open my $fh, '<', $plugin_path or die $!; <$fh> }; - -like($src, qr/autoinstall ip=dhcp netboot=nfs/, 'subiquity bootparams enable autoinstall'); -like($src, qr/ds=nocloud-net;s=http:\/\//, 'subiquity bootparams point at NoCloud seed'); -like($src, qr/\$kcmdline\s*\.=\s*" ---";/, 'subiquity bootparams end with installer argument separator'); - -done_testing();