From d8a01bcf1555e00e2b32df0ce16d95a9b982d65e Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:42:09 -0300 Subject: [PATCH 01/21] test(xcat-core): capture the Ubuntu Subiquity diskful install never completing The gating reg_linux_diskfull_installation_flat case cannot pass on Ubuntu: the compute node never reaches a booted, installed OS, and the case only ever reports "ssh: connect ... port 22: Connection refused". It is not one defect but a chain, each reachable only once the one before it is fixed. The installer never boots. Without boot=casper, casper never processes netboot=nfs -- it scans the local disks, finds no live media and panics "Unable to find a medium containing a live file system". nfsroot must also be a literal IP: casper mounts the live filesystem with klibc's nfsmount, which cannot resolve a hostname. And with the NFS root still mounted at end of install, systemd-shutdown blocks forever on an lvm/pvscan wedged in uninterruptible I/O on it, so the node never power-cycles into the disk it just installed. The installer has no usable DNS. A nameserver line in /etc/resolv.conf must hold an IP -- glibc's resolver discards a hostname written there -- so writing the xcatmaster name leaves the installer, and the in-target apt that inherits the file, hanging on archive.ubuntu.com. In-target apt cannot find its packages on classic-sources releases, because Subiquity renders the target sources.list from the install media alone. The node never leaves the installer. The boot flip to local disk goes through updateflag.awk, which needs gawk's |& /inet coprocess, but Ubuntu's /usr/bin/awk is mawk -- so the flip fails silently and the node reinstalls forever. And syncfiles runs inside the in-target chroot, asking the MN to scp files into a node that has no sshd yet, so it times out and the node reports failed on a good install. Cover each stage. All fail today. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-test/unit/debian_subiquity_netboot.t | 47 +++++++++++++ .../unit/postage_ubuntu_syncfiles_postboot.t | 59 ++++++++++++++++ xCAT-test/unit/ubuntu_resolvconf_ip.t | 42 +++++++++++ xCAT-test/unit/ubuntu_subiquity_apt_sources.t | 69 +++++++++++++++++++ xCAT-test/unit/ubuntu_subiquity_boot_flip.t | 56 +++++++++++++++ xCAT-test/unit/ubuntu_subiquity_bootparams.t | 3 +- 6 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 xCAT-test/unit/debian_subiquity_netboot.t create mode 100644 xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t create mode 100644 xCAT-test/unit/ubuntu_resolvconf_ip.t create mode 100644 xCAT-test/unit/ubuntu_subiquity_apt_sources.t create mode 100644 xCAT-test/unit/ubuntu_subiquity_boot_flip.t diff --git a/xCAT-test/unit/debian_subiquity_netboot.t b/xCAT-test/unit/debian_subiquity_netboot.t new file mode 100644 index 000000000..5b3b3029c --- /dev/null +++ b/xCAT-test/unit/debian_subiquity_netboot.t @@ -0,0 +1,47 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use Test::More; + +# Regression: the Ubuntu (subiquity) diskful install netboots the live installer over NFS. +# The kernel command line built in debian.pm must satisfy TWO casper requirements or the +# installer never boots and the node PXE-loops forever (surfacing to the test as +# "ssh: connect ... port 22: Connection refused"): +# +# 1. boot=casper -- without it casper does not process netboot=nfs at all; it scans the +# local disks, finds nothing, and panics "Unable to find a medium containing a live +# file system", dropping to the initramfs emergency shell. +# +# 2. nfsroot must be a literal IP, NOT a hostname. casper mounts the live filesystem with +# klibc's nfsmount, which does NOT resolve hostnames -- it fails with +# "nfsmount: can't parse IP address ''". (busybox/util-linux `mount -t nfs` +# resolves names, which is why manual mounts work while casper's does not.) The instserver +# must therefore be passed through xCAT::NetworkUtils->getipaddr() before it is put in +# nfsroot=. The ds=...http URL is fetched later by cloud-init in the booted live system, +# where normal DNS works, so only nfsroot needs the IP. + +sub slurp { my ($p) = @_; local $/; open my $fh, '<', $p or return undef; <$fh> } + +my $deb = slurp('xCAT-server/lib/xcat/plugins/debian.pm'); +plan skip_all => 'debian.pm not found' unless defined $deb; + +# The exact netboot fragment is unique to the subiquity install cmdline, so match it directly +# instead of trying to slice one of the several using_subiquity() blocks. +like($deb, qr/\bboot=casper\b/, + 'subiquity netboot kcmdline includes boot=casper (else casper scans local disks and panics)'); +like($deb, qr/autoinstall ip=dhcp boot=casper netboot=nfs nfsroot=\$\{nfsip\}:/, + 'subiquity kcmdline is: autoinstall ip=dhcp boot=casper netboot=nfs nfsroot='); +like($deb, qr/getipaddr\(\$instserver\)/, + 'instserver is resolved to an IP via getipaddr for nfsroot (klibc nfsmount cannot resolve hostnames)'); +unlike($deb, qr/nfsroot=\$\{instserver\}:/, + 'nfsroot does NOT use the bare instserver hostname (would break klibc nfsmount)'); + +# The subiquity netboot cmdline must include 'toram' so casper copies the squashfs into RAM and +# unmounts the NFS source -- otherwise the end-of-install reboot wedges in systemd-shutdown on a +# D-state process doing I/O to the still-mounted NFS root, and the node never boots the installed disk. +like($deb, qr/\btoram\b/, + 'subiquity netboot cmdline includes toram (run from RAM, no NFS root at shutdown -> no reboot hang)'); +unlike($deb, qr/nfsroot=[^ ]*,soft/, + 'nfsroot does NOT append ,soft (casper takes the whole nfsroot value as the path, which breaks the mount)'); + +done_testing(); diff --git a/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t b/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t new file mode 100644 index 000000000..00cde9e77 --- /dev/null +++ b/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t @@ -0,0 +1,59 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use FindBin; +use Test::More; + +# Regression: on Ubuntu/Debian the DISKFUL install runs a node's postscripts inside the +# installer's in-target chroot -- before the node has booted as itself. The syncfiles +# postscript works by asking the management node to scp files INTO the running node, which +# cannot happen in that phase: the not-yet-booted node has no sshd for the MN to reach, so the +# push times out, syncfiles exits 1, and the node reports status=failed even though the OS +# installed perfectly. +# +# syncfiles must therefore move to the postbootscripts set, which runs on the booted node +# where ssh is already listening. Two things bound that move: +# +# * It applies to the DISKFUL install path only. netboot and statelite already run their +# postscripts on the booted node, so moving syncfiles there changes working behaviour for +# no reason. EL/SLES are unaffected either way -- their postscripts already run on the +# booted node. +# * syncfiles must run BEFORE the existing postbootscripts, since a postbootscript may +# consume the files it synchronises. Appending it would invert that. + +my $repo_root = File::Spec->rel2abs( + File::Spec->catdir( $FindBin::Bin, '..', '..' ) +); +my $path = File::Spec->catfile( $repo_root, 'xCAT-server', 'lib', 'perl', 'xCAT', 'Postage.pm' ); +plan skip_all => "Postage.pm not found" unless -f $path; + +my $src = do { local $/; open my $fh, '<', $path or die $!; <$fh> }; + +# Isolate the deferral block so the assertions below cannot accidentally match code elsewhere. +my ($block) = $src =~ /(\n[^\n]*ubuntu\|debian[^\n]*\n(?:.*?\n)*?[^\n]*syncfiles(?:.*?\n)*?\s*\}\n\s*\}\n)/; +ok( defined $block, 'found the syncfiles deferral block in makescript' ) + or do { done_testing(); exit }; + +like( $block, qr/\$os\s*=~\s*.\^\(\?:ubuntu\|debian\)/, + 'the move is guarded to ubuntu/debian nodes only' ); + +like( $block, qr/\$diskful_install|\bnodesetstate\b|\bprovmethod\b/, + 'the move is guarded to the diskful install path (not netboot/statelite)' ); + +like( $block, qr/\$postscripts\s*=~\s*s\/\^\[[^\]]*\]\*syncfiles/, + 'syncfiles is stripped from the in-target postscripts list' ); + +like( $block, qr/\$postbootscripts\s*=\s*"[^"]*syncfiles[^"]*"\s*\.\s*\$postbootscripts/, + 'syncfiles is PREPENDED to postbootscripts, so it runs before scripts that consume its files' ); + +unlike( $block, qr/\$postbootscripts\s*\.=\s*"syncfiles/, + 'syncfiles is not appended after the existing postbootscripts' ); + +# The strip and the re-add must be paired, so a node with no syncfiles postscript never +# acquires a spurious one. +like( $block, qr/if\s*\(.*?\$postscripts.*?syncfiles.*?\)\s*\{\s*.*?\$postbootscripts/s, + 'syncfiles is only added to postbootscripts when it was present in postscripts' ); + +done_testing(); diff --git a/xCAT-test/unit/ubuntu_resolvconf_ip.t b/xCAT-test/unit/ubuntu_resolvconf_ip.t new file mode 100644 index 000000000..d35600d8b --- /dev/null +++ b/xCAT-test/unit/ubuntu_resolvconf_ip.t @@ -0,0 +1,42 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use Test::More; + +# Regression: the Ubuntu (subiquity) diskful install writes the installer's /etc/resolv.conf +# from compute.subiquity.tmpl's "DNS setup" early-command. A nameserver line in +# /etc/resolv.conf MUST be an IP address -- glibc's resolver does NOT resolve a hostname +# given on a nameserver line, it simply discards the entry. The template used to write the +# xcatmaster *name* there: +# +# echo "nameserver #TABLE:noderes:$NODE:xcatmaster#" >/etc/resolv.conf +# +# which left the installer (and the in-target apt-get update, which inherits this resolv.conf) +# with no usable DNS. The install then hangs resolving archive.ubuntu.com -- the node never +# finishes, and the test sees "ssh: connect ... port 22: Connection refused". (Everything the +# early-command did BEFORE this point still worked because the live installer's DHCP-provided +# resolv.conf can resolve the xcatmaster name -- so the failure only surfaces at the very next +# DNS-dependent step, the in-target apt.) +# +# The fix resolves the xcatmaster hostname to an IPv4 address (via getent, while DNS still +# works) and writes that IP into resolv.conf. + +sub slurp { my ($p) = @_; local $/; open my $fh, '<', $p or return undef; <$fh> } + +my $tmpl = slurp('xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl'); +plan skip_all => 'compute.subiquity.tmpl not found' unless defined $tmpl; + +# The old, broken form -- a bare hostname token written straight into the nameserver line -- +# must be gone. +unlike($tmpl, qr/nameserver \s+ \#TABLE:noderes:\$NODE:xcatmaster\# /x, + 'resolv.conf nameserver is NOT the bare xcatmaster hostname token (glibc cannot resolve a name there)'); + +# The xcatmaster name must be resolved to an IPv4 address before it is used as a nameserver. +like($tmpl, qr/getent \s+ ahostsv4 /x, + 'DNS setup resolves the xcatmaster to an IPv4 address via getent ahostsv4'); + +# And the nameserver line must be written from that resolved IP variable. +like($tmpl, qr/nameserver \s+ \$xcatmaster_ip/x, + 'resolv.conf nameserver is written from the resolved xcatmaster IP'); + +done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_apt_sources.t b/xCAT-test/unit/ubuntu_subiquity_apt_sources.t new file mode 100644 index 000000000..f857af48e --- /dev/null +++ b/xCAT-test/unit/ubuntu_subiquity_apt_sources.t @@ -0,0 +1,69 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::Bin/../../perl-xCAT"; +use lib "$FindBin::Bin/../../xCAT-server/lib/perl"; + +use Test::More; + +# Regression: on an ONLINE Subiquity install of a classic-sources release (20.04 / 22.04), +# Subiquity renders the target's /etc/apt/sources.list from the install media alone +# ("deb file:///cdrom"). A live-server ISO does not carry everything xCAT's postscripts need -- +# chrony, for one -- so curtin's in-target apt fails with "E: Unable to locate package chrony" +# and the install crashes. `sources_list:` is a curtin key that Subiquity's autoinstall schema +# ignores, so the online archive has to be added through `sources:`, which Subiquity honours by +# writing /etc/apt/sources.list.d/*.list. +# +# That fix belongs to the classic-sources releases ONLY. On a Deb822 release (24.04 / 26.04) +# the primary mirror already lands in /etc/apt/sources.list.d/ubuntu.sources, so adding legacy +# .list files on top produces duplicate apt entries for the same suites -- exactly the source +# conflict the Deb822 branch elsewhere in this function goes out of its way to avoid. + +require xCAT::Template; + +my $MIRROR = 'http://br.archive.ubuntu.com/ubuntu'; + +sub apt_config_for { + my (%opt) = @_; + no warnings 'redefine'; + local *xCAT::Template::ubuntu_subiquity_apt_mirror = sub { $opt{mirror} }; + local *xCAT::Template::ubuntu_subiquity_uses_deb822_sources = sub { $opt{deb822} }; + local *xCAT::Template::ubuntu_subiquity_otherpkg_sources = sub { () }; + local *xCAT::Template::ubuntu_subiquity_uses_generated_cdrom_source = sub { 0 }; + return xCAT::Template::ubuntu_subiquity_apt_config('/some/media/dir'); +} + +# --- online, classic sources (20.04 / 22.04): the archive must be added via sources: --- +my $classic = apt_config_for( mirror => $MIRROR, deb822 => 0 ); + +like( $classic, qr/^\s*mirror-selection:/m, + 'classic online install still sets the primary mirror' ); +like( $classic, qr/^\s*sources:/m, + 'classic online install adds apt sources (sources_list is ignored by Subiquity)' ); +like( $classic, qr/xcat-ubuntu-archive\.list:/, + 'classic online install writes an xCAT-owned archive source' ); +like( $classic, qr/xcat-ubuntu-updates\.list:/, + 'classic online install writes an xCAT-owned updates source' ); +like( $classic, qr/deb \Q$MIRROR\E \$RELEASE main restricted universe multiverse/, + 'the archive source uses the configured mirror and curtin\'s $RELEASE token' ); + +# --- online, Deb822 (24.04 / 26.04): no legacy .list files on top of ubuntu.sources --- +my $deb822 = apt_config_for( mirror => $MIRROR, deb822 => 1 ); + +like( $deb822, qr/^\s*mirror-selection:/m, + 'Deb822 online install still sets the primary mirror' ); +unlike( $deb822, qr/xcat-ubuntu-archive\.list:/, + 'Deb822 online install does NOT add a legacy archive .list (it would duplicate ubuntu.sources)' ); +unlike( $deb822, qr/xcat-ubuntu-updates\.list:/, + 'Deb822 online install does NOT add a legacy updates .list' ); + +# --- offline is untouched by any of this --- +my $offline = apt_config_for( mirror => '', deb822 => 0 ); +like( $offline, qr/fallback: offline-install/, + 'the offline path is unchanged' ); +unlike( $offline, qr/xcat-ubuntu-archive\.list:/, + 'the offline path adds no online archive source' ); + +done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t new file mode 100644 index 000000000..6d57d5e30 --- /dev/null +++ b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t @@ -0,0 +1,56 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use FindBin; +use Test::More; + +# Regression: after Subiquity installs the OS the node MUST be flipped to local-disk boot, or it +# PXE-loops straight back into the installer and the installed OS -- with its sshd -- never +# boots. The diskful case then only ever reports +# "ssh: connect to host port 22: Connection refused". +# +# xCAT performs that flip when the node reports "next" to xcatd on the install-monitor port, +# which makes xcatd run "nodeset next" and rewrite the node's xNBA script to fall through +# to the local disk. The in-target post-script tries this through updateflag.awk, but that +# depends on gawk's |& / /inet coprocess and on /usr/bin/awk being gawk -- on Ubuntu +# /usr/bin/awk is normally mawk, which has neither, so the flip fails silently. +# +# The template must therefore trigger the flip itself from the live installer, with no gawk +# dependency, and must NOT report success when it did not happen. + +my $repo_root = File::Spec->rel2abs( + File::Spec->catdir( $FindBin::Bin, '..', '..' ) +); +my $path = File::Spec->catfile( + $repo_root, 'xCAT-server', 'share', 'xcat', 'install', 'ubuntu', 'compute.subiquity.tmpl' +); +plan skip_all => 'compute.subiquity.tmpl not found' unless -f $path; + +my $tmpl = do { local $/; open my $fh, '<', $path or die $!; <$fh> }; + +like( $tmpl, qr{/dev/tcp/\$xm/3002}, + 'the flip contacts xcatd on the install-monitor port from the installer' ); +like( $tmpl, qr{printf "next}, + 'it sends the "next" token that makes xcatd run "nodeset next"' ); +like( $tmpl, qr{xm=#XCATVAR:XCATMASTER#}, + "the flip targets the node's own xcatmaster" ); +like( $tmpl, qr{^\s*-\s*\['bash',\s*'-c',}m, + 'the flip runs under bash (dash has no /dev/tcp) via an argv list command' ); + +# A failed flip must be visible. The original form broke out of its retry loop as soon as the +# socket connected -- never on a confirmed exchange -- and ended in `true`, so a total failure +# looked exactly like success and the node silently reinstalled forever. +like( $tmpl, qr{ok=1}, + 'success is recorded only after the exchange completes, not merely on connect' ); +like( $tmpl, qr{FAILED to flip}, + 'a failed flip is reported into the install log rather than passing silently' ); + +# The magic-SysRq forced reboot must be gone: toram already unmounts the NFS live root, so the +# shutdown hang it worked around cannot occur, and an unconditional timed reboot would race a +# slow install. +unlike( $tmpl, qr{sysrq-trigger}, + 'no unconditional magic-SysRq reboot (toram removes the hang it worked around)' ); + +done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_bootparams.t b/xCAT-test/unit/ubuntu_subiquity_bootparams.t index bea2de236..fccc9502f 100644 --- a/xCAT-test/unit/ubuntu_subiquity_bootparams.t +++ b/xCAT-test/unit/ubuntu_subiquity_bootparams.t @@ -11,7 +11,8 @@ plan skip_all => "debian.pm not found" unless -f $plugin_path; my $src = do { local $/; open my $fh, '<', $plugin_path or die $!; <$fh> }; -like($src, qr/autoinstall ip=dhcp netboot=nfs/, 'subiquity bootparams enable autoinstall'); +like($src, qr/autoinstall ip=dhcp boot=casper netboot=nfs/, + 'subiquity bootparams enable autoinstall (boot=casper is required for casper to process netboot=nfs)'); like($src, qr/ds=nocloud-net;s=http:\/\//, 'subiquity bootparams point at NoCloud seed'); like($src, qr/\$kcmdline\s*\.=\s*" ---";/, 'subiquity bootparams end with installer argument separator'); From a6f77a9732aa242630594251e42c1e06d63d2a25 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:43:20 -0300 Subject: [PATCH 02/21] fix(xcat-core): complete the Ubuntu Subiquity diskful install Boot the live installer correctly. Add boot=casper so casper actually processes netboot=nfs instead of scanning local disks and panicking, resolve the install server to a literal IP because casper mounts the live filesystem with klibc's nfsmount which has no resolver, and add toram so casper copies the squashfs into RAM and unmounts the NFS source. That last one is what lets the node reboot at all: with the NFS root still mounted, systemd-shutdown waits forever on an lvm/pvscan wedged in uninterruptible I/O on it and the node never power-cycles into the disk it just installed. casper has no cmdline knob for NFS mount options -- it parses only nfsroot= and takes the whole value as the path -- so toram is its supported way to avoid a network root. Write the installer's resolv.conf nameserver as an IP. glibc's resolver discards a hostname given on a nameserver line, so the xcatmaster name left the installer, and the in-target apt that inherits the file, with no DNS at all. Add the online archive through apt `sources:` on classic-sources releases, where Subiquity renders the target sources.list from the install media alone and in-target apt cannot find packages the ISO does not carry. Deb822 releases are excluded: their primary mirror already lands in ubuntu.sources, so legacy .list files would configure the same suites twice. Flip the node to local-disk boot from the live installer over bash's /dev/tcp instead of relying on updateflag.awk, which needs gawk's |& coprocess while Ubuntu's /usr/bin/awk is mawk. The exchange is checked, and a failure is recorded in the install log rather than silently PXE-looping into another install. Defer syncfiles to the postbootscripts on the diskful install path, so it runs on the booted node with sshd up rather than inside the in-target chroot where the MN cannot reach it. Make the test harness's provision wait overridable through WAIT_FOR_PROVISION; the default stays short so a boot loop still fails fast. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-server/lib/perl/xCAT/Postage.pm | 33 +++++++++++++++++++ xCAT-server/lib/perl/xCAT/Template.pm | 21 +++++++++++- xCAT-server/lib/xcat/plugins/debian.pm | 22 ++++++++++++- .../install/ubuntu/compute.subiquity.tmpl | 22 ++++++++++++- .../testcase/commoncmd/retry_install.sh | 2 +- 5 files changed, 96 insertions(+), 4 deletions(-) diff --git a/xCAT-server/lib/perl/xCAT/Postage.pm b/xCAT-server/lib/perl/xCAT/Postage.pm index b7d44b9d3..ddf949b67 100644 --- a/xCAT-server/lib/perl/xCAT/Postage.pm +++ b/xCAT-server/lib/perl/xCAT/Postage.pm @@ -546,6 +546,39 @@ sub makescript { my $postbootscripts; $postbootscripts = getPostbootScripts($node, $osimgname, $script_hash); + # On Ubuntu/Debian the DISKFUL install runs a node's postscripts inside the installer's + # in-target chroot -- before the node has booted as itself. The syncfiles postscript + # works by asking the management node to scp files INTO the running node, which cannot + # happen in that phase: the not-yet-booted node has no sshd for the MN to reach, so the + # push times out, syncfiles exits 1, and the node reports status=failed even though the + # OS installed perfectly. Defer it to the postbootscripts, which run on the booted node + # where ssh is already listening and the MN's push succeeds. + # + # Scope this to the diskful install path only: netboot and statelite already run their + # postscripts on the booted node, so moving syncfiles there would change behaviour that + # works. EL/SLES are unaffected either way -- their postscripts run on the booted node. + # syncfiles is PREPENDED so it still runs before any postbootscript that consumes the + # files it synchronises. + if (defined($os) && $os =~ /^(?:ubuntu|debian)/i) { + my $effective_provmethod = $provmethod; + if ($osimgname && defined($image_hash{$osimgname}{'provmethod'})) { + $effective_provmethod = $image_hash{$osimgname}{'provmethod'}; + } + my $diskful_install = + ($nodesetstate && $nodesetstate eq 'install') + || (!$nodesetstate + && defined($effective_provmethod) + && $effective_provmethod eq 'install'); + if ($diskful_install + && defined($postscripts) + && $postscripts =~ s/^[ \t]*syncfiles[ \t]*\n//m) { + $postbootscripts = "" unless defined $postbootscripts; + $postbootscripts = + "# ubuntu-deferred-postbootscripts-start-here\nsyncfiles\n# ubuntu-deferred-postbootscripts-end-here\n" + . $postbootscripts; + } + } + # if using zones then must go to the zone.sshbetweennodes # else go to site.sshbetweennodes my $enablesshbetweennodes; diff --git a/xCAT-server/lib/perl/xCAT/Template.pm b/xCAT-server/lib/perl/xCAT/Template.pm index bcf37d647..a1f71aafd 100644 --- a/xCAT-server/lib/perl/xCAT/Template.pm +++ b/xCAT-server/lib/perl/xCAT/Template.pm @@ -1686,8 +1686,27 @@ sub ubuntu_subiquity_apt_config ' primary:', " - uri: $online_mirror", ); - if (@otherpkg_sources) { + # On a classic-sources release (20.04/22.04) Subiquity renders the target's + # /etc/apt/sources.list from the install media alone (deb file:///cdrom), which lacks + # packages xCAT needs such as chrony -- curtin's in-target apt then fails with + # "E: Unable to locate package chrony". `sources_list:` is a curtin key that + # Subiquity's autoinstall schema ignores, so add the online archive through `sources:`, + # which Subiquity honours by writing /etc/apt/sources.list.d/*.list. $RELEASE is + # substituted with the release codename by curtin. + # + # Deb822 releases (24.04+) are deliberately excluded: there the primary mirror above + # already lands in /etc/apt/sources.list.d/ubuntu.sources, so adding these legacy .list + # files would configure the same suites twice. + my $need_sources_block = !$use_deb822; + if ($need_sources_block) { push @lines, ' sources:'; + push @lines, ' xcat-ubuntu-archive.list:'; + push @lines, qq( source: "deb $online_mirror \$RELEASE main restricted universe multiverse"); + push @lines, ' xcat-ubuntu-updates.list:'; + push @lines, qq( source: "deb $online_mirror \$RELEASE-updates main restricted universe multiverse"); + } + if (@otherpkg_sources) { + push @lines, ' sources:' unless $need_sources_block; my $index = 0; foreach my $source (@otherpkg_sources) { push @lines, " xcat-otherpkgs-$index.list:"; diff --git a/xCAT-server/lib/xcat/plugins/debian.pm b/xCAT-server/lib/xcat/plugins/debian.pm index 4051ca18e..04b470bf4 100644 --- a/xCAT-server/lib/xcat/plugins/debian.pm +++ b/xCAT-server/lib/xcat/plugins/debian.pm @@ -986,7 +986,27 @@ sub mkinstall { my $kcmdline = "nofb utf8 auto xcatd=" . $instserver; if (using_subiquity($os,$tmplfile)) { - $kcmdline .= " autoinstall ip=dhcp netboot=nfs nfsroot=${instserver}:${pkgdir}"; + # boot=casper is required: without it casper never processes netboot=nfs, it + # scans the local disks, finds no live media and panics "Unable to find a + # medium containing a live file system" (initramfs emergency shell -> PXE loop). + # nfsroot MUST be a literal IP: casper mounts the live filesystem with klibc's + # nfsmount, which cannot resolve hostnames ("nfsmount: can't parse IP address + # ''"), so resolve instserver to its IP. The ds= URL is fetched later by + # cloud-init in the booted live system where normal DNS works, so it may stay a + # hostname. + # 'toram' makes casper copy the live squashfs into RAM and UNMOUNT the NFS source + # (casper scripts/casper: copy_to_ram then `umount ${copyfrom}`), so the installer runs + # from RAM with NO network root. This fixes the end-of-install reboot hang: with the NFS + # root still mounted, a process doing I/O to it during systemd-shutdown (lvm, netplan, + # udev) blocks in uninterruptible D state -- it cannot be SIGKILLed, so systemd-shutdown + # waits forever ("Waiting for process: (lvm)") and the node never power-cycles into + # the installed disk. With the root in RAM there is nothing to wait on and the reboot + # completes on its own. (casper has no cmdline knob for NFS mount options -- it parses + # only nfsroot=, taking the whole value as the path, so appending ,soft breaks the mount; + # toram is casper's supported way to avoid the network root. The 24.04 layers total + # ~1.5G, well within the CN's RAM.) + my $nfsip = xCAT::NetworkUtils->getipaddr($instserver) || $instserver; + $kcmdline .= " autoinstall ip=dhcp boot=casper netboot=nfs nfsroot=${nfsip}:${pkgdir} toram"; $kcmdline .= " ds=nocloud-net;s=http://${instserver}:${httpport}/install/autoinst/${node}/"; $kcmdline .= " ---"; } else { diff --git a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl index 31a978cbc..547a48aa1 100644 --- a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl +++ b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl @@ -66,8 +66,17 @@ autoinstall: sed -i '/^\.\.\.$/d' /autoinstall.yaml cat /tmp/partitionfile >> /autoinstall.yaml echo "=== DNS setup ===" + # A nameserver line in /etc/resolv.conf MUST hold an IP address -- glibc's resolver does + # NOT resolve a hostname written there, it discards the entry. Writing the xcatmaster + # *name* leaves the installer (and the in-target apt-get, which inherits this file) with + # no usable DNS, so the install hangs resolving archive.ubuntu.com. DNS still works at + # this point through the live installer's DHCP resolv.conf -- the wget calls above relied + # on it -- so resolve the xcatmaster to an IPv4 address first and write that. + xcatmaster_host="#TABLE:noderes:$NODE:xcatmaster#" + xcatmaster_ip="$(getent ahostsv4 "$xcatmaster_host" | awk '{print $1; exit}')" + [ -n "$xcatmaster_ip" ] || xcatmaster_ip="$xcatmaster_host" rm -f /etc/resolv.conf - echo "nameserver #TABLE:noderes:$NODE:xcatmaster#" >/etc/resolv.conf + echo "nameserver $xcatmaster_ip" >/etc/resolv.conf echo "domain #TABLE:site:key=domain:value#" >>/etc/resolv.conf echo "=== early-commands complete ===" late-commands: @@ -101,5 +110,16 @@ autoinstall: cp ./#HOSTNAME#.post /target/root/post.script; curtin in-target --target /target /root/post.script; } >>/target/var/log/xcat/xcat.log 2>&1' + # Switch the node to boot from the local disk now the install is done, or it PXE-loops + # straight back into the installer when Subiquity reboots. xCAT flips the netboot state + # when the node reports "next" to xcatd on the install-monitor port, which makes xcatd run + # "nodeset next" and rewrite the node's xNBA script to fall through to local disk. + # The in-target post-script tries this through updateflag.awk, but that needs gawk's |& + # /inet coprocess and /usr/bin/awk on Ubuntu is normally mawk, so the flip silently fails. + # Send it from the live installer instead -- the connection then originates from the node's + # own IP, which xcatd trusts -- using bash's built-in /dev/tcp, with no gawk dependency. + # xcatd answers with a ready token, takes "next", and acknowledges; treat anything else as + # a failure and say so in the install log rather than rebooting into another install. + - ['bash', '-c', 'xm=#XCATVAR:XCATMASTER#; ok=0; for i in 1 2 3 4 5; do if exec 3<>/dev/tcp/$xm/3002; then read -r r <&3 || true; printf "next\n" >&3; if read -r r <&3; then ok=1; fi; exec 3>&- 3<&-; [ "$ok" = 1 ] && break; fi; sleep 5; done; if [ "$ok" != 1 ]; then echo "xcat: FAILED to flip $(hostname) to local-disk boot via $xm:3002; the node will PXE back into the installer" >>/target/var/log/xcat/xcat.log; fi; exit 0'] error-commands: - tar -c --ignore-failed-read --transform='s/^/#HOSTNAME#-logs\//' /var/crash /var/log/installer /tmp/pre-install.log /autoinstall.yaml 2>/dev/null |nc -l 8080 diff --git a/xCAT-test/autotest/testcase/commoncmd/retry_install.sh b/xCAT-test/autotest/testcase/commoncmd/retry_install.sh index 9f4c9e24e..d4589e673 100755 --- a/xCAT-test/autotest/testcase/commoncmd/retry_install.sh +++ b/xCAT-test/autotest/testcase/commoncmd/retry_install.sh @@ -7,7 +7,7 @@ node=$1 osimage=$2 vmhost=`lsdef $node -i vmhost -c | cut -d '=' -f 2` times=2 -wait_for_provision=20 #Min to wait for node to provision +wait_for_provision=${WAIT_FOR_PROVISION:-20} #Min to wait for node to provision (overridable via WAIT_FOR_PROVISION; kept short so a boot-loop fails fast, raised only where the happy path is slow -- e.g. Ubuntu subiquity diskful) check_status=10 #Sec to keep checking status iterations=$wait_for_provision*60/$check_status #Iterations to check for "booted" status From 00ef09add235a7344575fb8a863cc6e589e1f9b7 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:31:46 -0300 Subject: [PATCH 03/21] test(xcat-core): assert the Subiquity install behaviour instead of the source that implements it The tests added with this fix matched regexes against the text of debian.pm, Postage.pm and compute.subiquity.tmpl. A source match cannot tell whether the code it found ever runs: moving the Subiquity command line into its own routine leaves every one of those assertions passing, and reformatting a line fails them while the behaviour is untouched. One of them pinned Perl syntax outright, qr/\$kcmdline\s*\.=\s*" ---";/, and another matched the text of a substitution. Run the code instead. Two decisions were lifted out of the routines that had grown around them, so a test can call them: subiquity_kcmdline() in debian.pm builds the installer command line from its inputs, and defer_syncfiles_to_postboot() in Postage.pm returns the adjusted postscript lists. Both are pure and carry the reasoning that used to sit inline. The callers keep their behaviour exactly. The template's two shell fragments are extracted and executed: the boot flip runs against a stand-in for xcatd on the install-monitor port, and the resolv.conf step runs with a getent that answers as the case requires. Every assertion now fails when the behaviour it describes is removed, which is what the source matches only appeared to do: boot=casper dropped from the command line 1 assertion fails toram dropped 1 assertion fails nfsroot given the host name instead of the IP 2 assertions fail the deferral made a no-op 7 assertions fail the wrong token sent to xcatd 1 assertion fails the failed-flip log line removed 3 assertions fail the retry loop reduced to one attempt 2 assertions fail resolv.conf given the host name 3 assertions fail the resolution fallback removed 2 assertions fail ubuntu_subiquity_bootparams.t is removed. Its three matches are covered by execution in debian_subiquity_netboot.t, and it demonstrated the problem: it still passed after the command line moved into another routine. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-server/lib/perl/xCAT/Postage.pm | 90 +++++++---- xCAT-server/lib/xcat/plugins/debian.pm | 73 ++++++--- xCAT-test/unit/debian_subiquity_netboot.t | 95 +++++++---- .../unit/postage_ubuntu_syncfiles_postboot.t | 124 +++++++++----- xCAT-test/unit/ubuntu_resolvconf_ip.t | 95 +++++++---- xCAT-test/unit/ubuntu_subiquity_boot_flip.t | 151 +++++++++++++----- xCAT-test/unit/ubuntu_subiquity_bootparams.t | 19 --- 7 files changed, 427 insertions(+), 220 deletions(-) delete mode 100644 xCAT-test/unit/ubuntu_subiquity_bootparams.t diff --git a/xCAT-server/lib/perl/xCAT/Postage.pm b/xCAT-server/lib/perl/xCAT/Postage.pm index ddf949b67..3e009b684 100644 --- a/xCAT-server/lib/perl/xCAT/Postage.pm +++ b/xCAT-server/lib/perl/xCAT/Postage.pm @@ -173,6 +173,58 @@ my $mn; %::GLOBAL_SN_HASH; %::GLOBAL_TABDUMP_HASH; +#------------------------------------------------------------------------------- + +=head3 defer_syncfiles_to_postboot + + On Ubuntu/Debian the DISKFUL install runs a node's postscripts inside the installer's + in-target chroot -- before the node has booted as itself. The syncfiles postscript works + by asking the management node to scp files INTO the running node, which cannot happen in + that phase: the not-yet-booted node has no sshd for the MN to reach, so the push times + out, syncfiles exits 1, and the node reports status=failed even though the OS installed + perfectly. Defer it to the postbootscripts, which run on the booted node where ssh is + already listening and the MN's push succeeds. + + Scoped to the diskful install path only: netboot and statelite already run their + postscripts on the booted node, so moving syncfiles there would change behaviour that + works. EL/SLES are unaffected either way -- their postscripts run on the booted node. + syncfiles is PREPENDED so it still runs before any postbootscript that consumes the + files it synchronises. + + Arguments: + $os nodetype.os for the node + $provmethod the effective provmethod (the osimage's, when the node names one) + $nodesetstate the nodeset state, when the caller knows it + $postscripts the rendered postscripts list + $postbootscripts the rendered postbootscripts list + Returns: + ($postscripts, $postbootscripts), unchanged unless the deferral applies + +=cut + +#------------------------------------------------------------------------------- +sub defer_syncfiles_to_postboot { + my ($os, $provmethod, $nodesetstate, $postscripts, $postbootscripts) = @_; + + return ($postscripts, $postbootscripts) + unless defined($os) && $os =~ /^(?:ubuntu|debian)/i; + + my $diskful_install = + ($nodesetstate && $nodesetstate eq 'install') + || (!$nodesetstate && defined($provmethod) && $provmethod eq 'install'); + return ($postscripts, $postbootscripts) unless $diskful_install; + + return ($postscripts, $postbootscripts) + unless defined($postscripts) && $postscripts =~ s/^[ \t]*syncfiles[ \t]*\n//m; + + $postbootscripts = "" unless defined $postbootscripts; + $postbootscripts = + "# ubuntu-deferred-postbootscripts-start-here\nsyncfiles\n# ubuntu-deferred-postbootscripts-end-here\n" + . $postbootscripts; + + return ($postscripts, $postbootscripts); +} + sub makescript { my $nodes = shift; my $nodesetstate = shift; @@ -546,38 +598,14 @@ sub makescript { my $postbootscripts; $postbootscripts = getPostbootScripts($node, $osimgname, $script_hash); - # On Ubuntu/Debian the DISKFUL install runs a node's postscripts inside the installer's - # in-target chroot -- before the node has booted as itself. The syncfiles postscript - # works by asking the management node to scp files INTO the running node, which cannot - # happen in that phase: the not-yet-booted node has no sshd for the MN to reach, so the - # push times out, syncfiles exits 1, and the node reports status=failed even though the - # OS installed perfectly. Defer it to the postbootscripts, which run on the booted node - # where ssh is already listening and the MN's push succeeds. - # - # Scope this to the diskful install path only: netboot and statelite already run their - # postscripts on the booted node, so moving syncfiles there would change behaviour that - # works. EL/SLES are unaffected either way -- their postscripts run on the booted node. - # syncfiles is PREPENDED so it still runs before any postbootscript that consumes the - # files it synchronises. - if (defined($os) && $os =~ /^(?:ubuntu|debian)/i) { - my $effective_provmethod = $provmethod; - if ($osimgname && defined($image_hash{$osimgname}{'provmethod'})) { - $effective_provmethod = $image_hash{$osimgname}{'provmethod'}; - } - my $diskful_install = - ($nodesetstate && $nodesetstate eq 'install') - || (!$nodesetstate - && defined($effective_provmethod) - && $effective_provmethod eq 'install'); - if ($diskful_install - && defined($postscripts) - && $postscripts =~ s/^[ \t]*syncfiles[ \t]*\n//m) { - $postbootscripts = "" unless defined $postbootscripts; - $postbootscripts = - "# ubuntu-deferred-postbootscripts-start-here\nsyncfiles\n# ubuntu-deferred-postbootscripts-end-here\n" - . $postbootscripts; - } + # See defer_syncfiles_to_postboot(): on the Ubuntu/Debian diskful install path syncfiles + # has to run on the booted node, not in the installer's in-target chroot. + my $effective_provmethod = $provmethod; + if ($osimgname && defined($image_hash{$osimgname}{'provmethod'})) { + $effective_provmethod = $image_hash{$osimgname}{'provmethod'}; } + ($postscripts, $postbootscripts) = defer_syncfiles_to_postboot( + $os, $effective_provmethod, $nodesetstate, $postscripts, $postbootscripts); # if using zones then must go to the zone.sshbetweennodes # else go to site.sshbetweennodes diff --git a/xCAT-server/lib/xcat/plugins/debian.pm b/xCAT-server/lib/xcat/plugins/debian.pm index 04b470bf4..4c7987195 100644 --- a/xCAT-server/lib/xcat/plugins/debian.pm +++ b/xCAT-server/lib/xcat/plugins/debian.pm @@ -507,6 +507,56 @@ sub copycd } } +#------------------------------------------------------------------------------- + +=head3 subiquity_kcmdline + + Build the kernel command line for a Subiquity (Ubuntu live installer) diskful install. + + boot=casper is required: without it casper never processes netboot=nfs, it scans the local + disks, finds no live media and panics "Unable to find a medium containing a live file + system" (initramfs emergency shell -> PXE loop). + + nfsroot MUST be a literal IP: casper mounts the live filesystem with klibc's nfsmount, which + cannot resolve hostnames ("nfsmount: can't parse IP address ''"). The ds= URL is + fetched later by cloud-init in the booted live system where normal DNS works, so it keeps + the install server's name. + + 'toram' makes casper copy the live squashfs into RAM and UNMOUNT the NFS source (casper + scripts/casper: copy_to_ram then umount of the copy source), so the installer runs from RAM + with NO network root. This is what lets the node reboot at all: with the NFS root still + mounted, a process doing I/O to it during systemd-shutdown (lvm, netplan, udev) blocks in + uninterruptible D state -- it cannot be SIGKILLed, so systemd-shutdown waits forever + ("Waiting for process: (lvm)") and the node never power-cycles into the disk it just + installed. casper has no cmdline knob for NFS mount options -- it parses only nfsroot=, + taking the whole value as the path, so appending ,soft breaks the mount; toram is casper's + supported way to avoid the network root. The 24.04 layers total ~1.5G, well within the CN's + RAM. + + Arguments: + $base the command line built so far + $nfsip the install server as a literal IP, for casper's klibc nfsmount + $pkgdir the install media path exported over NFS + $instserver the install server name, for the cloud-init seed URL + $httpport the xCAT HTTP port + $node the node being installed + Returns: + the completed command line + +=cut + +#------------------------------------------------------------------------------- +sub subiquity_kcmdline { + my ($base, $nfsip, $pkgdir, $instserver, $httpport, $node) = @_; + + my $kcmdline = $base; + $kcmdline .= " autoinstall ip=dhcp boot=casper netboot=nfs nfsroot=${nfsip}:${pkgdir} toram"; + $kcmdline .= " ds=nocloud-net;s=http://${instserver}:${httpport}/install/autoinst/${node}/"; + $kcmdline .= " ---"; + + return $kcmdline; +} + sub mkinstall { xCAT::MsgUtils->message("S", "Doing debian mkinstall"); my $request = shift; @@ -986,29 +1036,8 @@ sub mkinstall { my $kcmdline = "nofb utf8 auto xcatd=" . $instserver; if (using_subiquity($os,$tmplfile)) { - # boot=casper is required: without it casper never processes netboot=nfs, it - # scans the local disks, finds no live media and panics "Unable to find a - # medium containing a live file system" (initramfs emergency shell -> PXE loop). - # nfsroot MUST be a literal IP: casper mounts the live filesystem with klibc's - # nfsmount, which cannot resolve hostnames ("nfsmount: can't parse IP address - # ''"), so resolve instserver to its IP. The ds= URL is fetched later by - # cloud-init in the booted live system where normal DNS works, so it may stay a - # hostname. - # 'toram' makes casper copy the live squashfs into RAM and UNMOUNT the NFS source - # (casper scripts/casper: copy_to_ram then `umount ${copyfrom}`), so the installer runs - # from RAM with NO network root. This fixes the end-of-install reboot hang: with the NFS - # root still mounted, a process doing I/O to it during systemd-shutdown (lvm, netplan, - # udev) blocks in uninterruptible D state -- it cannot be SIGKILLed, so systemd-shutdown - # waits forever ("Waiting for process: (lvm)") and the node never power-cycles into - # the installed disk. With the root in RAM there is nothing to wait on and the reboot - # completes on its own. (casper has no cmdline knob for NFS mount options -- it parses - # only nfsroot=, taking the whole value as the path, so appending ,soft breaks the mount; - # toram is casper's supported way to avoid the network root. The 24.04 layers total - # ~1.5G, well within the CN's RAM.) my $nfsip = xCAT::NetworkUtils->getipaddr($instserver) || $instserver; - $kcmdline .= " autoinstall ip=dhcp boot=casper netboot=nfs nfsroot=${nfsip}:${pkgdir} toram"; - $kcmdline .= " ds=nocloud-net;s=http://${instserver}:${httpport}/install/autoinst/${node}/"; - $kcmdline .= " ---"; + $kcmdline = subiquity_kcmdline($kcmdline, $nfsip, $pkgdir, $instserver, $httpport, $node); } else { $kcmdline .= " url=http://${instserver}:$httpport/install/autoinst/$node"; $kcmdline .= " mirror/http/hostname=${instserver}:$httpport"; diff --git a/xCAT-test/unit/debian_subiquity_netboot.t b/xCAT-test/unit/debian_subiquity_netboot.t index 5b3b3029c..0f512cd0d 100644 --- a/xCAT-test/unit/debian_subiquity_netboot.t +++ b/xCAT-test/unit/debian_subiquity_netboot.t @@ -1,47 +1,74 @@ #!/usr/bin/env perl use strict; use warnings; + +use FindBin; use Test::More; -# Regression: the Ubuntu (subiquity) diskful install netboots the live installer over NFS. -# The kernel command line built in debian.pm must satisfy TWO casper requirements or the -# installer never boots and the node PXE-loops forever (surfacing to the test as -# "ssh: connect ... port 22: Connection refused"): +# The Ubuntu live installer (Subiquity) is booted over NFS. Three things on the kernel command +# line decide whether it boots at all and whether the node can reboot into the disk afterwards: # -# 1. boot=casper -- without it casper does not process netboot=nfs at all; it scans the -# local disks, finds nothing, and panics "Unable to find a medium containing a live -# file system", dropping to the initramfs emergency shell. +# boot=casper without it casper never processes netboot=nfs -- it scans local disks, finds no +# live media and panics into the initramfs shell, which PXE-loops. +# nfsroot=IP casper mounts the live filesystem with klibc's nfsmount, which has no resolver, +# so a hostname there fails with "can't parse IP address". +# toram casper copies the squashfs to RAM and unmounts the NFS source. Without it the +# NFS root stays mounted, systemd-shutdown blocks forever on I/O to it and the +# node never power-cycles into the disk it just installed. # -# 2. nfsroot must be a literal IP, NOT a hostname. casper mounts the live filesystem with -# klibc's nfsmount, which does NOT resolve hostnames -- it fails with -# "nfsmount: can't parse IP address ''". (busybox/util-linux `mount -t nfs` -# resolves names, which is why manual mounts work while casper's does not.) The instserver -# must therefore be passed through xCAT::NetworkUtils->getipaddr() before it is put in -# nfsroot=. The ds=...http URL is fetched later by cloud-init in the booted live system, -# where normal DNS works, so only nfsroot needs the IP. +# Build the command line for real and inspect it, rather than reading the source that builds it. -sub slurp { my ($p) = @_; local $/; open my $fh, '<', $p or return undef; <$fh> } +use lib "$FindBin::Bin/../../perl-xCAT"; +use lib "$FindBin::Bin/../../xCAT-server/lib/perl"; +my $plugin = "$FindBin::Bin/../../xCAT-server/lib/xcat/plugins/debian.pm"; +plan skip_all => 'debian.pm not found' unless -r $plugin; +eval { require $plugin; 1 } or plan skip_all => "could not load debian.pm: $@"; -my $deb = slurp('xCAT-server/lib/xcat/plugins/debian.pm'); -plan skip_all => 'debian.pm not found' unless defined $deb; +my $cmdline = xCAT_plugin::debian::subiquity_kcmdline( + 'nofb utf8 auto xcatd=xcatmn', # what mkinstall has built so far + '10.0.0.1', # instserver resolved to a literal IP + '/install/ubuntu24.04/x86_64', # pkgdir exported over NFS + 'xcatmn', # instserver by name + '80', # httpport + 'node01', # node +); -# The exact netboot fragment is unique to the subiquity install cmdline, so match it directly -# instead of trying to slice one of the several using_subiquity() blocks. -like($deb, qr/\bboot=casper\b/, - 'subiquity netboot kcmdline includes boot=casper (else casper scans local disks and panics)'); -like($deb, qr/autoinstall ip=dhcp boot=casper netboot=nfs nfsroot=\$\{nfsip\}:/, - 'subiquity kcmdline is: autoinstall ip=dhcp boot=casper netboot=nfs nfsroot='); -like($deb, qr/getipaddr\(\$instserver\)/, - 'instserver is resolved to an IP via getipaddr for nfsroot (klibc nfsmount cannot resolve hostnames)'); -unlike($deb, qr/nfsroot=\$\{instserver\}:/, - 'nfsroot does NOT use the bare instserver hostname (would break klibc nfsmount)'); +# --- the three settings the installer cannot boot without ------------------ +like($cmdline, qr/(?:^| )boot=casper(?: |$)/, + 'casper is told to boot, so it processes netboot=nfs instead of scanning disks'); +like($cmdline, qr{(?:^| )nfsroot=10\.0\.0\.1:/install/ubuntu24\.04/x86_64(?: |$)}, + 'nfsroot names the install server by IP, which klibc nfsmount can parse'); +like($cmdline, qr/(?:^| )toram(?: |$)/, + 'toram copies the live filesystem to RAM so the NFS root is unmounted before shutdown'); -# The subiquity netboot cmdline must include 'toram' so casper copies the squashfs into RAM and -# unmounts the NFS source -- otherwise the end-of-install reboot wedges in systemd-shutdown on a -# D-state process doing I/O to the still-mounted NFS root, and the node never boots the installed disk. -like($deb, qr/\btoram\b/, - 'subiquity netboot cmdline includes toram (run from RAM, no NFS root at shutdown -> no reboot hang)'); -unlike($deb, qr/nfsroot=[^ ]*,soft/, - 'nfsroot does NOT append ,soft (casper takes the whole nfsroot value as the path, which breaks the mount)'); +unlike($cmdline, qr/nfsroot=xcatmn:/, + 'nfsroot never carries a hostname, which klibc nfsmount cannot resolve'); +unlike($cmdline, qr/nfsroot=[^ ]*,/, + 'no mount options are appended to nfsroot -- casper takes the whole value as the path'); + +# --- the rest of the line --------------------------------------------------- +like($cmdline, qr/(?:^| )autoinstall(?: |$)/, 'the installer runs unattended'); +like($cmdline, qr/(?:^| )ip=dhcp(?: |$)/, 'the live system configures its NIC by DHCP'); +like($cmdline, qr/(?:^| )netboot=nfs(?: |$)/, 'the live filesystem is fetched over NFS'); + +# cloud-init fetches the seed later, in the booted live system, where DNS works -- so this one +# keeps the install server's name rather than its address. +like($cmdline, qr{(?:^| )ds=nocloud-net;s=http://xcatmn:80/install/autoinst/node01/(?: |$)}, + 'the cloud-init seed URL addresses the install server by name'); + +is((split / /, $cmdline)[-1], '---', + 'the line ends with the separator that divides installer arguments from kernel arguments'); + +# What mkinstall had already built is preserved, not replaced. +like($cmdline, qr/^nofb utf8 auto xcatd=xcatmn /, + 'the command line built so far is kept ahead of the installer arguments'); + +# A non-default HTTP port reaches the seed URL. +{ + my $alt = xCAT_plugin::debian::subiquity_kcmdline( + 'base', '10.0.0.1', '/pkgdir', 'xcatmn', '8080', 'node02'); + like($alt, qr{ds=nocloud-net;s=http://xcatmn:8080/install/autoinst/node02/}, + 'the seed URL carries the configured HTTP port and node'); +} done_testing(); diff --git a/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t b/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t index 00cde9e77..e0e935ffb 100644 --- a/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t +++ b/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t @@ -2,58 +2,94 @@ use strict; use warnings; -use File::Spec; use FindBin; use Test::More; -# Regression: on Ubuntu/Debian the DISKFUL install runs a node's postscripts inside the -# installer's in-target chroot -- before the node has booted as itself. The syncfiles -# postscript works by asking the management node to scp files INTO the running node, which -# cannot happen in that phase: the not-yet-booted node has no sshd for the MN to reach, so the -# push times out, syncfiles exits 1, and the node reports status=failed even though the OS -# installed perfectly. -# -# syncfiles must therefore move to the postbootscripts set, which runs on the booted node -# where ssh is already listening. Two things bound that move: -# -# * It applies to the DISKFUL install path only. netboot and statelite already run their -# postscripts on the booted node, so moving syncfiles there changes working behaviour for -# no reason. EL/SLES are unaffected either way -- their postscripts already run on the -# booted node. -# * syncfiles must run BEFORE the existing postbootscripts, since a postbootscript may -# consume the files it synchronises. Appending it would invert that. +# On the Ubuntu/Debian DISKFUL install path a node's postscripts run inside the installer's +# in-target chroot, before the node has booted as itself. syncfiles asks the management node to +# scp files INTO the running node, which cannot work there -- no sshd yet -- so it times out and +# the node reports status=failed even though the OS installed fine. Postage defers it to the +# postbootscripts, which run on the booted node. Drive that decision directly. -my $repo_root = File::Spec->rel2abs( - File::Spec->catdir( $FindBin::Bin, '..', '..' ) +my $repo = "$FindBin::Bin/../.."; +plan skip_all => 'Postage.pm not found' unless -r "$repo/xCAT-server/lib/perl/xCAT/Postage.pm"; + +use lib "$FindBin::Bin/../../perl-xCAT"; +use lib "$FindBin::Bin/../../xCAT-server/lib/perl"; +eval { require xCAT::Postage; 1 } or plan skip_all => "could not load xCAT::Postage: $@"; + +my $DEFERRED = "# ubuntu-deferred-postbootscripts-start-here\nsyncfiles\n" + . "# ubuntu-deferred-postbootscripts-end-here\n"; + +sub defer { return xCAT::Postage::defer_syncfiles_to_postboot(@_) } + +# --- the case the fix exists for ------------------------------------------- +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', 'install', "syncfiles\notherpkgs\n", "setupntp\n"); + + is($post, "otherpkgs\n", 'syncfiles is removed from the postscripts'); + is($postboot, $DEFERRED . "setupntp\n", + 'syncfiles is prepended to the postbootscripts, ahead of what consumes its files'); +} + +# The nodeset state is not always known; the osimage provmethod decides then. +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', undef, "syncfiles\n", "setupntp\n"); + is($post, '', 'provmethod=install defers when no nodeset state is given'); + is($postboot, $DEFERRED . "setupntp\n", 'and the postbootscripts receive it'); +} + +# --- paths that must NOT change -------------------------------------------- +my @untouched = ( + [ 'netboot keeps its postscripts on the booted node already', + 'ubuntu24.04', 'netboot', 'netboot' ], + [ 'statelite is unaffected', + 'ubuntu22.04', 'statelite', 'statelite' ], + [ 'EL runs postscripts on the booted node, so nothing moves', + 'rhels9.4', 'install', 'install' ], + [ 'SLES is unaffected', + 'sles15.6', 'install', 'install' ], + [ 'an undefined os is left alone', + undef, 'install', 'install' ], ); -my $path = File::Spec->catfile( $repo_root, 'xCAT-server', 'lib', 'perl', 'xCAT', 'Postage.pm' ); -plan skip_all => "Postage.pm not found" unless -f $path; +foreach my $case (@untouched) { + my ($name, $os, $provmethod, $state) = @$case; + my ($post, $postboot) = defer($os, $provmethod, $state, "syncfiles\notherpkgs\n", "setupntp\n"); + is($post, "syncfiles\notherpkgs\n", "$name: the postscripts are unchanged"); + is($postboot, "setupntp\n", "$name: the postbootscripts are unchanged"); +} -my $src = do { local $/; open my $fh, '<', $path or die $!; <$fh> }; +# A node that does not run syncfiles must not gain an empty deferral block. +{ + my ($post, $postboot) = defer('ubuntu24.04', 'install', 'install', "otherpkgs\n", "setupntp\n"); + is($post, "otherpkgs\n", 'a node without syncfiles keeps its postscripts'); + is($postboot, "setupntp\n", 'and gains no deferral block'); +} -# Isolate the deferral block so the assertions below cannot accidentally match code elsewhere. -my ($block) = $src =~ /(\n[^\n]*ubuntu\|debian[^\n]*\n(?:.*?\n)*?[^\n]*syncfiles(?:.*?\n)*?\s*\}\n\s*\}\n)/; -ok( defined $block, 'found the syncfiles deferral block in makescript' ) - or do { done_testing(); exit }; +# The name is matched whole: a postscript whose name merely contains "syncfiles" stays put. +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', 'install', "syncfiles2\nmysyncfiles\n", "setupntp\n"); + is($post, "syncfiles2\nmysyncfiles\n", 'a lookalike postscript name is not deferred'); + is($postboot, "setupntp\n", 'and nothing is added to the postbootscripts'); +} -like( $block, qr/\$os\s*=~\s*.\^\(\?:ubuntu\|debian\)/, - 'the move is guarded to ubuntu/debian nodes only' ); +# Indented entries are still the syncfiles postscript. +{ + my ($post, $postboot) = defer('ubuntu24.04', 'install', 'install', " syncfiles \nfoo\n", undef); + is($post, "foo\n", 'an indented syncfiles entry is deferred'); + is($postboot, $DEFERRED, 'an undefined postbootscripts list becomes the deferral block'); +} -like( $block, qr/\$diskful_install|\bnodesetstate\b|\bprovmethod\b/, - 'the move is guarded to the diskful install path (not netboot/statelite)' ); - -like( $block, qr/\$postscripts\s*=~\s*s\/\^\[[^\]]*\]\*syncfiles/, - 'syncfiles is stripped from the in-target postscripts list' ); - -like( $block, qr/\$postbootscripts\s*=\s*"[^"]*syncfiles[^"]*"\s*\.\s*\$postbootscripts/, - 'syncfiles is PREPENDED to postbootscripts, so it runs before scripts that consume its files' ); - -unlike( $block, qr/\$postbootscripts\s*\.=\s*"syncfiles/, - 'syncfiles is not appended after the existing postbootscripts' ); - -# The strip and the re-add must be paired, so a node with no syncfiles postscript never -# acquires a spurious one. -like( $block, qr/if\s*\(.*?\$postscripts.*?syncfiles.*?\)\s*\{\s*.*?\$postbootscripts/s, - 'syncfiles is only added to postbootscripts when it was present in postscripts' ); +# Rendering the same node twice must not stack a second copy. +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', 'install', "syncfiles\nfoo\n", "setupntp\n"); + my ($post2, $postboot2) = defer('ubuntu24.04', 'install', 'install', $post, $postboot); + is($post2, $post, 'a second pass leaves the postscripts alone'); + is($postboot2, $postboot, 'a second pass does not duplicate syncfiles'); +} done_testing(); diff --git a/xCAT-test/unit/ubuntu_resolvconf_ip.t b/xCAT-test/unit/ubuntu_resolvconf_ip.t index d35600d8b..4a1f1ed8f 100644 --- a/xCAT-test/unit/ubuntu_resolvconf_ip.t +++ b/xCAT-test/unit/ubuntu_resolvconf_ip.t @@ -1,42 +1,79 @@ #!/usr/bin/env perl use strict; use warnings; + +use FindBin; +use File::Temp qw(tempdir); use Test::More; -# Regression: the Ubuntu (subiquity) diskful install writes the installer's /etc/resolv.conf -# from compute.subiquity.tmpl's "DNS setup" early-command. A nameserver line in -# /etc/resolv.conf MUST be an IP address -- glibc's resolver does NOT resolve a hostname -# given on a nameserver line, it simply discards the entry. The template used to write the -# xcatmaster *name* there: +# A nameserver line in /etc/resolv.conf must hold an IP address: glibc's resolver discards an +# entry naming a host. Writing the xcatmaster *name* left the installer -- and the in-target +# apt-get that inherits the file -- with no usable DNS, so the install hung resolving +# archive.ubuntu.com. The template resolves the name to an address first. # -# echo "nameserver #TABLE:noderes:$NODE:xcatmaster#" >/etc/resolv.conf -# -# which left the installer (and the in-target apt-get update, which inherits this resolv.conf) -# with no usable DNS. The install then hangs resolving archive.ubuntu.com -- the node never -# finishes, and the test sees "ssh: connect ... port 22: Connection refused". (Everything the -# early-command did BEFORE this point still worked because the live installer's DHCP-provided -# resolv.conf can resolve the xcatmaster name -- so the failure only surfaces at the very next -# DNS-dependent step, the in-target apt.) -# -# The fix resolves the xcatmaster hostname to an IPv4 address (via getent, while DNS still -# works) and writes that IP into resolv.conf. +# Run the template's own shell for that step and inspect the file it writes. -sub slurp { my ($p) = @_; local $/; open my $fh, '<', $p or return undef; <$fh> } +my $tmpl = "$FindBin::Bin/../../xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl"; +plan skip_all => 'compute.subiquity.tmpl not found' unless -r $tmpl; -my $tmpl = slurp('xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl'); -plan skip_all => 'compute.subiquity.tmpl not found' unless defined $tmpl; +open(my $fh, '<', $tmpl) or die "open $tmpl: $!"; +my $source = do { local $/; <$fh> }; +close $fh; -# The old, broken form -- a bare hostname token written straight into the nameserver line -- -# must be gone. -unlike($tmpl, qr/nameserver \s+ \#TABLE:noderes:\$NODE:xcatmaster\# /x, - 'resolv.conf nameserver is NOT the bare xcatmaster hostname token (glibc cannot resolve a name there)'); +my ($fragment) = $source =~ m{^(\s*xcatmaster_host=.*?echo "nameserver [^\n]*\n)}ms; +BAIL_OUT('the template does not build /etc/resolv.conf from the xcatmaster') unless $fragment; -# The xcatmaster name must be resolved to an IPv4 address before it is used as a nameserver. -like($tmpl, qr/getent \s+ ahostsv4 /x, - 'DNS setup resolves the xcatmaster to an IPv4 address via getent ahostsv4'); +# $NODE and the xcatmaster come from the xCAT template renderer; stand in for both. +sub write_resolv_conf { + my (%opt) = @_; + my $root = tempdir(CLEANUP => 1); -# And the nameserver line must be written from that resolved IP variable. -like($tmpl, qr/nameserver \s+ \$xcatmaster_ip/x, - 'resolv.conf nameserver is written from the resolved xcatmaster IP'); + my $script = $fragment; + $script =~ s/\#TABLE:noderes:\$NODE:xcatmaster\#/$opt{xcatmaster}/; + $script =~ s{/etc/resolv\.conf}{$root/resolv.conf}g; + + # getent is the resolver the fragment uses; make it answer as the test wants. + my $getent = $opt{resolves} + ? "getent() { printf '%s\\n' '$opt{resolves} $opt{xcatmaster}'; }\n" + : "getent() { return 2; }\n"; + + system('bash', '-c', $getent . $script) == 0 or return { rc => $? }; + + open my $rh, '<', "$root/resolv.conf" or return { rc => 0, content => '' }; + my $content = do { local $/; <$rh> }; + close $rh; + return { rc => 0, content => $content }; +} + +# --- the case the fix exists for ------------------------------------------- +{ + my $r = write_resolv_conf(xcatmaster => 'xcatmn', resolves => '10.0.0.1'); + is($r->{content}, "nameserver 10.0.0.1\n", + 'the nameserver line holds the address, which glibc will actually use'); + unlike($r->{content}, qr/nameserver \s+ xcatmn/x, + 'the nameserver line never holds a host name, which glibc discards'); +} + +# --- more than one address: the first is taken ----------------------------- +{ + my $r = write_resolv_conf(xcatmaster => 'xcatmn', resolves => '10.0.0.1'); + like($r->{content}, qr/^nameserver \d+\.\d+\.\d+\.\d+$/m, + 'exactly one IPv4 address is written'); +} + +# --- resolution fails: fall back rather than write an empty entry ---------- +{ + my $r = write_resolv_conf(xcatmaster => 'xcatmn'); + is($r->{content}, "nameserver xcatmn\n", + 'an unresolvable xcatmaster falls back to its name rather than an empty nameserver'); + unlike($r->{content}, qr/^nameserver\s*$/m, + 'no empty nameserver line is written'); +} + +# --- an xcatmaster already given as an address is left alone --------------- +{ + my $r = write_resolv_conf(xcatmaster => '10.0.0.1', resolves => '10.0.0.1'); + is($r->{content}, "nameserver 10.0.0.1\n", 'an address-valued xcatmaster is written as is'); +} done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t index 6d57d5e30..a916de89f 100644 --- a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t +++ b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t @@ -2,55 +2,124 @@ use strict; use warnings; -use File::Spec; use FindBin; +use File::Temp qw(tempdir); +use IO::Socket::INET; use Test::More; -# Regression: after Subiquity installs the OS the node MUST be flipped to local-disk boot, or it -# PXE-loops straight back into the installer and the installed OS -- with its sshd -- never -# boots. The diskful case then only ever reports -# "ssh: connect to host port 22: Connection refused". +# When Subiquity finishes it reboots, and unless the node has been flipped to local-disk boot it +# PXEs straight back into the installer. The in-target post-script does that through +# updateflag.awk, which needs gawk's |& coprocess -- Ubuntu's /usr/bin/awk is mawk, so the flip +# silently failed and the node reinstalled forever. The template now does the exchange from the +# live installer over bash's built-in /dev/tcp. # -# xCAT performs that flip when the node reports "next" to xcatd on the install-monitor port, -# which makes xcatd run "nodeset next" and rewrite the node's xNBA script to fall through -# to the local disk. The in-target post-script tries this through updateflag.awk, but that -# depends on gawk's |& / /inet coprocess and on /usr/bin/awk being gawk -- on Ubuntu -# /usr/bin/awk is normally mawk, which has neither, so the flip fails silently. -# -# The template must therefore trigger the flip itself from the live installer, with no gawk -# dependency, and must NOT report success when it did not happen. +# Run that command against a stand-in for xcatd and check the exchange, rather than reading the +# template text. -my $repo_root = File::Spec->rel2abs( - File::Spec->catdir( $FindBin::Bin, '..', '..' ) -); -my $path = File::Spec->catfile( - $repo_root, 'xCAT-server', 'share', 'xcat', 'install', 'ubuntu', 'compute.subiquity.tmpl' -); -plan skip_all => 'compute.subiquity.tmpl not found' unless -f $path; +my $tmpl = "$FindBin::Bin/../../xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl"; +plan skip_all => 'compute.subiquity.tmpl not found' unless -r $tmpl; -my $tmpl = do { local $/; open my $fh, '<', $path or die $!; <$fh> }; +my $XCATD_PORT = 3002; # the install-monitor port the template addresses -like( $tmpl, qr{/dev/tcp/\$xm/3002}, - 'the flip contacts xcatd on the install-monitor port from the installer' ); -like( $tmpl, qr{printf "next}, - 'it sends the "next" token that makes xcatd run "nodeset next"' ); -like( $tmpl, qr{xm=#XCATVAR:XCATMASTER#}, - "the flip targets the node's own xcatmaster" ); -like( $tmpl, qr{^\s*-\s*\['bash',\s*'-c',}m, - 'the flip runs under bash (dash has no /dev/tcp) via an argv list command' ); +my $probe = IO::Socket::INET->new( + LocalAddr => '127.0.0.1', LocalPort => $XCATD_PORT, Proto => 'tcp', + Listen => 5, ReuseAddr => 1); +plan skip_all => "port $XCATD_PORT is not available on the loopback interface" unless $probe; +close $probe; # each case below opens its own listener, or none at all -# A failed flip must be visible. The original form broke out of its retry loop as soon as the -# socket connected -- never on a confirmed exchange -- and ended in `true`, so a total failure -# looked exactly like success and the node silently reinstalled forever. -like( $tmpl, qr{ok=1}, - 'success is recorded only after the exchange completes, not merely on connect' ); -like( $tmpl, qr{FAILED to flip}, - 'a failed flip is reported into the install log rather than passing silently' ); +open(my $fh, '<', $tmpl) or die "open $tmpl: $!"; +my $source = do { local $/; <$fh> }; +close $fh; -# The magic-SysRq forced reboot must be gone: toram already unmounts the NFS live root, so the -# shutdown hang it worked around cannot occur, and an unconditional timed reboot would race a -# slow install. -unlike( $tmpl, qr{sysrq-trigger}, - 'no unconditional magic-SysRq reboot (toram removes the hang it worked around)' ); +# The boot flip is the late-command that talks to the install-monitor port. +my ($command) = $source =~ m{- \['bash', '-c', '(.*?/dev/tcp/.*?)'\]}; +BAIL_OUT('no late-command in the template performs the boot flip over /dev/tcp') unless $command; + +# Run the command with the install server pointed at our stand-in, and its log inside a scratch +# tree. Everything else is the template's own text. +sub run_flip { + my (%opt) = @_; + my $root = tempdir(CLEANUP => 1); + mkdir "$root/target"; mkdir "$root/target/var"; mkdir "$root/target/var/log"; + mkdir "$root/target/var/log/xcat"; + + my $script = $command; + $script =~ s/\#XCATVAR:XCATMASTER\#/127.0.0.1/; + $script =~ s{/target/var/log/xcat/xcat\.log}{$root/target/var/log/xcat/xcat.log}; + $script =~ s/sleep 5/sleep 1/; # shorten the retry pause, keep the retry + + my $pid; + if ($opt{listen}) { + $pid = fork(); + die "fork: $!" unless defined $pid; + if (!$pid) { # the stand-in xcatd + sleep $opt{delay} if $opt{delay}; # appear only after the first attempts fail + my $srv = IO::Socket::INET->new(LocalAddr => '127.0.0.1', LocalPort => $XCATD_PORT, + Proto => 'tcp', Listen => 5, ReuseAddr => 1) or exit 1; + open my $seen, '>', "$root/received" or exit 1; + $seen->autoflush(1); + for (1 .. $opt{listen}) { + my $c = $srv->accept() or last; + $c->autoflush(1); + print {$c} "ready\n"; + my $line = <$c>; + print {$seen} $line if defined $line; + print {$c} "ok\n" unless $opt{no_ack}; + close $c; + } + close $seen; + close $srv; + exit 0; + } + } + + # The installer would hang here if the exchange ever blocked, so bound it. + # the no-listener case prints "Connection refused" by design + my $rc = system("timeout 25 bash -c \Q$script\E 2>/dev/null"); + my $timed_out = (($rc >> 8) == 124); + if ($pid) { kill 'TERM', $pid; waitpid($pid, 0) } + + my $received = ''; + if (open my $rh, '<', "$root/received") { local $/; $received = <$rh>; close $rh } + + my $log = ''; + if (open my $lh, '<', "$root/target/var/log/xcat/xcat.log") { local $/; $log = <$lh>; close $lh } + return { rc => $rc, timed_out => $timed_out, log => $log, received => $received }; +} + +# --- xcatd answers: the node is flipped ------------------------------------ +{ + my $r = run_flip(listen => 1); + is($r->{rc}, 0, 'the boot flip exits cleanly so the install is not failed by it'); + is($r->{log}, '', 'nothing is written to the install log when the flip succeeds'); + is($r->{received}, "next\n", + 'the node sends the token that makes xcatd run "nodeset next"'); + ok(!$r->{timed_out}, 'the exchange completes rather than hanging the late-command'); +} + +# --- xcatd never answers: the failure is recorded, not swallowed ----------- +{ + my $r = run_flip(listen => 0); + is($r->{rc}, 0, 'a failed flip still exits 0 rather than aborting the install'); + like($r->{log}, qr/FAILED to flip/, + 'a failed flip is recorded in the install log instead of PXE-looping silently'); + like($r->{log}, qr/127\.0\.0\.1:3002/, + 'the log names the install server and port that could not be reached'); +} + +# --- xcatd accepts but never acknowledges ---------------------------------- +{ + my $r = run_flip(listen => 1, no_ack => 1); + like($r->{log}, qr/FAILED to flip/, + 'a connection without an acknowledgement counts as a failure, not a success'); +} + +# --- the command retries rather than giving up on the first refusal -------- +{ + # Answer only on a later connection: the flip must still succeed. + my $r = run_flip(listen => 1, delay => 2); + is($r->{log}, '', 'the flip retries until the install monitor answers'); + is($r->{received}, "next\n", 'and the token still reaches it on the later attempt'); +} done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_bootparams.t b/xCAT-test/unit/ubuntu_subiquity_bootparams.t deleted file mode 100644 index fccc9502f..000000000 --- a/xCAT-test/unit/ubuntu_subiquity_bootparams.t +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env perl -use strict; -use warnings; -use Test::More; - -my $plugin_path = defined $ENV{XCATROOT} ? "$ENV{XCATROOT}/lib/perl/xCAT_plugin/debian.pm" : ''; -$plugin_path = "xCAT-server/lib/xcat/plugins/debian.pm" - unless -f $plugin_path; - -plan skip_all => "debian.pm not found" unless -f $plugin_path; - -my $src = do { local $/; open my $fh, '<', $plugin_path or die $!; <$fh> }; - -like($src, qr/autoinstall ip=dhcp boot=casper netboot=nfs/, - 'subiquity bootparams enable autoinstall (boot=casper is required for casper to process netboot=nfs)'); -like($src, qr/ds=nocloud-net;s=http:\/\//, 'subiquity bootparams point at NoCloud seed'); -like($src, qr/\$kcmdline\s*\.=\s*" ---";/, 'subiquity bootparams end with installer argument separator'); - -done_testing(); From a0c8f7eae78c8d8041444eabeebeb128f330cbeb Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:00:57 -0300 Subject: [PATCH 04/21] fix(xcat-core): trim the commentary around the Subiquity diskful fix Three passes of the same reasoning had accumulated: in the code, in the POD of the routines the previous commit extracted, and again in the test headers. Say each once, where the reader needs it. The POD blocks were the worst of it -- extracting subiquity_kcmdline() and defer_syncfiles_to_postboot() moved the essays out of the routines but did not shorten them. The toram paragraph ran eleven lines for one kernel argument; the exact systemd-shutdown message and the size of the 24.04 layers are colour, not reasoning. The template's two comments and the apt-sources pair are trimmed the same way: the sources.list explanation lives in Template.pm, and the test says which case it is checking and points there. That the tests still pass after rewriting the template is the point of the previous commit -- a test matching its text would have broken here. 47 comment and POD lines removed, no behaviour change. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-server/lib/perl/xCAT/Postage.pm | 20 ++++++-------- xCAT-server/lib/perl/xCAT/Template.pm | 16 ++++------- xCAT-server/lib/xcat/plugins/debian.pm | 27 +++++++------------ .../install/ubuntu/compute.subiquity.tmpl | 23 +++++----------- xCAT-test/unit/ubuntu_subiquity_apt_sources.t | 15 +++-------- 5 files changed, 33 insertions(+), 68 deletions(-) diff --git a/xCAT-server/lib/perl/xCAT/Postage.pm b/xCAT-server/lib/perl/xCAT/Postage.pm index 3e009b684..5fda1acfd 100644 --- a/xCAT-server/lib/perl/xCAT/Postage.pm +++ b/xCAT-server/lib/perl/xCAT/Postage.pm @@ -177,19 +177,15 @@ my $mn; =head3 defer_syncfiles_to_postboot - On Ubuntu/Debian the DISKFUL install runs a node's postscripts inside the installer's - in-target chroot -- before the node has booted as itself. The syncfiles postscript works - by asking the management node to scp files INTO the running node, which cannot happen in - that phase: the not-yet-booted node has no sshd for the MN to reach, so the push times - out, syncfiles exits 1, and the node reports status=failed even though the OS installed - perfectly. Defer it to the postbootscripts, which run on the booted node where ssh is - already listening and the MN's push succeeds. + On the Ubuntu/Debian diskful install path a node's postscripts run inside the installer's + in-target chroot, before the node has booted as itself. syncfiles asks the management node + to scp files INTO the running node, which cannot work there -- there is no sshd yet -- so it + times out and the node reports status=failed although the OS installed. Defer it to the + postbootscripts, which run on the booted node. - Scoped to the diskful install path only: netboot and statelite already run their - postscripts on the booted node, so moving syncfiles there would change behaviour that - works. EL/SLES are unaffected either way -- their postscripts run on the booted node. - syncfiles is PREPENDED so it still runs before any postbootscript that consumes the - files it synchronises. + Scoped to the diskful install path: netboot and statelite already run their postscripts on + the booted node, and EL/SLES are unaffected either way. syncfiles is prepended so it still + runs before any postbootscript that consumes the files it synchronises. Arguments: $os nodetype.os for the node diff --git a/xCAT-server/lib/perl/xCAT/Template.pm b/xCAT-server/lib/perl/xCAT/Template.pm index a1f71aafd..920a99448 100644 --- a/xCAT-server/lib/perl/xCAT/Template.pm +++ b/xCAT-server/lib/perl/xCAT/Template.pm @@ -1686,17 +1686,11 @@ sub ubuntu_subiquity_apt_config ' primary:', " - uri: $online_mirror", ); - # On a classic-sources release (20.04/22.04) Subiquity renders the target's - # /etc/apt/sources.list from the install media alone (deb file:///cdrom), which lacks - # packages xCAT needs such as chrony -- curtin's in-target apt then fails with - # "E: Unable to locate package chrony". `sources_list:` is a curtin key that - # Subiquity's autoinstall schema ignores, so add the online archive through `sources:`, - # which Subiquity honours by writing /etc/apt/sources.list.d/*.list. $RELEASE is - # substituted with the release codename by curtin. - # - # Deb822 releases (24.04+) are deliberately excluded: there the primary mirror above - # already lands in /etc/apt/sources.list.d/ubuntu.sources, so adding these legacy .list - # files would configure the same suites twice. + # On 20.04/22.04 Subiquity renders the target's sources.list from the install media + # alone, so in-target apt cannot find packages the ISO does not carry (chrony). Add the + # online archive through `sources:` -- `sources_list:` is a curtin key Subiquity's + # schema ignores. Excluded on Deb822 releases, where the primary mirror already lands + # in ubuntu.sources and these legacy .list files would duplicate the same suites. my $need_sources_block = !$use_deb822; if ($need_sources_block) { push @lines, ' sources:'; diff --git a/xCAT-server/lib/xcat/plugins/debian.pm b/xCAT-server/lib/xcat/plugins/debian.pm index 4c7987195..1b039836d 100644 --- a/xCAT-server/lib/xcat/plugins/debian.pm +++ b/xCAT-server/lib/xcat/plugins/debian.pm @@ -513,25 +513,18 @@ sub copycd Build the kernel command line for a Subiquity (Ubuntu live installer) diskful install. - boot=casper is required: without it casper never processes netboot=nfs, it scans the local - disks, finds no live media and panics "Unable to find a medium containing a live file - system" (initramfs emergency shell -> PXE loop). + boot=casper: without it casper never processes netboot=nfs -- it scans the local disks, + finds no live media and panics into the initramfs shell, which PXE-loops. - nfsroot MUST be a literal IP: casper mounts the live filesystem with klibc's nfsmount, which - cannot resolve hostnames ("nfsmount: can't parse IP address ''"). The ds= URL is - fetched later by cloud-init in the booted live system where normal DNS works, so it keeps - the install server's name. + nfsroot must be a literal IP: casper mounts the live filesystem with klibc's nfsmount, + which has no resolver. The ds= URL is fetched later by cloud-init, where DNS works, so it + keeps the install server's name. - 'toram' makes casper copy the live squashfs into RAM and UNMOUNT the NFS source (casper - scripts/casper: copy_to_ram then umount of the copy source), so the installer runs from RAM - with NO network root. This is what lets the node reboot at all: with the NFS root still - mounted, a process doing I/O to it during systemd-shutdown (lvm, netplan, udev) blocks in - uninterruptible D state -- it cannot be SIGKILLed, so systemd-shutdown waits forever - ("Waiting for process: (lvm)") and the node never power-cycles into the disk it just - installed. casper has no cmdline knob for NFS mount options -- it parses only nfsroot=, - taking the whole value as the path, so appending ,soft breaks the mount; toram is casper's - supported way to avoid the network root. The 24.04 layers total ~1.5G, well within the CN's - RAM. + toram: casper copies the squashfs to RAM and unmounts the NFS source, so nothing holds the + network root at shutdown. Without it a process doing I/O to it blocks uninterruptibly, + systemd-shutdown waits forever and the node never reboots into the disk it just installed. + casper parses only nfsroot= and takes the whole value as the path, so mount options cannot + be appended; toram is its supported alternative. Arguments: $base the command line built so far diff --git a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl index 547a48aa1..783ebc2de 100644 --- a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl +++ b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl @@ -66,12 +66,9 @@ autoinstall: sed -i '/^\.\.\.$/d' /autoinstall.yaml cat /tmp/partitionfile >> /autoinstall.yaml echo "=== DNS setup ===" - # A nameserver line in /etc/resolv.conf MUST hold an IP address -- glibc's resolver does - # NOT resolve a hostname written there, it discards the entry. Writing the xcatmaster - # *name* leaves the installer (and the in-target apt-get, which inherits this file) with - # no usable DNS, so the install hangs resolving archive.ubuntu.com. DNS still works at - # this point through the live installer's DHCP resolv.conf -- the wget calls above relied - # on it -- so resolve the xcatmaster to an IPv4 address first and write that. + # glibc's resolver discards a nameserver line naming a host, so writing the xcatmaster + # *name* leaves the installer -- and the in-target apt that inherits this file -- with no + # DNS. Resolve it here, while the live installer's DHCP resolv.conf still works. xcatmaster_host="#TABLE:noderes:$NODE:xcatmaster#" xcatmaster_ip="$(getent ahostsv4 "$xcatmaster_host" | awk '{print $1; exit}')" [ -n "$xcatmaster_ip" ] || xcatmaster_ip="$xcatmaster_host" @@ -110,16 +107,10 @@ autoinstall: cp ./#HOSTNAME#.post /target/root/post.script; curtin in-target --target /target /root/post.script; } >>/target/var/log/xcat/xcat.log 2>&1' - # Switch the node to boot from the local disk now the install is done, or it PXE-loops - # straight back into the installer when Subiquity reboots. xCAT flips the netboot state - # when the node reports "next" to xcatd on the install-monitor port, which makes xcatd run - # "nodeset next" and rewrite the node's xNBA script to fall through to local disk. - # The in-target post-script tries this through updateflag.awk, but that needs gawk's |& - # /inet coprocess and /usr/bin/awk on Ubuntu is normally mawk, so the flip silently fails. - # Send it from the live installer instead -- the connection then originates from the node's - # own IP, which xcatd trusts -- using bash's built-in /dev/tcp, with no gawk dependency. - # xcatd answers with a ready token, takes "next", and acknowledges; treat anything else as - # a failure and say so in the install log rather than rebooting into another install. + # Flip the node to local-disk boot, or it PXE-loops back into the installer on reboot. + # The in-target post-script does this through updateflag.awk, which needs gawk's |& and + # Ubuntu's /usr/bin/awk is mawk, so it silently failed. Send "next" to xcatd from the live + # installer over bash's /dev/tcp instead, and log a failure rather than reinstalling. - ['bash', '-c', 'xm=#XCATVAR:XCATMASTER#; ok=0; for i in 1 2 3 4 5; do if exec 3<>/dev/tcp/$xm/3002; then read -r r <&3 || true; printf "next\n" >&3; if read -r r <&3; then ok=1; fi; exec 3>&- 3<&-; [ "$ok" = 1 ] && break; fi; sleep 5; done; if [ "$ok" != 1 ]; then echo "xcat: FAILED to flip $(hostname) to local-disk boot via $xm:3002; the node will PXE back into the installer" >>/target/var/log/xcat/xcat.log; fi; exit 0'] error-commands: - tar -c --ignore-failed-read --transform='s/^/#HOSTNAME#-logs\//' /var/crash /var/log/installer /tmp/pre-install.log /autoinstall.yaml 2>/dev/null |nc -l 8080 diff --git a/xCAT-test/unit/ubuntu_subiquity_apt_sources.t b/xCAT-test/unit/ubuntu_subiquity_apt_sources.t index f857af48e..0c55d0e8c 100644 --- a/xCAT-test/unit/ubuntu_subiquity_apt_sources.t +++ b/xCAT-test/unit/ubuntu_subiquity_apt_sources.t @@ -8,18 +8,9 @@ use lib "$FindBin::Bin/../../xCAT-server/lib/perl"; use Test::More; -# Regression: on an ONLINE Subiquity install of a classic-sources release (20.04 / 22.04), -# Subiquity renders the target's /etc/apt/sources.list from the install media alone -# ("deb file:///cdrom"). A live-server ISO does not carry everything xCAT's postscripts need -- -# chrony, for one -- so curtin's in-target apt fails with "E: Unable to locate package chrony" -# and the install crashes. `sources_list:` is a curtin key that Subiquity's autoinstall schema -# ignores, so the online archive has to be added through `sources:`, which Subiquity honours by -# writing /etc/apt/sources.list.d/*.list. -# -# That fix belongs to the classic-sources releases ONLY. On a Deb822 release (24.04 / 26.04) -# the primary mirror already lands in /etc/apt/sources.list.d/ubuntu.sources, so adding legacy -# .list files on top produces duplicate apt entries for the same suites -- exactly the source -# conflict the Deb822 branch elsewhere in this function goes out of its way to avoid. +# An online Subiquity install of a classic-sources release needs the online archive added, or +# in-target apt cannot find chrony and the install crashes. A Deb822 release must NOT get it, +# or the same suites are configured twice. See the comment in Template.pm. require xCAT::Template; From 1cbdad25312bd63aa56a20e7d1ef8b2560b6fcf9 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:38:44 -0300 Subject: [PATCH 05/21] fix(xcat-core): correct the boot-flip comment The comment blamed mawk: updateflag.awk needs gawk's |& and Ubuntu's /usr/bin/awk is mawk. That is wrong on this path -- compute.subiquity.tmpl has installed gawk in the target since Ubuntu 20.04 support was added, so updateflag.awk would have found it. The flip was going unanswered because xcatd's install monitor, which serves port 3002, dies and is never re-forked. That is fixed in #7759. The /dev/tcp exchange here is hardening: it retries and reports, rather than reinstalling in silence. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- .../share/xcat/install/ubuntu/compute.subiquity.tmpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl index 783ebc2de..f82aec459 100644 --- a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl +++ b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl @@ -108,9 +108,9 @@ autoinstall: curtin in-target --target /target /root/post.script; } >>/target/var/log/xcat/xcat.log 2>&1' # Flip the node to local-disk boot, or it PXE-loops back into the installer on reboot. - # The in-target post-script does this through updateflag.awk, which needs gawk's |& and - # Ubuntu's /usr/bin/awk is mawk, so it silently failed. Send "next" to xcatd from the live - # installer over bash's /dev/tcp instead, and log a failure rather than reinstalling. + # "next" is the request xcatd's install monitor answers with "nodeset next". Send it + # from the live installer, retried, and log a failure rather than silently reinstalling. A + # monitor that died and never came back is a separate problem; see #7759. - ['bash', '-c', 'xm=#XCATVAR:XCATMASTER#; ok=0; for i in 1 2 3 4 5; do if exec 3<>/dev/tcp/$xm/3002; then read -r r <&3 || true; printf "next\n" >&3; if read -r r <&3; then ok=1; fi; exec 3>&- 3<&-; [ "$ok" = 1 ] && break; fi; sleep 5; done; if [ "$ok" != 1 ]; then echo "xcat: FAILED to flip $(hostname) to local-disk boot via $xm:3002; the node will PXE back into the installer" >>/target/var/log/xcat/xcat.log; fi; exit 0'] error-commands: - tar -c --ignore-failed-read --transform='s/^/#HOSTNAME#-logs\//' /var/crash /var/log/installer /tmp/pre-install.log /autoinstall.yaml 2>/dev/null |nc -l 8080 From e62a66c70d6cf7efa090bd677f9648b4a542c5e2 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:41:42 -0300 Subject: [PATCH 06/21] docs(xcat-core): note the memory an Ubuntu diskful install needs toram copies the live filesystem into RAM, so the node needs memory for it on top of the installer -- about 1.5G on 24.04 and growing per release. Nothing enforced or documented that. A node with too little fails part-way and reboots into the installer, which looks like a boot-flip failure rather than an out-of-memory condition, so the troubleshooting page says where to look and how to raise vmmemory. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- .../troubleshooting/os_installation/index.rst | 1 + .../ubuntu_subiquity_memory.rst | 28 +++++++++++++++++++ xCAT-server/lib/xcat/plugins/debian.pm | 5 +++- 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 docs/source/troubleshooting/os_installation/ubuntu_subiquity_memory.rst diff --git a/docs/source/troubleshooting/os_installation/index.rst b/docs/source/troubleshooting/os_installation/index.rst index 6a263c5f3..ac8ed52cf 100644 --- a/docs/source/troubleshooting/os_installation/index.rst +++ b/docs/source/troubleshooting/os_installation/index.rst @@ -41,4 +41,5 @@ Next chapter introduces the procedures on how to troubleshoot operating system i log_to_mn_cn.rst debug_port.rst ssh_enable.rst + ubuntu_subiquity_memory.rst diff --git a/docs/source/troubleshooting/os_installation/ubuntu_subiquity_memory.rst b/docs/source/troubleshooting/os_installation/ubuntu_subiquity_memory.rst new file mode 100644 index 000000000..c31079284 --- /dev/null +++ b/docs/source/troubleshooting/os_installation/ubuntu_subiquity_memory.rst @@ -0,0 +1,28 @@ +Ubuntu diskful install needs memory for the live filesystem +=========================================================== + +An Ubuntu diskful (Subiquity) install boots the live installer over NFS and then copies it +into RAM, because the kernel command line xCAT generates carries ``toram``. This is +deliberate: with the NFS root still mounted at the end of the install, ``systemd-shutdown`` +blocks on I/O to it and the node never reboots into the disk it just installed. + +The consequence is that the compute node must have enough memory for the live filesystem on +top of whatever the installer itself needs. The squashfs layers are roughly 1.5 GB on 24.04 +and grow with each release, so **4 GB is a practical floor and 8 GB is comfortable**. + +A node with too little memory fails part-way through the install, and the symptom resembles +an unrelated problem: the node reboots back into the installer, so it looks like a PXE loop +rather than an out-of-memory condition. Check the console before assuming a boot-flip +failure. :: + + rcons + +To confirm, look for the OOM killer in the installer's kernel messages, or watch the node's +memory from the hypervisor while it installs. For a libvirt-hosted node the memory is +``vmm``/``vmmemory`` in the **vm** table. :: + + lsdef -i vmmemory + chdef vmmemory=8192 + +Diskless (netboot) nodes have a related but separate requirement: the rootimg is unpacked +into a tmpfs, and on ppc64le the 64 KB page size inflates it considerably. diff --git a/xCAT-server/lib/xcat/plugins/debian.pm b/xCAT-server/lib/xcat/plugins/debian.pm index 1b039836d..a4ac3a6ef 100644 --- a/xCAT-server/lib/xcat/plugins/debian.pm +++ b/xCAT-server/lib/xcat/plugins/debian.pm @@ -524,7 +524,10 @@ sub copycd network root at shutdown. Without it a process doing I/O to it blocks uninterruptibly, systemd-shutdown waits forever and the node never reboots into the disk it just installed. casper parses only nfsroot= and takes the whole value as the path, so mount options cannot - be appended; toram is its supported alternative. + be appended; toram is its supported alternative. It does mean the node needs memory for the + live filesystem on top of the installer -- roughly 1.5G on 24.04 -- or it dies part-way + through and reboots into the installer again, which looks like a boot-flip failure. See + docs/source/troubleshooting/os_installation/ubuntu_subiquity_memory.rst. Arguments: $base the command line built so far From 0f12faa0fc7d4532a2308f04c0fbefca9a7621fe Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:45:56 -0300 Subject: [PATCH 07/21] fix(xcat-core): do not add a second syncfiles when the node already defers it A node may list syncfiles in both postscripts and postbootscripts -- once for the install and once for the booted node. The deferral prepended its own copy regardless, so the booted node ran syncfiles twice in a row inside a marker block that made it look like xCAT had generated the duplicate. Keep the entry the node already has and still remove the install-time one, which is the copy that cannot work in the in-target chroot. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-server/lib/perl/xCAT/Postage.pm | 5 +++++ xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/xCAT-server/lib/perl/xCAT/Postage.pm b/xCAT-server/lib/perl/xCAT/Postage.pm index 5fda1acfd..6da8b3277 100644 --- a/xCAT-server/lib/perl/xCAT/Postage.pm +++ b/xCAT-server/lib/perl/xCAT/Postage.pm @@ -214,6 +214,11 @@ sub defer_syncfiles_to_postboot { unless defined($postscripts) && $postscripts =~ s/^[ \t]*syncfiles[ \t]*\n//m; $postbootscripts = "" unless defined $postbootscripts; + + # A node may already list syncfiles as a postbootscript of its own; do not add a second copy. + return ($postscripts, $postbootscripts) + if $postbootscripts =~ /^[ \t]*syncfiles[ \t]*$/m; + $postbootscripts = "# ubuntu-deferred-postbootscripts-start-here\nsyncfiles\n# ubuntu-deferred-postbootscripts-end-here\n" . $postbootscripts; diff --git a/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t b/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t index e0e935ffb..9962ea20c 100644 --- a/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t +++ b/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t @@ -83,6 +83,16 @@ foreach my $case (@untouched) { is($postboot, $DEFERRED, 'an undefined postbootscripts list becomes the deferral block'); } +# A node that already lists syncfiles as its own postbootscript keeps that entry, and does not +# gain a second one -- the install-time copy is still removed, since that is the one that cannot work. +{ + my ($post, $postboot) = + defer('ubuntu24.04', 'install', 'install', "syncfiles\nfoo\n", "syncfiles\nsetupntp\n"); + is($post, "foo\n", 'the postscripts entry is still removed'); + is($postboot, "syncfiles\nsetupntp\n", + "an admin's own syncfiles postbootscript is left as it was, not duplicated"); +} + # Rendering the same node twice must not stack a second copy. { my ($post, $postboot) = From bca12ca29daebadca5d3a88c26b05f65ecb28af9 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:51:55 -0300 Subject: [PATCH 08/21] fix(xcat-core): bound the boot-flip exchange, and fail when the install server will not resolve Two ways the Subiquity install could fail without saying anything useful. The boot flip read from the install monitor with no timeout. A monitor that accepts the connection and then never answers -- which #7759 shows is a state it gets into -- blocked the read forever, and with it the Subiquity late-command and the install. The retry loop could not help: it never reached the retry. Both reads now take -t 10, so five attempts are bounded at roughly two minutes and end in the failure that is already logged. The regression test stands up a listener that accepts and holds the connection; removing the timeouts fails it. mkinstall resolved the install server for nfsroot and fell back to the name when that failed: "getipaddr($instserver) || $instserver". The name is the original defect -- klibc's nfsmount cannot resolve one -- so the node panicked "can't parse IP address" at boot, on the node, with nothing reported on the management node. The management node knows at template time, so it says so there and skips the node, as the other unrecoverable per-node conditions in this routine do. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-server/lib/xcat/plugins/debian.pm | 10 +++++++++- .../xcat/install/ubuntu/compute.subiquity.tmpl | 6 ++++-- xCAT-test/unit/ubuntu_subiquity_boot_flip.t | 16 +++++++++++++++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/xCAT-server/lib/xcat/plugins/debian.pm b/xCAT-server/lib/xcat/plugins/debian.pm index a4ac3a6ef..d7cd438d6 100644 --- a/xCAT-server/lib/xcat/plugins/debian.pm +++ b/xCAT-server/lib/xcat/plugins/debian.pm @@ -1032,7 +1032,15 @@ sub mkinstall { my $kcmdline = "nofb utf8 auto xcatd=" . $instserver; if (using_subiquity($os,$tmplfile)) { - my $nfsip = xCAT::NetworkUtils->getipaddr($instserver) || $instserver; + # Fail here rather than handing casper a name: klibc's nfsmount cannot resolve + # one, so the node would panic "can't parse IP address" at boot, on the node, + # with nothing said on the management node. + my $nfsip = xCAT::NetworkUtils->getipaddr($instserver); + unless ($nfsip) { + xCAT::MsgUtils->report_node_error($callback, $node, + "Could not resolve the install server '$instserver' to an address. The Ubuntu live installer mounts its root with klibc nfsmount, which cannot resolve names, so nfsroot must be an address."); + next; + } $kcmdline = subiquity_kcmdline($kcmdline, $nfsip, $pkgdir, $instserver, $httpport, $node); } else { $kcmdline .= " url=http://${instserver}:$httpport/install/autoinst/$node"; diff --git a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl index f82aec459..4ec7faf20 100644 --- a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl +++ b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl @@ -110,7 +110,9 @@ autoinstall: # Flip the node to local-disk boot, or it PXE-loops back into the installer on reboot. # "next" is the request xcatd's install monitor answers with "nodeset next". Send it # from the live installer, retried, and log a failure rather than silently reinstalling. A - # monitor that died and never came back is a separate problem; see #7759. - - ['bash', '-c', 'xm=#XCATVAR:XCATMASTER#; ok=0; for i in 1 2 3 4 5; do if exec 3<>/dev/tcp/$xm/3002; then read -r r <&3 || true; printf "next\n" >&3; if read -r r <&3; then ok=1; fi; exec 3>&- 3<&-; [ "$ok" = 1 ] && break; fi; sleep 5; done; if [ "$ok" != 1 ]; then echo "xcat: FAILED to flip $(hostname) to local-disk boot via $xm:3002; the node will PXE back into the installer" >>/target/var/log/xcat/xcat.log; fi; exit 0'] + # monitor that died and never came back is a separate problem; see #7759 -- but one that + # accepts the connection and then never answers would block a bare read forever and hang the + # install here, so both reads are bounded. + - ['bash', '-c', 'xm=#XCATVAR:XCATMASTER#; ok=0; for i in 1 2 3 4 5; do if exec 3<>/dev/tcp/$xm/3002; then read -r -t 10 r <&3 || true; printf "next\n" >&3; if read -r -t 10 r <&3; then ok=1; fi; exec 3>&- 3<&-; [ "$ok" = 1 ] && break; fi; sleep 5; done; if [ "$ok" != 1 ]; then echo "xcat: FAILED to flip $(hostname) to local-disk boot via $xm:3002; the node will PXE back into the installer" >>/target/var/log/xcat/xcat.log; fi; exit 0'] error-commands: - tar -c --ignore-failed-read --transform='s/^/#HOSTNAME#-logs\//' /var/crash /var/log/installer /tmp/pre-install.log /autoinstall.yaml 2>/dev/null |nc -l 8080 diff --git a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t index a916de89f..8102718d0 100644 --- a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t +++ b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t @@ -61,6 +61,7 @@ sub run_flip { for (1 .. $opt{listen}) { my $c = $srv->accept() or last; $c->autoflush(1); + if ($opt{mute}) { sleep 600; close $c; next } # accept and hold, never answer print {$c} "ready\n"; my $line = <$c>; print {$seen} $line if defined $line; @@ -75,7 +76,8 @@ sub run_flip { # The installer would hang here if the exchange ever blocked, so bound it. # the no-listener case prints "Connection refused" by design - my $rc = system("timeout 25 bash -c \Q$script\E 2>/dev/null"); + my $cap = $opt{cap} || 25; + my $rc = system("timeout $cap bash -c \Q$script\E 2>/dev/null"); my $timed_out = (($rc >> 8) == 124); if ($pid) { kill 'TERM', $pid; waitpid($pid, 0) } @@ -114,6 +116,18 @@ sub run_flip { 'a connection without an acknowledgement counts as a failure, not a success'); } +# --- a monitor that accepts and never answers must not hang the install ---- +# The late-command runs inside Subiquity: a bare read on a socket that is open but silent blocks +# forever and the install never finishes. #7759 fixes the monitor dying; this bounds the wait. +{ + # Five attempts, each bounded by two 10s reads plus the retry pause: ~2 minutes worst case. + my $r = run_flip(listen => 1, mute => 1, cap => 200); + ok(!$r->{timed_out}, + 'a monitor that accepts but never replies does not hang the late-command'); + like($r->{log}, qr/FAILED to flip/, + 'it is recorded as a failed flip rather than waiting indefinitely'); +} + # --- the command retries rather than giving up on the first refusal -------- { # Answer only on a later connection: the flip must still succeed. From 2558e847360b2831e1959748c842b7c1238e3ca0 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:47:10 -0300 Subject: [PATCH 09/21] test(debian): nothing catches !myipfn! being treated as a hostname A node whose noderes.xcatmaster is unset gets $instserver = '!myipfn!'. That is a placeholder, not a name: pxe.pm:176 and grub2.pm:129 substitute it with my_ip_facing($node) -- an address -- when they write the boot config. The subiquity path resolves $instserver with getipaddr, which returns undef for the placeholder, so it reports "Could not resolve the install server" and `next`s past the node. Before this branch it produced exactly the numeric nfsroot the change is trying to guarantee. anaconda.pm and sles.pm both guard the same placeholder with `unless ($instserver eq '!myipfn!')`. noderes.5.rst:125 documents an unset xcatmaster as supported. No pipeline can catch it: reg_linux_diskfull_installation_flat chdefs xcatmaster=$$MN and no ci/conf/pipelines/*.conf leaves it unset, so CI always takes the resolvable branch. This drives the decision with an injected resolver, so it also checks the resolver is never ASKED about the placeholder rather than only checking the return value. It fails at the extraction guard until the routine exists, so the behavioural proof is the mutation on top of the fix, not this red alone. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- .../unit/debian_subiquity_nfsroot_server.t | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 xCAT-test/unit/debian_subiquity_nfsroot_server.t diff --git a/xCAT-test/unit/debian_subiquity_nfsroot_server.t b/xCAT-test/unit/debian_subiquity_nfsroot_server.t new file mode 100644 index 000000000..23d93a3ad --- /dev/null +++ b/xCAT-test/unit/debian_subiquity_nfsroot_server.t @@ -0,0 +1,81 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use FindBin; +use Test::More; + +# Regression: nodes whose noderes.xcatmaster is unset stopped getting a boot config at all. +# +# When xcatmaster is unset, debian.pm sets $instserver to the literal '!myipfn!'. That is a +# placeholder, not a hostname: pxe.pm:176 and grub2.pm:129 substitute it with +# my_ip_facing($node) -- an address -- when they write the boot config. Resolving it as a name +# returns undef, so the subiquity path reported "Could not resolve the install server" and +# `next`ed past the node, where before it had produced exactly the numeric nfsroot the fix +# wants. anaconda.pm and sles.pm both guard the same placeholder with +# `unless ($instserver eq '!myipfn!')`; the Ubuntu path did not. +# +# noderes.5.rst:125 documents an unset xcatmaster as supported, and the CI confs all set it +# (reg_linux_diskfull_installation_flat chdefs xcatmaster=$$MN), which is why no pipeline +# would catch this. +# +# The decision lives in its own routine so it can be driven with an injected resolver instead +# of a management node. + +my $repo_root = File::Spec->rel2abs( + File::Spec->catdir( $FindBin::Bin, '..', '..' ) +); +my $plugin = File::Spec->catfile( + $repo_root, 'xCAT-server', 'lib', 'xcat', 'plugins', 'debian.pm' +); +plan skip_all => "debian.pm not found" unless -f $plugin; + +my $src = do { local $/; open my $fh, '<', $plugin or die $!; <$fh> }; + +# Lift the routine into a scratch package: debian.pm itself needs a management node to load. +# BAIL_OUT rather than skip, so a rename fails loudly instead of silently covering nothing. +my ($body) = $src =~ /\n(sub subiquity_nfsroot_server \{.*?\n\})\n/s; +BAIL_OUT('could not extract subiquity_nfsroot_server from debian.pm') + unless defined $body; + +{ + package T; + eval "$body; 1" or main::BAIL_OUT("could not eval subiquity_nfsroot_server: $@"); +} + +# A resolver that records what it was asked, so "never asked" is checkable. +my @asked; +my $resolver = sub { push @asked, $_[0]; return $_[0] eq 'mn.cluster' ? '10.0.0.1' : undef }; + +# The bug: the placeholder must survive to the boot config, not be resolved. +@asked = (); +is( T::subiquity_nfsroot_server( '!myipfn!', $resolver ), '!myipfn!', + 'the !myipfn! placeholder is passed through untouched' ); +is_deeply( \@asked, [], + 'and the resolver is never asked to resolve it' ); + +# A name that resolves still resolves. +@asked = (); +is( T::subiquity_nfsroot_server( 'mn.cluster', $resolver ), '10.0.0.1', + 'a resolvable install server name becomes its address' ); +is_deeply( \@asked, ['mn.cluster'], 'by asking the resolver' ); + +# An address passes through the resolver unchanged, as getipaddr does. +is( T::subiquity_nfsroot_server( '10.0.0.1', sub { $_[0] } ), '10.0.0.1', + 'an address is returned as itself' ); + +# The guard that motivated the original commit must survive: a name that does not resolve is +# still a failure, because klibc's nfsmount cannot resolve it either. +@asked = (); +ok( !defined T::subiquity_nfsroot_server( 'nosuchhost', $resolver ), + 'a name that does not resolve is still rejected' ); +is_deeply( \@asked, ['nosuchhost'], 'after actually trying to resolve it' ); + +# Degenerate inputs are rejected rather than passed to the resolver. +ok( !defined T::subiquity_nfsroot_server( undef, $resolver ), + 'an undefined install server is rejected' ); +ok( !defined T::subiquity_nfsroot_server( '', $resolver ), + 'an empty install server is rejected' ); + +done_testing(); From 5ce5edf24b8e99ff3a46eebd13e1099303be8780 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:47:45 -0300 Subject: [PATCH 10/21] fix(debian): nodes with no xcatmaster get no boot config at all The subiquity path resolved $instserver with getipaddr and `next`ed the node when that returned nothing. For a node whose noderes.xcatmaster is unset, $instserver is the literal '!myipfn!' -- a placeholder that pxe.pm:176 and grub2.pm:129 substitute with my_ip_facing($node) when they write the boot config. Resolving it as a name always fails, so those nodes went from a working numeric nfsroot to no boot config and a "Could not resolve the install server" error. noderes.5.rst:125 documents an unset xcatmaster as supported, and anaconda.pm and sles.pm both already guard the placeholder with `unless ($instserver eq '!myipfn!')`. Exempt the placeholder, keeping the guard for a real name that does not resolve -- klibc's nfsmount still cannot resolve one of those, which is what the original commit was for. The decision moves into subiquity_nfsroot_server(), taking the resolver as an optional argument, so it can be driven directly instead of through a routine that needs a management node. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-server/lib/xcat/plugins/debian.pm | 36 ++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/xCAT-server/lib/xcat/plugins/debian.pm b/xCAT-server/lib/xcat/plugins/debian.pm index d7cd438d6..226eb5545 100644 --- a/xCAT-server/lib/xcat/plugins/debian.pm +++ b/xCAT-server/lib/xcat/plugins/debian.pm @@ -509,6 +509,37 @@ sub copycd #------------------------------------------------------------------------------- +=head3 subiquity_nfsroot_server + + Resolve the install server to the address casper's klibc nfsmount needs. + + '!myipfn!' is a placeholder, not a name: pxe.pm and grub2.pm substitute it with + my_ip_facing($node) when they write the boot config, so it is already an address by the + time klibc sees it. Resolving it here would only ever fail, and failing would drop a node + whose noderes.xcatmaster is simply unset -- which noderes.5.rst documents as supported. + anaconda.pm and sles.pm guard the same placeholder the same way. + + Arguments: + $instserver the install server name, address, or the '!myipfn!' placeholder + $resolver optional coderef, for tests; defaults to NetworkUtils::getipaddr + Returns: + the value to put in nfsroot, or undef when a real name does not resolve + +=cut + +#------------------------------------------------------------------------------- +sub subiquity_nfsroot_server { + my ($instserver, $resolver) = @_; + + return undef unless defined($instserver) && length($instserver); + return $instserver if $instserver eq '!myipfn!'; + + $resolver ||= sub { xCAT::NetworkUtils->getipaddr($_[0]) }; + return $resolver->($instserver); +} + +#------------------------------------------------------------------------------- + =head3 subiquity_kcmdline Build the kernel command line for a Subiquity (Ubuntu live installer) diskful install. @@ -1034,8 +1065,9 @@ sub mkinstall { if (using_subiquity($os,$tmplfile)) { # Fail here rather than handing casper a name: klibc's nfsmount cannot resolve # one, so the node would panic "can't parse IP address" at boot, on the node, - # with nothing said on the management node. - my $nfsip = xCAT::NetworkUtils->getipaddr($instserver); + # with nothing said on the management node. '!myipfn!' is exempt -- pxe.pm and + # grub2.pm turn it into an address when they write the boot config. + my $nfsip = subiquity_nfsroot_server($instserver); unless ($nfsip) { xCAT::MsgUtils->report_node_error($callback, $node, "Could not resolve the install server '$instserver' to an address. The Ubuntu live installer mounts its root with klibc nfsmount, which cannot resolve names, so nfsroot must be an address."); From 7082fabe0b64545fdec4559613c78cd8612599e7 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:50:31 -0300 Subject: [PATCH 11/21] test(subiquity): the boot-flip test skips wherever the suite runs ubuntu_subiquity_boot_flip.t bound 3002 on the loopback to check it was free and skip_all'd when it was not. 3002 is the install-monitor port, so on any management node xcatd is already listening there -- which is precisely where the suite runs. The CI log for #7761 reads ubuntu_subiquity_boot_flip.t ... skipped: port 3002 is not available on the loopback interface so all 139 lines and 12 assertions never executed once, and never would have. Coverage that reports as a skip is worse than none, because the gap is invisible. Take an ephemeral port from the kernel and rewrite the extracted command to use it, in both the /dev/tcp target and the log message. The port number is not what is under test -- the retry-and-log behaviour is. The template's own port is read out of the command rather than hard-coded, and asserted to be 3002, so moving the install-monitor still leaves the file covering something instead of silently testing a port nothing uses. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-test/unit/ubuntu_subiquity_boot_flip.t | 25 ++++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t index 8102718d0..7cc1a9ab2 100644 --- a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t +++ b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t @@ -19,12 +19,15 @@ use Test::More; my $tmpl = "$FindBin::Bin/../../xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl"; plan skip_all => 'compute.subiquity.tmpl not found' unless -r $tmpl; -my $XCATD_PORT = 3002; # the install-monitor port the template addresses - +# The template addresses the install-monitor on 3002, and on any management node xcatd is +# already listening there -- so binding it here made the whole file skip_all exactly where the +# suite runs. Take an ephemeral port from the kernel instead and rewrite the extracted command +# to use it: what is under test is the retry-and-log behaviour, not the port number. my $probe = IO::Socket::INET->new( - LocalAddr => '127.0.0.1', LocalPort => $XCATD_PORT, Proto => 'tcp', - Listen => 5, ReuseAddr => 1); -plan skip_all => "port $XCATD_PORT is not available on the loopback interface" unless $probe; + LocalAddr => '127.0.0.1', LocalPort => 0, Proto => 'tcp', + Listen => 5, ReuseAddr => 1) + or BAIL_OUT("could not take an ephemeral port on the loopback interface: $!"); +my $XCATD_PORT = $probe->sockport; close $probe; # each case below opens its own listener, or none at all open(my $fh, '<', $tmpl) or die "open $tmpl: $!"; @@ -35,6 +38,13 @@ close $fh; my ($command) = $source =~ m{- \['bash', '-c', '(.*?/dev/tcp/.*?)'\]}; BAIL_OUT('no late-command in the template performs the boot flip over /dev/tcp') unless $command; +# Read the port out of the template rather than hard-coding it, so a template that moves the +# install-monitor still gets covered instead of silently testing the wrong port. +my ($TEMPLATE_PORT) = $command =~ m{/dev/tcp/\$xm/(\d+)}; +BAIL_OUT('could not read the install-monitor port from the boot-flip command') + unless $TEMPLATE_PORT; +is($TEMPLATE_PORT, 3002, 'the template addresses the install-monitor port xcatd listens on'); + # Run the command with the install server pointed at our stand-in, and its log inside a scratch # tree. Everything else is the template's own text. sub run_flip { @@ -45,6 +55,9 @@ sub run_flip { my $script = $command; $script =~ s/\#XCATVAR:XCATMASTER\#/127.0.0.1/; + # point the flip at the ephemeral listener, in both the /dev/tcp target and the log message + $script =~ s{/dev/tcp/\$xm/\Q$TEMPLATE_PORT\E\b}{/dev/tcp/\$xm/$XCATD_PORT}; + $script =~ s{\$xm:\Q$TEMPLATE_PORT\E\b}{\$xm:$XCATD_PORT}g; $script =~ s{/target/var/log/xcat/xcat\.log}{$root/target/var/log/xcat/xcat.log}; $script =~ s/sleep 5/sleep 1/; # shorten the retry pause, keep the retry @@ -105,7 +118,7 @@ sub run_flip { is($r->{rc}, 0, 'a failed flip still exits 0 rather than aborting the install'); like($r->{log}, qr/FAILED to flip/, 'a failed flip is recorded in the install log instead of PXE-looping silently'); - like($r->{log}, qr/127\.0\.0\.1:3002/, + like($r->{log}, qr/127\.0\.0\.1:\Q$XCATD_PORT\E\b/, 'the log names the install server and port that could not be reached'); } From 1cebb4d9e8c497729ee5d7f712791db0e318eed2 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:51:05 -0300 Subject: [PATCH 12/21] test(subiquity): the resolv.conf sandbox fails open, as root ubuntu_resolvconf_ip.t sandboxes the fragment by rewriting /etc/resolv.conf to a path inside a tempdir. The fragment contains `rm -f /etc/resolv.conf` and the unit suite runs as root in CI, so if that substitution ever stops matching the test deletes the runner's resolver configuration instead of failing. It matches today. It is one respelling away from not: writing the path in the template as `etcdir=/etc; rm -f "$etcdir/resolv.conf"` slips straight past it, and the existing BAIL_OUT does not catch that -- it guards only the fragment extraction, not the rewrite. Check the rewritten script for any /etc path outside the scratch tree and BAIL_OUT rather than execute it. AGENTS.md asks that a test never escape its scratch tree and notes that rewriting paths in the source under test is the fragile way to arrange it; this makes the fragile part fail closed. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-test/unit/ubuntu_resolvconf_ip.t | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/xCAT-test/unit/ubuntu_resolvconf_ip.t b/xCAT-test/unit/ubuntu_resolvconf_ip.t index 4a1f1ed8f..696a91ab8 100644 --- a/xCAT-test/unit/ubuntu_resolvconf_ip.t +++ b/xCAT-test/unit/ubuntu_resolvconf_ip.t @@ -32,6 +32,15 @@ sub write_resolv_conf { $script =~ s/\#TABLE:noderes:\$NODE:xcatmaster\#/$opt{xcatmaster}/; $script =~ s{/etc/resolv\.conf}{$root/resolv.conf}g; + # The fragment contains `rm -f /etc/resolv.conf` and this suite runs as root in CI, so a + # rewrite that stops matching would delete the runner's resolver configuration rather than + # fail a test. Sandboxing by rewriting paths is fragile by nature -- respelling the path in + # the template as, say, `etcdir=/etc; rm -f "$etcdir/resolv.conf"` slips straight past the + # substitution above. Refuse to execute anything that still points outside the scratch tree. + if ($script =~ m{(? Date: Tue, 1 Sep 2026 21:05:20 -0300 Subject: [PATCH 13/21] test(debian): nothing connects the subiquity helpers to mkinstall subiquity_nfsroot_server and subiquity_kcmdline each have tests, but reverting the call site in mkinstall -- putting xCAT::NetworkUtils->getipaddr back in place of subiquity_nfsroot_server, which is precisely the regression the fix removes -- leaves the whole unit suite green. A helper can be perfectly covered while nothing links it to production, and that is the shape the review found. mkinstall needs a management node, so this drives the composition it performs: resolve the install server, then build the command line, or explain why not. It fails at the extraction guard until that composition is a routine that can be called, so the behavioural proof is the mutation on top of the fix, not this red alone. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-test/unit/debian_subiquity_boot_params.t | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 xCAT-test/unit/debian_subiquity_boot_params.t diff --git a/xCAT-test/unit/debian_subiquity_boot_params.t b/xCAT-test/unit/debian_subiquity_boot_params.t new file mode 100644 index 000000000..6c503dfcc --- /dev/null +++ b/xCAT-test/unit/debian_subiquity_boot_params.t @@ -0,0 +1,82 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use FindBin; +use Test::More; + +# Regression: the helpers were covered and the call site was not. +# +# subiquity_nfsroot_server and subiquity_kcmdline each had tests, but reverting the CALL SITE in +# mkinstall -- putting `xCAT::NetworkUtils->getipaddr($instserver)` back in place of +# subiquity_nfsroot_server, i.e. the exact regression the fix removes -- left the whole unit +# suite green. A helper can be perfectly covered while nothing connects it to production. +# +# mkinstall needs a management node, so the composition it performs (resolve the install server, +# then build the command line, or explain why not) lives in subiquity_boot_params, which takes +# its inputs and returns an answer. The caller keeps report_node_error and the loop's `next`. + +my $repo_root = File::Spec->rel2abs( + File::Spec->catdir( $FindBin::Bin, '..', '..' ) +); +my $plugin = File::Spec->catfile( + $repo_root, 'xCAT-server', 'lib', 'xcat', 'plugins', 'debian.pm' +); +plan skip_all => "debian.pm not found" unless -f $plugin; + +my $src = do { local $/; open my $fh, '<', $plugin or die $!; <$fh> }; + +# BAIL_OUT rather than skip, so a rename fails loudly instead of silently covering nothing. +my $body = ''; +for my $name (qw(subiquity_nfsroot_server subiquity_kcmdline subiquity_boot_params)) { + my ($sub) = $src =~ /\n(sub \Q$name\E \{.*?\n\})\n/s; + BAIL_OUT("could not extract $name from debian.pm") unless defined $sub; + $body .= "$sub\n"; +} + +{ + package T; + eval "$body; 1" or main::BAIL_OUT("could not eval the subiquity helpers: $@"); +} + +my $resolver = sub { $_[0] eq 'mn.cluster' ? '10.0.0.1' : undef }; + +# The regression this exists to catch: an unset noderes.xcatmaster gives '!myipfn!', and the +# boot config must still be produced with the placeholder intact, because pxe.pm and grub2.pm +# substitute it with an address when they write the config. +{ + my ($kcmdline, $err) = T::subiquity_boot_params( + 'nofb utf8 auto xcatd=!myipfn!', '!myipfn!', + '/install/ubuntu24.04/x86_64', '80', 'cn1', $resolver ); + ok( !defined $err, 'a node with no xcatmaster still gets a boot config' ) + or diag("error was: $err"); + like( $kcmdline, qr/nfsroot=!myipfn!:/, + 'and nfsroot keeps the placeholder for pxe.pm/grub2.pm to substitute' ); + like( $kcmdline, qr/boot=casper/, 'the casper boot flag survives the composition' ); +} + +# A resolvable name still becomes an address, because klibc's nfsmount has no resolver. +{ + my ($kcmdline, $err) = T::subiquity_boot_params( + 'nofb utf8 auto xcatd=mn.cluster', 'mn.cluster', + '/install/ubuntu24.04/x86_64', '80', 'cn1', $resolver ); + ok( !defined $err, 'a resolvable install server produces a boot config' ); + like( $kcmdline, qr{nfsroot=10\.0\.0\.1:/install/ubuntu24\.04/x86_64}, + 'and nfsroot carries the address, not the name' ); + like( $kcmdline, qr{ds=nocloud-net;s=http://mn\.cluster:80/}, + 'while the cloud-init seed URL keeps the name, where DNS works' ); +} + +# The guard the original commit added must survive: a name that does not resolve is an error, +# not a command line with a name in nfsroot. +{ + my ($kcmdline, $err) = T::subiquity_boot_params( + 'nofb utf8 auto xcatd=nosuchhost', 'nosuchhost', + '/install/ubuntu24.04/x86_64', '80', 'cn1', $resolver ); + ok( !defined $kcmdline, 'an unresolvable install server yields no boot config' ); + like( $err, qr/nosuchhost/, 'and the error names the server that failed' ); + like( $err, qr/klibc|nfsmount|address/i, 'and says why it matters' ); +} + +done_testing(); From b784aa782c405908b82ca3fa780ab7ee6ab73325 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:09:31 -0300 Subject: [PATCH 14/21] fix(debian): connect the subiquity helpers to mkinstall, and close the sandbox guard Two things the review found, both in code added by this branch. Reverting mkinstall's call site -- putting xCAT::NetworkUtils->getipaddr back in place of subiquity_nfsroot_server, the exact regression the fix removes -- left the entire unit suite green. The helpers were covered; nothing linked them to production. Compose the two steps in subiquity_boot_params(), which takes its inputs and returns either a command line or the reason there isn't one, so the composition can be driven; mkinstall keeps report_node_error and the loop's `next`. That same revert now reddens 6 of 9 assertions. The test stubs xCAT::NetworkUtils::getipaddr deliberately. Without it a call site that bypassed the injected resolver died on a missing module -- a red, but for the wrong reason. With it, bypassing the resolver returns the wrong answer, which is what the assertions are there to catch. The resolv.conf sandbox guard matched `/etc/` with a trailing slash, so the one respelling its own comment names -- `etcdir=/etc; rm -f "$etcdir/resolv.conf"` -- walked straight past it and the fragment would rm the runner's real resolv.conf, as root in CI. `/etc\b` catches it: applying that respelling now BAIL_OUTs instead of running. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-server/lib/xcat/plugins/debian.pm | 53 +++++++++++++++---- xCAT-test/unit/debian_subiquity_boot_params.t | 9 ++++ xCAT-test/unit/ubuntu_resolvconf_ip.t | 4 +- 3 files changed, 56 insertions(+), 10 deletions(-) diff --git a/xCAT-server/lib/xcat/plugins/debian.pm b/xCAT-server/lib/xcat/plugins/debian.pm index 226eb5545..b923c37c8 100644 --- a/xCAT-server/lib/xcat/plugins/debian.pm +++ b/xCAT-server/lib/xcat/plugins/debian.pm @@ -584,6 +584,42 @@ sub subiquity_kcmdline { return $kcmdline; } +#------------------------------------------------------------------------------- + +=head3 subiquity_boot_params + + Resolve the install server and build the Subiquity command line, or say why not. + + The two steps are composed here rather than in mkinstall so the composition can be driven: + mkinstall needs a management node, and the decision that matters -- which install server + ends up in nfsroot -- is exactly what a regression would change. The caller keeps the side + effects: reporting the error and skipping the node. + + Arguments: + $base the command line built so far + $instserver the install server name, address, or the '!myipfn!' placeholder + $pkgdir the install media path exported over NFS + $httpport the xCAT HTTP port + $node the node being installed + $resolver optional coderef, for tests; passed through to subiquity_nfsroot_server + Returns: + ($kcmdline, undef) on success, or (undef, $message) when the server does not resolve + +=cut + +#------------------------------------------------------------------------------- +sub subiquity_boot_params { + my ($base, $instserver, $pkgdir, $httpport, $node, $resolver) = @_; + + my $nfsip = subiquity_nfsroot_server($instserver, $resolver); + return (undef, "Could not resolve the install server '$instserver' to an address. " + . "The Ubuntu live installer mounts its root with klibc nfsmount, which cannot " + . "resolve names, so nfsroot must be an address.") + unless $nfsip; + + return (subiquity_kcmdline($base, $nfsip, $pkgdir, $instserver, $httpport, $node), undef); +} + sub mkinstall { xCAT::MsgUtils->message("S", "Doing debian mkinstall"); my $request = shift; @@ -1063,17 +1099,16 @@ sub mkinstall { my $kcmdline = "nofb utf8 auto xcatd=" . $instserver; if (using_subiquity($os,$tmplfile)) { - # Fail here rather than handing casper a name: klibc's nfsmount cannot resolve - # one, so the node would panic "can't parse IP address" at boot, on the node, - # with nothing said on the management node. '!myipfn!' is exempt -- pxe.pm and - # grub2.pm turn it into an address when they write the boot config. - my $nfsip = subiquity_nfsroot_server($instserver); - unless ($nfsip) { - xCAT::MsgUtils->report_node_error($callback, $node, - "Could not resolve the install server '$instserver' to an address. The Ubuntu live installer mounts its root with klibc nfsmount, which cannot resolve names, so nfsroot must be an address."); + # Fail rather than hand casper a name: klibc's nfsmount cannot resolve one, so + # the node would panic "can't parse IP address" at boot, on the node, with + # nothing said on the management node. + my ($subiquity_cmdline, $subiquity_error) = + subiquity_boot_params($kcmdline, $instserver, $pkgdir, $httpport, $node); + if ($subiquity_error) { + xCAT::MsgUtils->report_node_error($callback, $node, $subiquity_error); next; } - $kcmdline = subiquity_kcmdline($kcmdline, $nfsip, $pkgdir, $instserver, $httpport, $node); + $kcmdline = $subiquity_cmdline; } else { $kcmdline .= " url=http://${instserver}:$httpport/install/autoinst/$node"; $kcmdline .= " mirror/http/hostname=${instserver}:$httpport"; diff --git a/xCAT-test/unit/debian_subiquity_boot_params.t b/xCAT-test/unit/debian_subiquity_boot_params.t index 6c503dfcc..7a658864a 100644 --- a/xCAT-test/unit/debian_subiquity_boot_params.t +++ b/xCAT-test/unit/debian_subiquity_boot_params.t @@ -35,6 +35,15 @@ for my $name (qw(subiquity_nfsroot_server subiquity_kcmdline subiquity_boot_para $body .= "$sub\n"; } +# Stand in for the module the routine falls back to when no resolver is injected. Without this +# a call site that bypassed the injected resolver would die on a missing module -- a red, but +# for the wrong reason. With it, bypassing the resolver produces the WRONG ANSWER instead of an +# exception, which is what the assertions below are meant to catch. +{ + package xCAT::NetworkUtils; + sub getipaddr { return undef } +} + { package T; eval "$body; 1" or main::BAIL_OUT("could not eval the subiquity helpers: $@"); diff --git a/xCAT-test/unit/ubuntu_resolvconf_ip.t b/xCAT-test/unit/ubuntu_resolvconf_ip.t index 696a91ab8..325c835ab 100644 --- a/xCAT-test/unit/ubuntu_resolvconf_ip.t +++ b/xCAT-test/unit/ubuntu_resolvconf_ip.t @@ -37,7 +37,9 @@ sub write_resolv_conf { # fail a test. Sandboxing by rewriting paths is fragile by nature -- respelling the path in # the template as, say, `etcdir=/etc; rm -f "$etcdir/resolv.conf"` slips straight past the # substitution above. Refuse to execute anything that still points outside the scratch tree. - if ($script =~ m{(? Date: Tue, 1 Sep 2026 21:48:54 -0300 Subject: [PATCH 15/21] fix(postage): the provmethod override in makescript can never fire The syncfiles deferral resolved the node's provmethod through $image_hash{$osimgname}{provmethod} when the node names an osimage. makescript fills %image_hash from getImage(), which stores pkglist, pkgdir, otherpkglist, otherpkgdir and environvar -- and no provmethod. getScripts() has a separate hash that does store one, which is where the pattern was copied from. So the lookup was always undef, the override never fired, and the code claimed a behaviour it did not have. Pass $provmethod directly and say in the comment why there is nothing to resolve it with. No behaviour changes -- the branch was inert -- so there is no red to show first; what the deletion needs is coverage that the path it was supposed to serve still works. That is what the two new assertions do: an osimage-named provmethod with nodesetstate 'install' still defers, and the same name with no nodesetstate is not mistaken for a diskful install. nodesetstate is what carries the install signal here, which is why the override was never load-bearing. Making the deferral ignore nodesetstate and require provmethod eq 'install' reddens both. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-server/lib/perl/xCAT/Postage.pm | 13 ++++++----- .../unit/postage_ubuntu_syncfiles_postboot.t | 22 +++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/xCAT-server/lib/perl/xCAT/Postage.pm b/xCAT-server/lib/perl/xCAT/Postage.pm index 6da8b3277..c575304cd 100644 --- a/xCAT-server/lib/perl/xCAT/Postage.pm +++ b/xCAT-server/lib/perl/xCAT/Postage.pm @@ -601,12 +601,15 @@ sub makescript { # See defer_syncfiles_to_postboot(): on the Ubuntu/Debian diskful install path syncfiles # has to run on the booted node, not in the installer's in-target chroot. - my $effective_provmethod = $provmethod; - if ($osimgname && defined($image_hash{$osimgname}{'provmethod'})) { - $effective_provmethod = $image_hash{$osimgname}{'provmethod'}; - } + # + # $provmethod is passed as-is. An earlier version tried to resolve it through + # $image_hash{$osimgname}{provmethod} when the node names an osimage, but makescript + # fills %image_hash from getImage(), which stores pkglist/pkgdir/otherpkg*/environvar and + # no provmethod -- getScripts() has a separate hash that does. So that lookup was always + # undef and the override never fired. $nodesetstate is what carries the install signal on + # this path anyway: nodeset sets it, and defer_syncfiles_to_postboot checks it first. ($postscripts, $postbootscripts) = defer_syncfiles_to_postboot( - $os, $effective_provmethod, $nodesetstate, $postscripts, $postbootscripts); + $os, $provmethod, $nodesetstate, $postscripts, $postbootscripts); # if using zones then must go to the zone.sshbetweennodes # else go to site.sshbetweennodes diff --git a/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t b/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t index 9962ea20c..7603c97aa 100644 --- a/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t +++ b/xCAT-test/unit/postage_ubuntu_syncfiles_postboot.t @@ -102,4 +102,26 @@ foreach my $case (@untouched) { is($postboot2, $postboot, 'a second pass does not duplicate syncfiles'); } +# The node's provmethod is frequently an osimage NAME rather than the literal 'install' -- that +# is what makescript passes, and resolving it to a real provmethod is not possible there +# (getImage does not store one). nodesetstate is what carries the install signal on that path, +# so the deferral must key off it and not require provmethod to say 'install'. +{ + my ($post, $postboot) = defer( + 'ubuntu24.04', 'ubuntu24.04-x86_64-install-compute', 'install', + "otherpkgs\nsyncfiles\nremoteshell\n", "" ); + unlike( $post, qr/^syncfiles$/m, + 'an osimage-named provmethod still defers when nodesetstate says install' ); + like( $postboot, qr/^syncfiles$/m, + 'and syncfiles lands in the postboot scripts' ); +} + +{ + my ($post, $postboot) = defer( + 'ubuntu24.04', 'ubuntu24.04-x86_64-install-compute', undef, + "otherpkgs\nsyncfiles\nremoteshell\n", "" ); + like( $post, qr/^syncfiles$/m, + 'with no nodesetstate an osimage name is not mistaken for a diskful install' ); +} + done_testing(); From abc45b1f740a3774783716e26cd40b3dd14f1c63 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:28:44 -0300 Subject: [PATCH 16/21] fix(postage): restore the provmethod override, and cover mkinstall's call site The previous commit deleted the $effective_provmethod override on the grounds that %image_hash never carries a provmethod. That was wrong, and the review caught it: makescript builds %image_hash, calls getImage() on it, and then hands the SAME hashref to getScripts(), which fills provmethod for every osimage from the osimage table. getDisklessNet() already reads that key the same way. The override was live, not dead. Restore it and say what is actually true in the comment. nodetype.provmethod is frequently an osimage name rather than 'install', and resolving it is the point of the lookup. Also close the gap that made the wrong deletion so easy to ship: reverting mkinstall's subiquity branch to its pre-fix body left the whole unit suite green. debian_mkinstall_subiquity_branch.t lifts that branch out and drives it inside a real loop, so the `next` it performs is the one under test, with report_node_error and the getipaddr seam stood in for. It calls subiquity_boot_params with no injected resolver, exactly as production does. The branch is selected out of debian.pm by what it contains rather than by where it sits -- there are four `if (using_subiquity(...))` in that file, and an earlier draft of this test silently matched the wrong one and ran past its block. Now observable, each verified by mutation: swapping $pkgdir and $httpport at the call site reddens the nfsroot assertion; reverting the branch wholesale fails the extraction guard rather than passing. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-server/lib/perl/xCAT/Postage.pm | 15 +- .../unit/debian_mkinstall_subiquity_branch.t | 130 ++++++++++++++++++ 2 files changed, 138 insertions(+), 7 deletions(-) create mode 100644 xCAT-test/unit/debian_mkinstall_subiquity_branch.t diff --git a/xCAT-server/lib/perl/xCAT/Postage.pm b/xCAT-server/lib/perl/xCAT/Postage.pm index c575304cd..e9d40ba8c 100644 --- a/xCAT-server/lib/perl/xCAT/Postage.pm +++ b/xCAT-server/lib/perl/xCAT/Postage.pm @@ -602,14 +602,15 @@ sub makescript { # See defer_syncfiles_to_postboot(): on the Ubuntu/Debian diskful install path syncfiles # has to run on the booted node, not in the installer's in-target chroot. # - # $provmethod is passed as-is. An earlier version tried to resolve it through - # $image_hash{$osimgname}{provmethod} when the node names an osimage, but makescript - # fills %image_hash from getImage(), which stores pkglist/pkgdir/otherpkg*/environvar and - # no provmethod -- getScripts() has a separate hash that does. So that lookup was always - # undef and the override never fired. $nodesetstate is what carries the install signal on - # this path anyway: nodeset sets it, and defer_syncfiles_to_postboot checks it first. + # nodetype.provmethod is often an osimage NAME rather than 'install'. getScripts() above + # was handed this same %image_hash and filled it from the osimage table, provmethod + # included, so resolve through it -- getDisklessNet() reads the same key the same way. + my $effective_provmethod = $provmethod; + if ($osimgname && defined($image_hash{$osimgname}{'provmethod'})) { + $effective_provmethod = $image_hash{$osimgname}{'provmethod'}; + } ($postscripts, $postbootscripts) = defer_syncfiles_to_postboot( - $os, $provmethod, $nodesetstate, $postscripts, $postbootscripts); + $os, $effective_provmethod, $nodesetstate, $postscripts, $postbootscripts); # if using zones then must go to the zone.sshbetweennodes # else go to site.sshbetweennodes diff --git a/xCAT-test/unit/debian_mkinstall_subiquity_branch.t b/xCAT-test/unit/debian_mkinstall_subiquity_branch.t new file mode 100644 index 000000000..a40ee8297 --- /dev/null +++ b/xCAT-test/unit/debian_mkinstall_subiquity_branch.t @@ -0,0 +1,130 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use FindBin; +use Test::More; + +# Regression: the helpers were covered, the CALL SITE was not. +# +# subiquity_boot_params and subiquity_nfsroot_server each have tests, but putting mkinstall's +# pre-fix branch back -- getipaddr plus the inline command line -- left the entire unit suite +# green. Argument order, the error branch, and the `next` that skips the node were all +# unobservable, which is exactly where the bug this PR fixes lived. +# +# mkinstall needs a management node and a database, so the branch is lifted out and eval'd into +# a scratch package with report_node_error stubbed, and driven inside a real loop so the `next` +# it performs is the `next` under test. + +my $repo_root = File::Spec->rel2abs( + File::Spec->catdir( $FindBin::Bin, '..', '..' ) +); +my $plugin = File::Spec->catfile( + $repo_root, 'xCAT-server', 'lib', 'xcat', 'plugins', 'debian.pm' +); +plan skip_all => "debian.pm not found" unless -f $plugin; + +my $src = do { local $/; open my $fh, '<', $plugin or die $!; <$fh> }; + +# The helpers the branch calls, plus the branch itself. BAIL_OUT rather than skip, so a rename +# fails loudly instead of silently covering nothing. +my $helpers = ''; +for my $name (qw(subiquity_nfsroot_server subiquity_kcmdline subiquity_boot_params)) { + my ($sub) = $src =~ /\n(sub \Q$name\E \{.*?\n\})\n/s; + BAIL_OUT("could not extract $name from debian.pm") unless defined $sub; + $helpers .= "$sub\n"; +} + +# There are several `if (using_subiquity(...))` in debian.pm; take the one that actually builds +# the boot parameters, selected by what it contains rather than by where it sits, so adding +# another above it does not silently swap which branch is under test. +my @candidates = $src =~ /\n[ ]+if \(using_subiquity\([^)]*\)\) \{\n(.*?)\n[ ]+\} else \{\n/gs; +my @wanted = grep { /subiquity_boot_params\(/ } @candidates; +BAIL_OUT('could not find the mkinstall branch that builds the subiquity boot parameters') + unless @wanted == 1; +my $branch = $wanted[0]; +BAIL_OUT('the extracted branch does not skip the node on error') + unless $branch =~ /\bnext\b/; +BAIL_OUT('the extracted branch is implausibly large -- the match ran past its block') + if ($branch =~ tr/\n//) > 20; + +my @reported; + +# The branch calls subiquity_boot_params with no resolver, exactly as production does, so the +# fallback to xCAT::NetworkUtils->getipaddr is the seam to stand in at. That keeps the call path +# under test identical to the real one -- nothing is injected into it. +{ + package xCAT::NetworkUtils; + sub getipaddr { + my (undef, $name) = @_; + return $name if defined($name) && $name =~ /^\d+\.\d+\.\d+\.\d+$/; + return '10.0.0.1' if defined($name) && $name eq 'mn.cluster'; + return undef; + } +} + +{ + package xCAT::MsgUtils; + sub report_node_error { shift; my ($cb, $node, $msg) = @_; push @reported, [ $node, $msg ]; } +} + +my $driver = <<"CODE"; +$helpers +sub drive { + my (\$kcmdline, \$instserver, \$pkgdir, \$httpport, \$node) = \@_; + my \$callback; + my \$result; + NODE: foreach my \$n (\$node) { +$branch + \$result = \$kcmdline; + } + return \$result; +} +CODE + +# `next` inside the branch belongs to the loop the driver wraps around it. +$driver =~ s/\bnext;/next NODE;/g; + +{ + package T; + eval "$driver; 1" or main::BAIL_OUT("could not eval the mkinstall branch: $@"); +} + +# A resolvable install server: the branch must produce a live command line. +{ + @reported = (); + my $out = T::drive( 'nofb utf8 auto xcatd=10.0.0.1', '10.0.0.1', + '/install/ubuntu24.04/x86_64', '80', 'cn1' ); + ok( defined $out, 'a resolvable install server yields a command line' ); + like( $out, qr/boot=casper/, 'the branch puts boot=casper on the command line' ); + like( $out, qr/\btoram\b/, 'and toram' ); + like( $out, qr{nfsroot=10\.0\.0\.1:/install/ubuntu24\.04/x86_64}, + 'and nfsroot as a literal address, in the media path' ); + is( scalar @reported, 0, 'and nothing is reported as an error' ); +} + +# The placeholder: this is the case the fix restored, driven through the call site rather than +# through the helper. +{ + @reported = (); + my $out = T::drive( 'nofb utf8 auto xcatd=!myipfn!', '!myipfn!', + '/install/ubuntu24.04/x86_64', '80', 'cn1' ); + ok( defined $out, 'a node with no xcatmaster is not skipped' ); + like( $out, qr/nfsroot=!myipfn!:/, + 'and the placeholder reaches the boot config for pxe.pm/grub2.pm to substitute' ); + is( scalar @reported, 0, 'and no error is reported for it' ); +} + +# An install server that does not resolve must skip the node, not emit a command line naming it. +{ + @reported = (); + my $out = T::drive( 'nofb utf8 auto xcatd=nosuchhost', 'nosuchhost', + '/install/ubuntu24.04/x86_64', '80', 'cn1' ); + ok( !defined $out, 'an unresolvable install server skips the node' ); + is( scalar @reported, 1, 'and reports exactly one error' ); + like( $reported[0][1], qr/nosuchhost/, 'naming the server that failed' ); + is( $reported[0][0], 'cn1', 'and the node it happened on' ); +} + +done_testing(); From a07b1f8ae432fb719ea9c644e1cac7b9ca719621 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:02:17 -0300 Subject: [PATCH 17/21] test(postage): nothing covers makescript's syncfiles deferral call site defer_syncfiles_to_postboot has tests, but deleting the whole block out of makescript -- the provmethod override and the call -- left the entire unit suite green. That is precisely how the wrong deletion shipped two commits ago: the override was removed on the mistaken grounds that %image_hash never carries a provmethod, and nothing noticed. makescript needs a management node and a database, so the block is lifted out and eval'd into a scratch package, driven with the hash makescript actually builds. What it pins is the resolution -- an osimage NAME becoming the osimage's real provmethod -- and that the resolved value is what reaches the helper. Verified by mutation rather than by shape: an override that is present but never fires reddens 1-3, and passing $provmethod instead of $effective_provmethod reddens 2-3. Deleting the block, or the override, trips the extraction guard instead, which is a loud failure rather than a silent pass. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- .../unit/postage_makescript_defer_call_site.t | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 xCAT-test/unit/postage_makescript_defer_call_site.t diff --git a/xCAT-test/unit/postage_makescript_defer_call_site.t b/xCAT-test/unit/postage_makescript_defer_call_site.t new file mode 100644 index 000000000..902585ded --- /dev/null +++ b/xCAT-test/unit/postage_makescript_defer_call_site.t @@ -0,0 +1,122 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use File::Spec; +use FindBin; +use Test::More; + +# Regression: makescript's call to defer_syncfiles_to_postboot was covered by nothing. +# +# The helper itself has tests, but deleting the whole block out of makescript -- the +# provmethod override AND the call -- left the entire unit suite green. That is how a wrong +# deletion shipped once already: the override was removed on the mistaken grounds that +# %image_hash never carries a provmethod, and no test noticed. +# +# makescript needs a management node and a database, so the block is lifted out and eval'd into +# a scratch package, driven with the hash makescript actually builds. What is under test is the +# resolution -- an osimage NAME becoming the osimage's real provmethod -- and that the result is +# what reaches the helper. + +my $repo_root = File::Spec->rel2abs( + File::Spec->catdir( $FindBin::Bin, '..', '..' ) +); +my $postage = File::Spec->catfile( + $repo_root, 'xCAT-server', 'lib', 'perl', 'xCAT', 'Postage.pm' +); +plan skip_all => "Postage.pm not found" unless -f $postage; + +my $src = do { local $/; open my $fh, '<', $postage or die $!; <$fh> }; + +# The helper, and the block in makescript that calls it. BAIL_OUT rather than skip, so a rename +# fails loudly instead of silently covering nothing. +my ($helper) = $src =~ /\n(sub defer_syncfiles_to_postboot \{.*?\n\})\n/s; +BAIL_OUT('could not extract defer_syncfiles_to_postboot from Postage.pm') unless defined $helper; + +my ($block) = $src =~ /\n([ ]+my \$effective_provmethod = \$provmethod;\n.*?defer_syncfiles_to_postboot\(\n.*?\);)\n/s; +BAIL_OUT('could not extract the makescript call site from Postage.pm') unless defined $block; +BAIL_OUT('the extracted call site does not consult the image hash') + unless $block =~ /\$image_hash\{\$osimgname\}\{'provmethod'\}/; + +my $driver = <<"CODE"; +$helper + +sub drive { + my (\%a) = \@_; + my \$provmethod = \$a{provmethod}; + my \$osimgname = \$a{osimgname}; + my \$os = \$a{os}; + my \$nodesetstate = \$a{nodesetstate}; + my \$postscripts = \$a{postscripts}; + my \$postbootscripts = \$a{postbootscripts}; + my \%image_hash = \%{ \$a{image_hash} || {} }; +$block + return (\$postscripts, \$postbootscripts, \$effective_provmethod); +} +CODE + +{ + package T; + eval "$driver; 1" or main::BAIL_OUT("could not eval the makescript call site: $@"); +} + +my $POST = "otherpkgs\nsyncfiles\nremoteshell\n"; + +# nodetype.provmethod naming an osimage: getScripts() fills %image_hash from the osimage table, +# so the override is what turns that name into the real provmethod. +{ + my ($post, $postboot, $eff) = T::drive( + os => 'ubuntu24.04', nodesetstate => undef, + provmethod => 'ubuntu24.04-x86_64-install-compute', + osimgname => 'ubuntu24.04-x86_64-install-compute', + image_hash => { 'ubuntu24.04-x86_64-install-compute' => { provmethod => 'install' } }, + postscripts => $POST, postbootscripts => '' ); + is( $eff, 'install', 'an osimage name resolves to the osimage provmethod' ); + unlike( $post, qr/^syncfiles$/m, 'so syncfiles leaves the in-target postscripts' ); + like( $postboot, qr/^syncfiles$/m, 'and is deferred to the postboot scripts' ); +} + +# Without the resolution there is nothing to key off when nodesetstate is absent. +{ + my ($post, $postboot, $eff) = T::drive( + os => 'ubuntu24.04', nodesetstate => undef, + provmethod => 'ubuntu24.04-x86_64-install-compute', + osimgname => 'ubuntu24.04-x86_64-install-compute', + image_hash => {}, + postscripts => $POST, postbootscripts => '' ); + is( $eff, 'ubuntu24.04-x86_64-install-compute', + 'an osimage with no row in the hash leaves the provmethod alone' ); + like( $post, qr/^syncfiles$/m, 'and nothing is deferred' ); +} + +# A literal provmethod is passed straight through -- $osimgname is undef in that case. +{ + my ($post, $postboot, $eff) = T::drive( + os => 'ubuntu24.04', nodesetstate => undef, provmethod => 'install', + osimgname => undef, image_hash => {}, + postscripts => $POST, postbootscripts => '' ); + is( $eff, 'install', 'a literal provmethod is used as-is' ); + unlike( $post, qr/^syncfiles$/m, 'and still defers' ); +} + +# nodesetstate wins when it is set, which is the ordinary nodeset path. +{ + my ($post, $postboot) = T::drive( + os => 'ubuntu24.04', nodesetstate => 'install', + provmethod => 'ubuntu24.04-x86_64-install-compute', + osimgname => 'ubuntu24.04-x86_64-install-compute', + image_hash => {}, postscripts => $POST, postbootscripts => '' ); + unlike( $post, qr/^syncfiles$/m, 'nodesetstate install defers regardless of provmethod' ); + like( $postboot, qr/^syncfiles$/m, 'and syncfiles lands in the postboot scripts' ); +} + +# Not Ubuntu: the block must not touch anything. +{ + my ($post, $postboot) = T::drive( + os => 'rhels9', nodesetstate => 'install', provmethod => 'install', + osimgname => undef, image_hash => {}, postscripts => $POST, postbootscripts => '' ); + like( $post, qr/^syncfiles$/m, 'a non-Debian OS keeps syncfiles where it was' ); + is( $postboot, '', 'and gains no postboot scripts' ); +} + +done_testing(); From 173cbea55049cb0a044a923dc31871f419c60c71 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:44:17 -0300 Subject: [PATCH 18/21] test(subiquity): three values the Ubuntu install path accepts and cannot use The Subiquity diskful path builds three values from configuration, and nothing checks that the value it builds is one the consumer can use. The boot flip addresses port 3002. xcatd's install monitor listens on site.xcatiport, so a cluster that moves the port loses the flip and every node PXE-loops. The exchange also counts any reply as an accepted request, so a different service on that port reads as a flipped node. xcatd greets with "ready" and answers every request with "done". subiquity_nfsroot_server resolves the install server without a family. A dual-stack management node answers with its IPv6 address, and nfsroot=: is unparseable, because klibc nfsmount takes everything after the first colon as the path. The DNS setup writes the xcatmaster name as a nameserver when getent fails, which is the case the step exists to prevent. ubuntu_subiquity_boot_flip.t drives the template's own late-command against a stand-in monitor. ubuntu_resolvconf_ip.t runs the DNS step and reads the file it writes. debian_subiquity_boot_params.t drives the helpers with a dual-stack resolver. All three fail on this commit. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-test/unit/debian_subiquity_boot_params.t | 23 +++++++ xCAT-test/unit/ubuntu_resolvconf_ip.t | 34 +++++++--- xCAT-test/unit/ubuntu_subiquity_boot_flip.t | 62 +++++++++++++------ 3 files changed, 89 insertions(+), 30 deletions(-) diff --git a/xCAT-test/unit/debian_subiquity_boot_params.t b/xCAT-test/unit/debian_subiquity_boot_params.t index 7a658864a..c22430565 100644 --- a/xCAT-test/unit/debian_subiquity_boot_params.t +++ b/xCAT-test/unit/debian_subiquity_boot_params.t @@ -77,6 +77,29 @@ my $resolver = sub { $_[0] eq 'mn.cluster' ? '10.0.0.1' : undef }; 'while the cloud-init seed URL keeps the name, where DNS works' ); } +# A dual-stack management node has both an A and an AAAA record, and getipaddr answers with +# whichever the resolver returns first. casper mounts the live filesystem with klibc's nfsmount, +# which takes an IPv4 address, and it takes everything after the first colon in nfsroot= as the +# path -- so an IPv6 address there is not a slower path, it is an unparseable one. This install +# asks for ip=dhcp, so the address must be IPv4. +{ + no warnings 'redefine'; + local *xCAT::NetworkUtils::getipaddr = sub { + my ($class, $host, %opt) = @_; + return $opt{OnlyV4} ? '10.0.0.1' : '2001:db8::1'; + }; + + my ($kcmdline, $err) = T::subiquity_boot_params( + 'nofb utf8 auto xcatd=mn.cluster', 'mn.cluster', + '/install/ubuntu24.04/x86_64', '80', 'cn1' ); + ok( !defined $err, 'a dual-stack install server produces a boot config' ) + or diag("error was: $err"); + like( $kcmdline, qr{nfsroot=10\.0\.0\.1:/install/ubuntu24\.04/x86_64}, + 'and nfsroot carries the IPv4 address of the install server' ); + unlike( $kcmdline, qr/nfsroot=[^ ]*::/, + 'never the IPv6 address, which klibc nfsmount cannot use' ); +} + # The guard the original commit added must survive: a name that does not resolve is an error, # not a command line with a name in nfsroot. { diff --git a/xCAT-test/unit/ubuntu_resolvconf_ip.t b/xCAT-test/unit/ubuntu_resolvconf_ip.t index 325c835ab..a82340add 100644 --- a/xCAT-test/unit/ubuntu_resolvconf_ip.t +++ b/xCAT-test/unit/ubuntu_resolvconf_ip.t @@ -9,7 +9,9 @@ use Test::More; # A nameserver line in /etc/resolv.conf must hold an IP address: glibc's resolver discards an # entry naming a host. Writing the xcatmaster *name* left the installer -- and the in-target # apt-get that inherits the file -- with no usable DNS, so the install hung resolving -# archive.ubuntu.com. The template resolves the name to an address first. +# archive.ubuntu.com. The template resolves the name to an address first, and when the name does +# not resolve it keeps the resolver the live installer already got from DHCP, because a name +# written into resolv.conf resolves nothing. # # Run the template's own shell for that step and inspect the file it writes. @@ -20,16 +22,22 @@ open(my $fh, '<', $tmpl) or die "open $tmpl: $!"; my $source = do { local $/; <$fh> }; close $fh; -my ($fragment) = $source =~ m{^(\s*xcatmaster_host=.*?echo "nameserver [^\n]*\n)}ms; +my ($fragment) = $source =~ m{^(\s*xcatmaster_host=.*?\n)\s*echo "=== early-commands complete}ms; BAIL_OUT('the template does not build /etc/resolv.conf from the xcatmaster') unless $fragment; -# $NODE and the xcatmaster come from the xCAT template renderer; stand in for both. +# $NODE, the xcatmaster and the domain come from the xCAT template renderer; stand in for them. sub write_resolv_conf { my (%opt) = @_; my $root = tempdir(CLEANUP => 1); + # What DHCP left behind in the live installer, which the step either replaces or keeps. + open my $seed, '>', "$root/resolv.conf" or die $!; + print {$seed} "nameserver 192.168.0.53\n"; + close $seed; + my $script = $fragment; $script =~ s/\#TABLE:noderes:\$NODE:xcatmaster\#/$opt{xcatmaster}/; + $script =~ s/\#TABLE:site:key=domain:value\#/cluster/; $script =~ s{/etc/resolv\.conf}{$root/resolv.conf}g; # The fragment contains `rm -f /etc/resolv.conf` and this suite runs as root in CI, so a @@ -59,32 +67,38 @@ sub write_resolv_conf { # --- the case the fix exists for ------------------------------------------- { my $r = write_resolv_conf(xcatmaster => 'xcatmn', resolves => '10.0.0.1'); - is($r->{content}, "nameserver 10.0.0.1\n", + like($r->{content}, qr/^nameserver 10\.0\.0\.1$/m, 'the nameserver line holds the address, which glibc will actually use'); unlike($r->{content}, qr/nameserver \s+ xcatmn/x, 'the nameserver line never holds a host name, which glibc discards'); + like($r->{content}, qr/^domain cluster$/m, 'and the search domain is written'); } # --- more than one address: the first is taken ----------------------------- { my $r = write_resolv_conf(xcatmaster => 'xcatmn', resolves => '10.0.0.1'); - like($r->{content}, qr/^nameserver \d+\.\d+\.\d+\.\d+$/m, - 'exactly one IPv4 address is written'); + my @ns = ($r->{content} =~ /^nameserver (\S+)$/mg); + is_deeply(\@ns, ['10.0.0.1'], 'exactly one IPv4 address is written'); } -# --- resolution fails: fall back rather than write an empty entry ---------- +# --- resolution fails: keep the resolver DHCP gave the live installer ------ +# Writing the name back was the original defect. It leaves the installer with no DNS, and the +# in-target apt inherits the same file. { my $r = write_resolv_conf(xcatmaster => 'xcatmn'); - is($r->{content}, "nameserver xcatmn\n", - 'an unresolvable xcatmaster falls back to its name rather than an empty nameserver'); + unlike($r->{content}, qr/^nameserver \s* xcatmn/xm, + 'an unresolvable xcatmaster is never written as a nameserver'); unlike($r->{content}, qr/^nameserver\s*$/m, 'no empty nameserver line is written'); + like($r->{content}, qr/^nameserver 192\.168\.0\.53$/m, + 'the resolver DHCP gave the live installer is kept instead'); } # --- an xcatmaster already given as an address is left alone --------------- { my $r = write_resolv_conf(xcatmaster => '10.0.0.1', resolves => '10.0.0.1'); - is($r->{content}, "nameserver 10.0.0.1\n", 'an address-valued xcatmaster is written as is'); + like($r->{content}, qr/^nameserver 10\.0\.0\.1$/m, + 'an address-valued xcatmaster is written as is'); } done_testing(); diff --git a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t index 7cc1a9ab2..cf98f2e8e 100644 --- a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t +++ b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t @@ -19,10 +19,9 @@ use Test::More; my $tmpl = "$FindBin::Bin/../../xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl"; plan skip_all => 'compute.subiquity.tmpl not found' unless -r $tmpl; -# The template addresses the install-monitor on 3002, and on any management node xcatd is -# already listening there -- so binding it here made the whole file skip_all exactly where the -# suite runs. Take an ephemeral port from the kernel instead and rewrite the extracted command -# to use it: what is under test is the retry-and-log behaviour, not the port number. +# The install monitor listens on site.xcatiport, and on any management node xcatd is already +# there -- so binding it here made the whole file skip_all exactly where the suite runs. Take an +# ephemeral port from the kernel instead and hand it to the template as the site value. my $probe = IO::Socket::INET->new( LocalAddr => '127.0.0.1', LocalPort => 0, Proto => 'tcp', Listen => 5, ReuseAddr => 1) @@ -38,26 +37,20 @@ close $fh; my ($command) = $source =~ m{- \['bash', '-c', '(.*?/dev/tcp/.*?)'\]}; BAIL_OUT('no late-command in the template performs the boot flip over /dev/tcp') unless $command; -# Read the port out of the template rather than hard-coding it, so a template that moves the -# install-monitor still gets covered instead of silently testing the wrong port. -my ($TEMPLATE_PORT) = $command =~ m{/dev/tcp/\$xm/(\d+)}; -BAIL_OUT('could not read the install-monitor port from the boot-flip command') - unless $TEMPLATE_PORT; -is($TEMPLATE_PORT, 3002, 'the template addresses the install-monitor port xcatd listens on'); - # Run the command with the install server pointed at our stand-in, and its log inside a scratch -# tree. Everything else is the template's own text. +# tree. site.xcatiport is substituted the way the template renderer substitutes it. Everything +# else is the template's own text. sub run_flip { my (%opt) = @_; my $root = tempdir(CLEANUP => 1); mkdir "$root/target"; mkdir "$root/target/var"; mkdir "$root/target/var/log"; mkdir "$root/target/var/log/xcat"; + my $site_port = exists $opt{site_port} ? $opt{site_port} : $XCATD_PORT; + my $script = $command; $script =~ s/\#XCATVAR:XCATMASTER\#/127.0.0.1/; - # point the flip at the ephemeral listener, in both the /dev/tcp target and the log message - $script =~ s{/dev/tcp/\$xm/\Q$TEMPLATE_PORT\E\b}{/dev/tcp/\$xm/$XCATD_PORT}; - $script =~ s{\$xm:\Q$TEMPLATE_PORT\E\b}{\$xm:$XCATD_PORT}g; + $script =~ s/\#TABLEBLANKOKAY:site:key=xcatiport:value\#/$site_port/; $script =~ s{/target/var/log/xcat/xcat\.log}{$root/target/var/log/xcat/xcat.log}; $script =~ s/sleep 5/sleep 1/; # shorten the retry pause, keep the retry @@ -75,10 +68,11 @@ sub run_flip { my $c = $srv->accept() or last; $c->autoflush(1); if ($opt{mute}) { sleep 600; close $c; next } # accept and hold, never answer - print {$c} "ready\n"; + # xcatd greets with "ready", then answers every request with "done". + print {$c} ($opt{greeting} || "ready\n"); my $line = <$c>; print {$seen} $line if defined $line; - print {$c} "ok\n" unless $opt{no_ack}; + print {$c} ($opt{ack} || "done\n") unless $opt{no_ack}; close $c; } close $seen; @@ -95,10 +89,10 @@ sub run_flip { if ($pid) { kill 'TERM', $pid; waitpid($pid, 0) } my $received = ''; - if (open my $rh, '<', "$root/received") { local $/; $received = <$rh>; close $rh } + if (open my $rh, '<', "$root/received") { local $/; $received = <$rh> || ''; close $rh } my $log = ''; - if (open my $lh, '<', "$root/target/var/log/xcat/xcat.log") { local $/; $log = <$lh>; close $lh } + if (open my $lh, '<', "$root/target/var/log/xcat/xcat.log") { local $/; $log = <$lh> || ''; close $lh } return { rc => $rc, timed_out => $timed_out, log => $log, received => $received }; } @@ -112,6 +106,16 @@ sub run_flip { ok(!$r->{timed_out}, 'the exchange completes rather than hanging the late-command'); } +# --- the port comes from site.xcatiport ------------------------------------ +# The case above already proves it: the stand-in listens on an ephemeral port, not on 3002, and +# the exchange only completes because the template asks site for the port. What is left is the +# default, for a site table that does not carry the key. +{ + my $r = run_flip(site_port => '', listen => 0, cap => 90); + like($r->{log}, qr/:3002\b/, + 'an unset site.xcatiport falls back to the port xcatd listens on by default'); +} + # --- xcatd never answers: the failure is recorded, not swallowed ----------- { my $r = run_flip(listen => 0); @@ -122,9 +126,27 @@ sub run_flip { 'the log names the install server and port that could not be reached'); } +# --- something else is listening on the port ------------------------------- +# Only xcatd's install monitor answers "nodeset next". A service that accepts the +# connection and talks its own protocol must not be counted as a flipped node. +{ + my $r = run_flip(listen => 5, greeting => "220 smtp\n"); + like($r->{log}, qr/FAILED to flip/, + 'a peer that does not greet with "ready" is not treated as the install monitor'); + is($r->{received}, '', + 'and the flip token is never sent to it'); +} + +# --- the peer greets but does not acknowledge the request ------------------ +{ + my $r = run_flip(listen => 5, ack => "?\n"); + like($r->{log}, qr/FAILED to flip/, + 'a reply other than "done" is not counted as an accepted request'); +} + # --- xcatd accepts but never acknowledges ---------------------------------- { - my $r = run_flip(listen => 1, no_ack => 1); + my $r = run_flip(listen => 5, no_ack => 1); like($r->{log}, qr/FAILED to flip/, 'a connection without an acknowledgement counts as a failure, not a success'); } From 05fc81f7b5e45eeb43283e12d99f06cae7f61688 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:52:49 -0300 Subject: [PATCH 19/21] fix(subiquity): three values the Ubuntu install path accepts and cannot use The boot flip in compute.subiquity.tmpl addressed port 3002. xcatd's install monitor listens on site.xcatiport, so a cluster that moves the port loses the flip and every node PXE-loops back into the installer. The flip now reads site.xcatiport and keeps 3002 as the default. TABLEBLANKOKAY, because the key is optional and a plain TABLE lookup of an absent key fails the whole template. The flip also counted any reply as an accepted request. It now requires the monitor's "ready" greeting before it sends "next", and "done" afterwards, so a different service on that port is not read as a flipped node. subiquity_nfsroot_server in debian.pm called getipaddr without a family. A dual-stack management node answers with its IPv6 address, and casper takes everything after the first colon in nfsroot= as the path, so the live filesystem never mounts. It now asks for IPv4, as dhcp.pm and mknb.pm do. The DNS setup wrote the xcatmaster name as a nameserver when getent found no address, which is the case the step exists to prevent. It now keeps the resolv.conf DHCP gave the live installer. ubuntu_subiquity_boot_flip.t, debian_subiquity_boot_params.t and ubuntu_resolvconf_ip.t fail on the parent commit and pass here. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-server/lib/xcat/plugins/debian.pm | 6 +++-- .../install/ubuntu/compute.subiquity.tmpl | 27 ++++++++++++------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/xCAT-server/lib/xcat/plugins/debian.pm b/xCAT-server/lib/xcat/plugins/debian.pm index b923c37c8..07fcbfb20 100644 --- a/xCAT-server/lib/xcat/plugins/debian.pm +++ b/xCAT-server/lib/xcat/plugins/debian.pm @@ -521,7 +521,7 @@ sub copycd Arguments: $instserver the install server name, address, or the '!myipfn!' placeholder - $resolver optional coderef, for tests; defaults to NetworkUtils::getipaddr + $resolver optional coderef, for tests; defaults to NetworkUtils::getipaddr, IPv4 only Returns: the value to put in nfsroot, or undef when a real name does not resolve @@ -534,7 +534,9 @@ sub subiquity_nfsroot_server { return undef unless defined($instserver) && length($instserver); return $instserver if $instserver eq '!myipfn!'; - $resolver ||= sub { xCAT::NetworkUtils->getipaddr($_[0]) }; + # A dual-stack management node also has an AAAA record. casper takes everything after the + # first colon in nfsroot= as the path, so an IPv6 address there cannot be parsed. + $resolver ||= sub { xCAT::NetworkUtils->getipaddr($_[0], OnlyV4 => 1) }; return $resolver->($instserver); } diff --git a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl index 4ec7faf20..9da43e4c0 100644 --- a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl +++ b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl @@ -68,12 +68,17 @@ autoinstall: echo "=== DNS setup ===" # glibc's resolver discards a nameserver line naming a host, so writing the xcatmaster # *name* leaves the installer -- and the in-target apt that inherits this file -- with no - # DNS. Resolve it here, while the live installer's DHCP resolv.conf still works. + # DNS. Resolve it here, while the live installer's DHCP resolv.conf still works, and keep + # that file when the name does not resolve. A name written back is the case this step + # exists to prevent. xcatmaster_host="#TABLE:noderes:$NODE:xcatmaster#" xcatmaster_ip="$(getent ahostsv4 "$xcatmaster_host" | awk '{print $1; exit}')" - [ -n "$xcatmaster_ip" ] || xcatmaster_ip="$xcatmaster_host" - rm -f /etc/resolv.conf - echo "nameserver $xcatmaster_ip" >/etc/resolv.conf + if [ -n "$xcatmaster_ip" ]; then + rm -f /etc/resolv.conf + echo "nameserver $xcatmaster_ip" >/etc/resolv.conf + else + echo "xcat: $xcatmaster_host has no IPv4 address; keeping the resolver DHCP supplied" + fi echo "domain #TABLE:site:key=domain:value#" >>/etc/resolv.conf echo "=== early-commands complete ===" late-commands: @@ -108,11 +113,13 @@ autoinstall: curtin in-target --target /target /root/post.script; } >>/target/var/log/xcat/xcat.log 2>&1' # Flip the node to local-disk boot, or it PXE-loops back into the installer on reboot. - # "next" is the request xcatd's install monitor answers with "nodeset next". Send it - # from the live installer, retried, and log a failure rather than silently reinstalling. A - # monitor that died and never came back is a separate problem; see #7759 -- but one that - # accepts the connection and then never answers would block a bare read forever and hang the - # install here, so both reads are bounded. - - ['bash', '-c', 'xm=#XCATVAR:XCATMASTER#; ok=0; for i in 1 2 3 4 5; do if exec 3<>/dev/tcp/$xm/3002; then read -r -t 10 r <&3 || true; printf "next\n" >&3; if read -r -t 10 r <&3; then ok=1; fi; exec 3>&- 3<&-; [ "$ok" = 1 ] && break; fi; sleep 5; done; if [ "$ok" != 1 ]; then echo "xcat: FAILED to flip $(hostname) to local-disk boot via $xm:3002; the node will PXE back into the installer" >>/target/var/log/xcat/xcat.log; fi; exit 0'] + # xcatd's install monitor greets with "ready", then answers "next" with "done" and runs + # "nodeset next". Require both tokens: another service on that port is not a flipped + # node. The monitor listens on site.xcatiport; TABLEBLANKOKAY because an absent key fails a + # plain TABLE lookup, and with it the whole template. + # Retry, and log a failure rather than reinstall silently. Both reads are bounded: a monitor + # that accepts and never answers would block a bare read and hang the install here. A + # monitor that died and never came back is #7759. + - ['bash', '-c', 'xm=#XCATVAR:XCATMASTER#; port="#TABLEBLANKOKAY:site:key=xcatiport:value#"; [ -n "$port" ] || port=3002; ok=0; for i in 1 2 3 4 5; do if exec 3<>/dev/tcp/$xm/$port; then if read -r -t 10 hello <&3 && [ "$hello" = "ready" ]; then printf "next\n" >&3; if read -r -t 10 ack <&3 && [ "$ack" = "done" ]; then ok=1; fi; fi; exec 3>&- 3<&-; [ "$ok" = 1 ] && break; fi; sleep 5; done; if [ "$ok" != 1 ]; then echo "xcat: FAILED to flip $(hostname) to local-disk boot via $xm:$port; the node will PXE back into the installer" >>/target/var/log/xcat/xcat.log; fi; exit 0'] error-commands: - tar -c --ignore-failed-read --transform='s/^/#HOSTNAME#-logs\//' /var/crash /var/log/installer /tmp/pre-install.log /autoinstall.yaml 2>/dev/null |nc -l 8080 From 2b6337d67d59827a289fe9004adb079106c1a22d Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:14:28 -0300 Subject: [PATCH 20/21] test(networkutils): capture OnlyV4 being answered with a cached IPv6 address getipaddr answers from %::hostiphash before it resolves. The bypass tests OnlyV6 and GetAllAddresses and does not test OnlyV4, so a caller that asks for IPv4 is handed whatever the first lookup cached. An unrestricted lookup passes AF_UNSPEC to getaddrinfo, so on a dual-stack management node with an AAAA record it caches the IPv6 address. xcatd is long-lived and the hash is a global, so one earlier caller poisons every OnlyV4 caller after it. debian.pm then writes nfsroot=2001:db8::1:/install, which is not a parseable nfsroot, and the Subiquity install never mounts. dhcp.pm and mknb.pm hold four more OnlyV4 callers with the same exposure. The test also pins what the fix must not break: an IPv4 cache entry is still served to an OnlyV4 caller, and an unrestricted caller still gets its cache hit whatever family it holds. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- .../unit/networkutils_getipaddr_onlyv4.t | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 xCAT-test/unit/networkutils_getipaddr_onlyv4.t diff --git a/xCAT-test/unit/networkutils_getipaddr_onlyv4.t b/xCAT-test/unit/networkutils_getipaddr_onlyv4.t new file mode 100644 index 000000000..4e64315c6 --- /dev/null +++ b/xCAT-test/unit/networkutils_getipaddr_onlyv4.t @@ -0,0 +1,47 @@ +#!/usr/bin/env perl +# +# getipaddr caches a resolved address in %::hostiphash and answers later calls +# from it. The cache bypass tests OnlyV6 and GetAllAddresses, and does not test +# OnlyV4, so a caller that asks for IPv4 can be handed a cached IPv6 address. +# +# The cache is filled by whichever lookup ran first. An unrestricted lookup +# passes AF_UNSPEC to getaddrinfo, and on a dual-stack management node with an +# AAAA record that answers with the IPv6 address, which is then stored. xcatd is +# long-lived and %::hostiphash is a global, so any earlier caller in the process +# poisons every OnlyV4 caller that follows. +# +# What it costs: debian.pm builds the Subiquity install command line with +# getipaddr($host, OnlyV4 => 1) and writes nfsroot=
:/install. With an +# IPv6 address that renders as nfsroot=2001:db8::1:/install, which is not a +# parseable nfsroot, so the installer never mounts and the node never installs. +# dhcp.pm and mknb.pm carry four more OnlyV4 callers with the same exposure. + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::Bin/../../perl-xCAT"; +use Test::More; + +use xCAT::NetworkUtils; + +# The address an unrestricted lookup left behind on a dual-stack host. +$::hostiphash{'mn.cluster'}{hostip} = '2001:db8::1'; + +my $only_v4 = xCAT::NetworkUtils->getipaddr('mn.cluster', OnlyV4 => 1); + +ok(!defined($only_v4) || $only_v4 !~ /:/, + 'OnlyV4 does not return the IPv6 address an earlier lookup cached') + or diag("getipaddr returned '$only_v4', which renders as nfsroot=$only_v4:/install"); + +# An IPv4 entry must still be served from the cache: the bypass is about the +# family of the cached answer, not about disabling the cache for OnlyV4. +$::hostiphash{'v4.cluster'}{hostip} = '10.1.2.3'; +is(xCAT::NetworkUtils->getipaddr('v4.cluster', OnlyV4 => 1), '10.1.2.3', + 'an IPv4 cache entry is still served to an OnlyV4 caller'); + +# An unrestricted caller keeps its cache hit, whatever family it holds. +is(xCAT::NetworkUtils->getipaddr('mn.cluster'), '2001:db8::1', + 'a caller that did not ask for IPv4 still gets the cached address'); + +done_testing(); From a3e16d83f96224b62e41b141da68c960b0946556 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:17:04 -0300 Subject: [PATCH 21/21] fix(networkutils): OnlyV4 is answered from the cache whatever family it holds getipaddr returns a cached address before it resolves, and the bypass tests OnlyV6 and GetAllAddresses only. A caller asking OnlyV4 therefore receives whatever the first lookup stored. An unrestricted lookup asks for AF_UNSPEC, so on a dual-stack management node it stores the AAAA record, and %::hostiphash is a global in a long-lived xcatd. debian.pm resolves the install server with OnlyV4 and writes nfsroot=
:/install. Given the cached IPv6 that renders nfsroot=2001:db8::1:/install, which does not parse, so the Subiquity installer never mounts and the node never completes. dhcp.pm and mknb.pm hold four more OnlyV4 callers with the same exposure. Skip the cache when OnlyV4 is asked for and the cached address is IPv6. The lookup that follows requests AF_INET and replaces the entry with the IPv4 address, so the next caller of either kind gets a usable one. An IPv4 entry is still served from the cache: the bypass is about the family of the cached answer, not about disabling the cache for OnlyV4. networkutils_getipaddr_onlyv4.t fails without this change. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- perl-xCAT/xCAT/NetworkUtils.pm | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/perl-xCAT/xCAT/NetworkUtils.pm b/perl-xCAT/xCAT/NetworkUtils.pm index 02b0fe8ea..92e16631e 100644 --- a/perl-xCAT/xCAT/NetworkUtils.pm +++ b/perl-xCAT/xCAT/NetworkUtils.pm @@ -288,8 +288,20 @@ sub getipaddr #print "============================\n"; #cache, do not lookup DNS each time + # + # An unrestricted lookup asks for AF_UNSPEC and caches whatever came back, which + # on a dual-stack host is the AAAA record. Serving that to an OnlyV4 caller gives + # it an IPv6 address it cannot use: debian.pm renders nfsroot=
:, + # and nfsroot=2001:db8::1:/install does not parse. + my $cached_v6 = + defined($::hostiphash{$iporhost}) + && $::hostiphash{$iporhost} + && $::hostiphash{$iporhost}{hostip} + && $::hostiphash{$iporhost}{hostip} =~ /:/; if ( - ((not $extraarguments{OnlyV6}) and (not $extraarguments{GetAllAddresses})) and defined($::hostiphash{$iporhost}) and $::hostiphash{$iporhost}) + ((not $extraarguments{OnlyV6}) and (not $extraarguments{GetAllAddresses})) + and (not($extraarguments{OnlyV4} and $cached_v6)) + and defined($::hostiphash{$iporhost}) and $::hostiphash{$iporhost}) { if($extraarguments{GetNumber} ) {