From bca12ca29daebadca5d3a88c26b05f65ecb28af9 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:51:55 -0300 Subject: [PATCH] fix(xcat-core): bound the boot-flip exchange, and fail when the install server will not resolve Two ways the Subiquity install could fail without saying anything useful. The boot flip read from the install monitor with no timeout. A monitor that accepts the connection and then never answers -- which #7759 shows is a state it gets into -- blocked the read forever, and with it the Subiquity late-command and the install. The retry loop could not help: it never reached the retry. Both reads now take -t 10, so five attempts are bounded at roughly two minutes and end in the failure that is already logged. The regression test stands up a listener that accepts and holds the connection; removing the timeouts fails it. mkinstall resolved the install server for nfsroot and fell back to the name when that failed: "getipaddr($instserver) || $instserver". The name is the original defect -- klibc's nfsmount cannot resolve one -- so the node panicked "can't parse IP address" at boot, on the node, with nothing reported on the management node. The management node knows at template time, so it says so there and skips the node, as the other unrecoverable per-node conditions in this routine do. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-server/lib/xcat/plugins/debian.pm | 10 +++++++++- .../xcat/install/ubuntu/compute.subiquity.tmpl | 6 ++++-- xCAT-test/unit/ubuntu_subiquity_boot_flip.t | 16 +++++++++++++++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/xCAT-server/lib/xcat/plugins/debian.pm b/xCAT-server/lib/xcat/plugins/debian.pm index a4ac3a6ef..d7cd438d6 100644 --- a/xCAT-server/lib/xcat/plugins/debian.pm +++ b/xCAT-server/lib/xcat/plugins/debian.pm @@ -1032,7 +1032,15 @@ sub mkinstall { my $kcmdline = "nofb utf8 auto xcatd=" . $instserver; if (using_subiquity($os,$tmplfile)) { - my $nfsip = xCAT::NetworkUtils->getipaddr($instserver) || $instserver; + # Fail here rather than handing casper a name: klibc's nfsmount cannot resolve + # one, so the node would panic "can't parse IP address" at boot, on the node, + # with nothing said on the management node. + my $nfsip = xCAT::NetworkUtils->getipaddr($instserver); + unless ($nfsip) { + xCAT::MsgUtils->report_node_error($callback, $node, + "Could not resolve the install server '$instserver' to an address. The Ubuntu live installer mounts its root with klibc nfsmount, which cannot resolve names, so nfsroot must be an address."); + next; + } $kcmdline = subiquity_kcmdline($kcmdline, $nfsip, $pkgdir, $instserver, $httpport, $node); } else { $kcmdline .= " url=http://${instserver}:$httpport/install/autoinst/$node"; diff --git a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl index f82aec459..4ec7faf20 100644 --- a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl +++ b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl @@ -110,7 +110,9 @@ autoinstall: # Flip the node to local-disk boot, or it PXE-loops back into the installer on reboot. # "next" is the request xcatd's install monitor answers with "nodeset next". Send it # from the live installer, retried, and log a failure rather than silently reinstalling. A - # monitor that died and never came back is a separate problem; see #7759. - - ['bash', '-c', 'xm=#XCATVAR:XCATMASTER#; ok=0; for i in 1 2 3 4 5; do if exec 3<>/dev/tcp/$xm/3002; then read -r r <&3 || true; printf "next\n" >&3; if read -r r <&3; then ok=1; fi; exec 3>&- 3<&-; [ "$ok" = 1 ] && break; fi; sleep 5; done; if [ "$ok" != 1 ]; then echo "xcat: FAILED to flip $(hostname) to local-disk boot via $xm:3002; the node will PXE back into the installer" >>/target/var/log/xcat/xcat.log; fi; exit 0'] + # monitor that died and never came back is a separate problem; see #7759 -- but one that + # accepts the connection and then never answers would block a bare read forever and hang the + # install here, so both reads are bounded. + - ['bash', '-c', 'xm=#XCATVAR:XCATMASTER#; ok=0; for i in 1 2 3 4 5; do if exec 3<>/dev/tcp/$xm/3002; then read -r -t 10 r <&3 || true; printf "next\n" >&3; if read -r -t 10 r <&3; then ok=1; fi; exec 3>&- 3<&-; [ "$ok" = 1 ] && break; fi; sleep 5; done; if [ "$ok" != 1 ]; then echo "xcat: FAILED to flip $(hostname) to local-disk boot via $xm:3002; the node will PXE back into the installer" >>/target/var/log/xcat/xcat.log; fi; exit 0'] error-commands: - tar -c --ignore-failed-read --transform='s/^/#HOSTNAME#-logs\//' /var/crash /var/log/installer /tmp/pre-install.log /autoinstall.yaml 2>/dev/null |nc -l 8080 diff --git a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t index a916de89f..8102718d0 100644 --- a/xCAT-test/unit/ubuntu_subiquity_boot_flip.t +++ b/xCAT-test/unit/ubuntu_subiquity_boot_flip.t @@ -61,6 +61,7 @@ sub run_flip { for (1 .. $opt{listen}) { my $c = $srv->accept() or last; $c->autoflush(1); + if ($opt{mute}) { sleep 600; close $c; next } # accept and hold, never answer print {$c} "ready\n"; my $line = <$c>; print {$seen} $line if defined $line; @@ -75,7 +76,8 @@ sub run_flip { # The installer would hang here if the exchange ever blocked, so bound it. # the no-listener case prints "Connection refused" by design - my $rc = system("timeout 25 bash -c \Q$script\E 2>/dev/null"); + my $cap = $opt{cap} || 25; + my $rc = system("timeout $cap bash -c \Q$script\E 2>/dev/null"); my $timed_out = (($rc >> 8) == 124); if ($pid) { kill 'TERM', $pid; waitpid($pid, 0) } @@ -114,6 +116,18 @@ sub run_flip { 'a connection without an acknowledgement counts as a failure, not a success'); } +# --- a monitor that accepts and never answers must not hang the install ---- +# The late-command runs inside Subiquity: a bare read on a socket that is open but silent blocks +# forever and the install never finishes. #7759 fixes the monitor dying; this bounds the wait. +{ + # Five attempts, each bounded by two 10s reads plus the retry pause: ~2 minutes worst case. + my $r = run_flip(listen => 1, mute => 1, cap => 200); + ok(!$r->{timed_out}, + 'a monitor that accepts but never replies does not hang the late-command'); + like($r->{log}, qr/FAILED to flip/, + 'it is recorded as a failed flip rather than waiting indefinitely'); +} + # --- the command retries rather than giving up on the first refusal -------- { # Answer only on a later connection: the flip must still succeed.