diff --git a/xCAT-server/lib/perl/xCAT/Postage.pm b/xCAT-server/lib/perl/xCAT/Postage.pm index b7d44b9d3..ddf949b67 100644 --- a/xCAT-server/lib/perl/xCAT/Postage.pm +++ b/xCAT-server/lib/perl/xCAT/Postage.pm @@ -546,6 +546,39 @@ sub makescript { my $postbootscripts; $postbootscripts = getPostbootScripts($node, $osimgname, $script_hash); + # On Ubuntu/Debian the DISKFUL install runs a node's postscripts inside the installer's + # in-target chroot -- before the node has booted as itself. The syncfiles postscript + # works by asking the management node to scp files INTO the running node, which cannot + # happen in that phase: the not-yet-booted node has no sshd for the MN to reach, so the + # push times out, syncfiles exits 1, and the node reports status=failed even though the + # OS installed perfectly. Defer it to the postbootscripts, which run on the booted node + # where ssh is already listening and the MN's push succeeds. + # + # Scope this to the diskful install path only: netboot and statelite already run their + # postscripts on the booted node, so moving syncfiles there would change behaviour that + # works. EL/SLES are unaffected either way -- their postscripts run on the booted node. + # syncfiles is PREPENDED so it still runs before any postbootscript that consumes the + # files it synchronises. + if (defined($os) && $os =~ /^(?:ubuntu|debian)/i) { + my $effective_provmethod = $provmethod; + if ($osimgname && defined($image_hash{$osimgname}{'provmethod'})) { + $effective_provmethod = $image_hash{$osimgname}{'provmethod'}; + } + my $diskful_install = + ($nodesetstate && $nodesetstate eq 'install') + || (!$nodesetstate + && defined($effective_provmethod) + && $effective_provmethod eq 'install'); + if ($diskful_install + && defined($postscripts) + && $postscripts =~ s/^[ \t]*syncfiles[ \t]*\n//m) { + $postbootscripts = "" unless defined $postbootscripts; + $postbootscripts = + "# ubuntu-deferred-postbootscripts-start-here\nsyncfiles\n# ubuntu-deferred-postbootscripts-end-here\n" + . $postbootscripts; + } + } + # if using zones then must go to the zone.sshbetweennodes # else go to site.sshbetweennodes my $enablesshbetweennodes; diff --git a/xCAT-server/lib/perl/xCAT/Template.pm b/xCAT-server/lib/perl/xCAT/Template.pm index bcf37d647..a1f71aafd 100644 --- a/xCAT-server/lib/perl/xCAT/Template.pm +++ b/xCAT-server/lib/perl/xCAT/Template.pm @@ -1686,8 +1686,27 @@ sub ubuntu_subiquity_apt_config ' primary:', " - uri: $online_mirror", ); - if (@otherpkg_sources) { + # On a classic-sources release (20.04/22.04) Subiquity renders the target's + # /etc/apt/sources.list from the install media alone (deb file:///cdrom), which lacks + # packages xCAT needs such as chrony -- curtin's in-target apt then fails with + # "E: Unable to locate package chrony". `sources_list:` is a curtin key that + # Subiquity's autoinstall schema ignores, so add the online archive through `sources:`, + # which Subiquity honours by writing /etc/apt/sources.list.d/*.list. $RELEASE is + # substituted with the release codename by curtin. + # + # Deb822 releases (24.04+) are deliberately excluded: there the primary mirror above + # already lands in /etc/apt/sources.list.d/ubuntu.sources, so adding these legacy .list + # files would configure the same suites twice. + my $need_sources_block = !$use_deb822; + if ($need_sources_block) { push @lines, ' sources:'; + push @lines, ' xcat-ubuntu-archive.list:'; + push @lines, qq( source: "deb $online_mirror \$RELEASE main restricted universe multiverse"); + push @lines, ' xcat-ubuntu-updates.list:'; + push @lines, qq( source: "deb $online_mirror \$RELEASE-updates main restricted universe multiverse"); + } + if (@otherpkg_sources) { + push @lines, ' sources:' unless $need_sources_block; my $index = 0; foreach my $source (@otherpkg_sources) { push @lines, " xcat-otherpkgs-$index.list:"; diff --git a/xCAT-server/lib/xcat/plugins/debian.pm b/xCAT-server/lib/xcat/plugins/debian.pm index 4051ca18e..04b470bf4 100644 --- a/xCAT-server/lib/xcat/plugins/debian.pm +++ b/xCAT-server/lib/xcat/plugins/debian.pm @@ -986,7 +986,27 @@ sub mkinstall { my $kcmdline = "nofb utf8 auto xcatd=" . $instserver; if (using_subiquity($os,$tmplfile)) { - $kcmdline .= " autoinstall ip=dhcp netboot=nfs nfsroot=${instserver}:${pkgdir}"; + # boot=casper is required: without it casper never processes netboot=nfs, it + # scans the local disks, finds no live media and panics "Unable to find a + # medium containing a live file system" (initramfs emergency shell -> PXE loop). + # nfsroot MUST be a literal IP: casper mounts the live filesystem with klibc's + # nfsmount, which cannot resolve hostnames ("nfsmount: can't parse IP address + # ''"), so resolve instserver to its IP. The ds= URL is fetched later by + # cloud-init in the booted live system where normal DNS works, so it may stay a + # hostname. + # 'toram' makes casper copy the live squashfs into RAM and UNMOUNT the NFS source + # (casper scripts/casper: copy_to_ram then `umount ${copyfrom}`), so the installer runs + # from RAM with NO network root. This fixes the end-of-install reboot hang: with the NFS + # root still mounted, a process doing I/O to it during systemd-shutdown (lvm, netplan, + # udev) blocks in uninterruptible D state -- it cannot be SIGKILLed, so systemd-shutdown + # waits forever ("Waiting for process: (lvm)") and the node never power-cycles into + # the installed disk. With the root in RAM there is nothing to wait on and the reboot + # completes on its own. (casper has no cmdline knob for NFS mount options -- it parses + # only nfsroot=, taking the whole value as the path, so appending ,soft breaks the mount; + # toram is casper's supported way to avoid the network root. The 24.04 layers total + # ~1.5G, well within the CN's RAM.) + my $nfsip = xCAT::NetworkUtils->getipaddr($instserver) || $instserver; + $kcmdline .= " autoinstall ip=dhcp boot=casper netboot=nfs nfsroot=${nfsip}:${pkgdir} toram"; $kcmdline .= " ds=nocloud-net;s=http://${instserver}:${httpport}/install/autoinst/${node}/"; $kcmdline .= " ---"; } else { diff --git a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl index 31a978cbc..547a48aa1 100644 --- a/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl +++ b/xCAT-server/share/xcat/install/ubuntu/compute.subiquity.tmpl @@ -66,8 +66,17 @@ autoinstall: sed -i '/^\.\.\.$/d' /autoinstall.yaml cat /tmp/partitionfile >> /autoinstall.yaml echo "=== DNS setup ===" + # A nameserver line in /etc/resolv.conf MUST hold an IP address -- glibc's resolver does + # NOT resolve a hostname written there, it discards the entry. Writing the xcatmaster + # *name* leaves the installer (and the in-target apt-get, which inherits this file) with + # no usable DNS, so the install hangs resolving archive.ubuntu.com. DNS still works at + # this point through the live installer's DHCP resolv.conf -- the wget calls above relied + # on it -- so resolve the xcatmaster to an IPv4 address first and write that. + xcatmaster_host="#TABLE:noderes:$NODE:xcatmaster#" + xcatmaster_ip="$(getent ahostsv4 "$xcatmaster_host" | awk '{print $1; exit}')" + [ -n "$xcatmaster_ip" ] || xcatmaster_ip="$xcatmaster_host" rm -f /etc/resolv.conf - echo "nameserver #TABLE:noderes:$NODE:xcatmaster#" >/etc/resolv.conf + echo "nameserver $xcatmaster_ip" >/etc/resolv.conf echo "domain #TABLE:site:key=domain:value#" >>/etc/resolv.conf echo "=== early-commands complete ===" late-commands: @@ -101,5 +110,16 @@ autoinstall: cp ./#HOSTNAME#.post /target/root/post.script; curtin in-target --target /target /root/post.script; } >>/target/var/log/xcat/xcat.log 2>&1' + # Switch the node to boot from the local disk now the install is done, or it PXE-loops + # straight back into the installer when Subiquity reboots. xCAT flips the netboot state + # when the node reports "next" to xcatd on the install-monitor port, which makes xcatd run + # "nodeset next" and rewrite the node's xNBA script to fall through to local disk. + # The in-target post-script tries this through updateflag.awk, but that needs gawk's |& + # /inet coprocess and /usr/bin/awk on Ubuntu is normally mawk, so the flip silently fails. + # Send it from the live installer instead -- the connection then originates from the node's + # own IP, which xcatd trusts -- using bash's built-in /dev/tcp, with no gawk dependency. + # xcatd answers with a ready token, takes "next", and acknowledges; treat anything else as + # a failure and say so in the install log rather than rebooting into another install. + - ['bash', '-c', 'xm=#XCATVAR:XCATMASTER#; ok=0; for i in 1 2 3 4 5; do if exec 3<>/dev/tcp/$xm/3002; then read -r r <&3 || true; printf "next\n" >&3; if read -r r <&3; then ok=1; fi; exec 3>&- 3<&-; [ "$ok" = 1 ] && break; fi; sleep 5; done; if [ "$ok" != 1 ]; then echo "xcat: FAILED to flip $(hostname) to local-disk boot via $xm:3002; the node will PXE back into the installer" >>/target/var/log/xcat/xcat.log; fi; exit 0'] error-commands: - tar -c --ignore-failed-read --transform='s/^/#HOSTNAME#-logs\//' /var/crash /var/log/installer /tmp/pre-install.log /autoinstall.yaml 2>/dev/null |nc -l 8080 diff --git a/xCAT-test/autotest/testcase/commoncmd/retry_install.sh b/xCAT-test/autotest/testcase/commoncmd/retry_install.sh index 9f4c9e24e..d4589e673 100755 --- a/xCAT-test/autotest/testcase/commoncmd/retry_install.sh +++ b/xCAT-test/autotest/testcase/commoncmd/retry_install.sh @@ -7,7 +7,7 @@ node=$1 osimage=$2 vmhost=`lsdef $node -i vmhost -c | cut -d '=' -f 2` times=2 -wait_for_provision=20 #Min to wait for node to provision +wait_for_provision=${WAIT_FOR_PROVISION:-20} #Min to wait for node to provision (overridable via WAIT_FOR_PROVISION; kept short so a boot-loop fails fast, raised only where the happy path is slow -- e.g. Ubuntu subiquity diskful) check_status=10 #Sec to keep checking status iterations=$wait_for_provision*60/$check_status #Iterations to check for "booted" status