From 8e2fbbf2772882ab51663201df931dc80ce27698 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 14 Sep 2026 19:18:39 -0300 Subject: [PATCH 1/2] test(goconserver): cover the Go toolchain the Ubuntu build installs The riscv64 goconserver build never finished. It installs a Go toolchain built FOR the chroot architecture, so on riscv64 the compiler itself runs under qemu-user: three xcat-dep-ubuntu-cd cells (jammy, noble, resolute) each burned the full 9000s budget, two of them with no CPU ticks at all in the stall sample. Nothing asserted which toolchain the build fetches, or which architecture it compiles for. This test lifts the build shell out of goconserver/sbuild.pl and runs it with the commands it calls shadowed, then asserts on what the run asked for: the toolchain tarball must name the build host, and the real debian/rules must reach `go build` with GOARCH set to the chroot architecture. The recorders refuse any write outside the scratch tree and report it, so the build's `rm -rf /usr/local/go` is an assertion here rather than damage to the host. It fails on the current builder: the toolchain is fetched for riscv64, GOARCH is unset and two writes escape the build tree. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- .github/workflows/genesis-openembedded.yml | 1 + goconserver/sbuild.pl | 2 + t/goconserver_cross_build.t | 182 +++++++++++++++++++++ 3 files changed, 185 insertions(+) create mode 100644 t/goconserver_cross_build.t diff --git a/.github/workflows/genesis-openembedded.yml b/.github/workflows/genesis-openembedded.yml index 9d47f94..dc7d2b6 100644 --- a/.github/workflows/genesis-openembedded.yml +++ b/.github/workflows/genesis-openembedded.yml @@ -58,6 +58,7 @@ jobs: prove -v t/build_utils.t prove -v t/build_timeout.t prove -v t/sbuild-all.t + prove -v t/goconserver_cross_build.t prove -v t/mockbuild-all.t prove -v -It/lib t/genesis_openembedded_release.t sudo -E prove -v -It/lib t/genesis_openembedded_consumer.t diff --git a/goconserver/sbuild.pl b/goconserver/sbuild.pl index cf509cb..6c77988 100755 --- a/goconserver/sbuild.pl +++ b/goconserver/sbuild.pl @@ -44,6 +44,8 @@ $build_timestamp = time() unless defined $build_timestamp; # The maintained debian/ is at ./debian in the copied package dir; the upstream source is cloned fresh # at the pinned SHA into ./gcsrc, the maintained debian/ copied in, and dpkg-buildpackage run there # (its .deb(s) land in the copied package dir, which the collector picks up). +# The build script below is lifted by t/goconserver_cross_build.t and run with the commands it +# calls shadowed. Keep the marker: the test dies when it can no longer find this region. my $build = <<'BUILD'; set -e VERSION=0.3.3 diff --git a/t/goconserver_cross_build.t b/t/goconserver_cross_build.t new file mode 100644 index 0000000..5fea87e --- /dev/null +++ b/t/goconserver_cross_build.t @@ -0,0 +1,182 @@ +#!/usr/bin/perl +# Behaviour test for the Go toolchain the Ubuntu goconserver build uses. +# +# riscv64 has no build host, so its chroot runs under qemu-user. A Go toolchain built FOR riscv64 +# therefore runs emulated, and `go build` parks its threads in futex_wait and never finishes: three +# xcat-dep-ubuntu-cd riscv64 cells burned the whole 9000s budget with no CPU ticks at all. Go +# cross-compiles, so the toolchain must be the BUILD HOST's and the target must come from GOARCH. +# +# The test LIFTS the build shell out of goconserver/sbuild.pl, RUNS it, and asserts on what the run +# asked for -- the toolchain tarball it fetched, and the environment the real debian/rules passed to +# `go build`. It never matches the source of the thing it tests. +# +# Every command that could write outside the scratch tree is shadowed by a recorder that refuses the +# write and reports it, so a build that reaches for /usr/local is a FAILED assertion here rather than +# damage to the host running the suite. +use strict; +use warnings; +use Test::More; +use File::Temp qw(tempdir); +use File::Path qw(make_path); +use FindBin qw($RealBin); + +my $pkg_dir = "$RealBin/../goconserver"; +plan skip_all => 'goconserver/sbuild.pl not found' unless -f "$pkg_dir/sbuild.pl"; +plan skip_all => 'bash is not available' unless -x '/bin/bash'; + +my $LIFT = 'lifted by t/goconserver_cross_build.t'; + +# The build shell, produced by the real builder code. The whole marked region is evaluated, so the +# architecture the builder stamps into the script comes from the builder rather than from this test. +# die (never BAIL_OUT) when the lift stops matching: prove stops the WHOLE suite on a bail-out, and a +# silent miss would leave this file covering nothing. +sub build_script { + open(my $fh, '<', "$pkg_dir/sbuild.pl") or die "read sbuild.pl: $!"; + my $src = do { local $/; <$fh> }; + close($fh); + my ($region) = $src =~ /^\#[^\n]*\Q$LIFT\E[^\n]*\n(.*?^BUILD$)/ms + or die "goconserver/sbuild.pl no longer marks its build script with '$LIFT' -- " + . "this test can no longer reach the code it covers\n"; + my $build = eval "$region\n\$build"; ## no critic + die "could not evaluate the lifted build script: $@\n" if $@; + die "the lifted build script is empty\n" unless defined $build && $build =~ /\S/; + return $build; +} + +sub write_stub { + my ($dir, $name, $body) = @_; + open(my $fh, '>', "$dir/$name") or die "write $dir/$name: $!"; + print {$fh} "#!/bin/bash\n$body\n"; + close($fh); + chmod(0755, "$dir/$name") or die "chmod $dir/$name: $!"; +} + +# run_build($target_arch): run the build shell with the chroot's architecture reported as +# $target_arch, and return what it asked the outside world to do. +sub run_build { + my ($target_arch) = @_; + my $root = tempdir(CLEANUP => 1); + my ($bin, $rec, $work) = ("$root/bin", "$root/rec", "$root/work"); + make_path($bin, $rec, $work); + + # The build runs with CWD = a copy of the package dir, and reads ../gomod and ./debian from it. + system('cp', '-rL', "$pkg_dir/$_", "$work/$_") == 0 or die "stage $_: $!" for qw(gomod debian); + + # dpkg answers for the CHROOT, which is the architecture the build must produce. + write_stub($bin, 'dpkg', qq{ + [ "\$1" = --print-architecture ] && { echo '$target_arch'; exit 0; } + exec /usr/bin/dpkg "\$\@" + }); + write_stub($bin, 'curl', qq{ + for a in "\$\@"; do case "\$a" in http*) echo "\$a" >> '$rec/curl-urls';; esac; done + exit 0 + }); + # tar and rm police their target: anything outside the scratch tree is recorded, not performed. + write_stub($bin, 'tar', qq{ + dest=''; prev='' + for a in "\$\@"; do [ "\$prev" = -C ] && dest="\$a"; prev="\$a"; done + case "\$dest" in '$root'/*) ;; *) echo "tar -C \$dest" >> '$rec/escapes';; esac + exit 0 + }); + write_stub($bin, 'rm', qq{ + for a in "\$\@"; do + case "\$a" in + -*|'$root'/*) ;; + /*) echo "rm \$a" >> '$rec/escapes'; exit 0;; + esac + done + exec /bin/rm "\$\@" + }); + # Only `git init ` has to have an effect; the clone has no source this test needs. + write_stub($bin, 'git', qq{ + if [ "\$1" = init ]; then shift + for a in "\$\@"; do case "\$a" in -*) ;; *) mkdir -p "\$a";; esac; done + fi + exit 0 + }); + write_stub($bin, 'dch', 'exit 0'); + # The real debian/rules has to see the environment, so run its build target for real. + write_stub($bin, 'dpkg-buildpackage', 'make -f debian/rules override_dh_auto_build'); + # The downloaded toolchain never lands, so `go` always resolves here. It records the environment + # of each invocation, which is the thing under test. + write_stub($bin, 'go', qq{ + echo "GOARCH=\${GOARCH-} GOOS=\${GOOS-} ARGV=\$*" >> '$rec/go-calls' + exit 0 + }); + + open(my $fh, '>', "$root/build.sh") or die "write build.sh: $!"; + print {$fh} build_script(); + close($fh); + + my $rc = system('/bin/bash', '-c', + "cd '$work' && PATH=\"$bin:\$PATH\" SOURCE_DATE_EPOCH=1789413339 " + . "bash '$root/build.sh' > '$root/build.log' 2>&1"); + + my $slurp = sub { + my ($f) = @_; + return () unless -f "$rec/$f"; + open(my $h, '<', "$rec/$f") or return (); + my @l = <$h>; close($h); chomp @l; return @l; + }; + open(my $lh, '<', "$root/build.log") or die "read build.log: $!"; + my $log = do { local $/; <$lh> }; + close($lh); + return { rc => $rc, log => $log // '', + curl => [ $slurp->('curl-urls') ], + go_calls => [ $slurp->('go-calls') ], + escapes => [ $slurp->('escapes') ] }; +} + +# The architecture the toolchain must be built for: this machine's, in Go's spelling. +my $host_deb = `dpkg --print-architecture 2>/dev/null` // ''; +chomp $host_deb; +plan skip_all => 'dpkg is not available' unless $host_deb =~ /^[a-z0-9]+$/; +my $host_go = $host_deb eq 'ppc64el' ? 'ppc64le' : $host_deb; + +# ---- the cell that failed: a riscv64 chroot on this build host ---------------------------------- +my $r = run_build('riscv64'); + +my ($toolchain) = grep { m{/go[\d.]+\.linux-} } @{ $r->{curl} }; +ok(defined $toolchain, 'the build fetches a pinned Go toolchain') + or diag("curl was asked for: @{ $r->{curl} }\n$r->{log}"); + +SKIP: { + skip 'no toolchain download to inspect', 1 unless defined $toolchain; + like($toolchain, qr/\.linux-\Q$host_go\E\.tar\.gz$/, + "the toolchain is built for the build host ($host_go), so it runs natively not under qemu") + or diag("fetched: $toolchain"); +} + +# compiles_for($result, $goarch, $label): every `go build` the run reached was told to emit $goarch. +# A run that reached NO `go build` fails here: an empty list would otherwise satisfy any claim. +sub compiles_for { + my ($res, $goarch, $label) = @_; + my @builds = grep { /ARGV=.*\bbuild\b/ } @{ $res->{go_calls} }; + unless (@builds) { + fail("$label -- the run never reached `go build`"); + diag("go was called: @{ $res->{go_calls} }\nrc=$res->{rc}\n$res->{log}"); + return; + } + is_deeply([ grep { !/\bGOARCH=\Q$goarch\E\b/ } @builds ], [], $label) + or diag("go build calls:\n" . join("\n", @builds)); +} + +compiles_for($r, 'riscv64', + 'every `go build` is told to emit riscv64, so the native toolchain cross-compiles'); + +is_deeply($r->{escapes}, [], + 'the build writes and deletes only inside its own build tree') + or diag("escaped the build tree:\n" . join("\n", @{ $r->{escapes} })); + +# ---- a chroot of the host's own architecture still builds natively ------------------------------ +my $n = run_build($host_deb); +my ($native) = grep { m{/go[\d.]+\.linux-} } @{ $n->{curl} }; +like($native // '', qr/\.linux-\Q$host_go\E\.tar\.gz$/, + 'a native cell fetches the same toolchain'); +compiles_for($n, $host_go, 'a native cell compiles for its own architecture'); + +# ---- dpkg and Go spell the POWER architecture differently ---------------------------------------- +my $p = run_build('ppc64el'); +compiles_for($p, 'ppc64le', 'the dpkg name ppc64el reaches go as ppc64le'); + +done_testing; From 2a4551f05c02341195ef7eb026f484f1970c0742 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Mon, 14 Sep 2026 19:26:29 -0300 Subject: [PATCH 2/2] fix(xcat-dep): the riscv64 goconserver build never finishes xcat-dep-ubuntu-cd build 83 failed in its riscv64 branch. The goconserver build for jammy, noble and resolute each ran the full 9000s budget and produced no deb. The stall report found the Go processes parked in futex_wait, two of the three cells with no CPU ticks at all during the sample. riscv64 has no build host, so its chroot runs on the amd64 agent under qemu-user. goconserver/sbuild.pl installed the Go toolchain for the chroot architecture, so the compiler itself ran emulated, and `go build` did not return. The EL builder already avoids this: goconserver/mockbuild.pl cross-compiles for a forcearch target rather than running the toolchain in the emulated chroot. sbuild.pl now stamps the build host architecture into the build script and fetches the toolchain for that architecture, then sets GOARCH to the chroot architecture. A Go toolchain is statically linked, so the host one runs inside the foreign chroot, and goconserver is CGO-free, so it cross-compiles. The toolchain also unpacks into the build tree instead of overwriting /usr/local/go. A native cell is unchanged: host and target agree, and GOARCH names the architecture it already used. t/goconserver_cross_build.t covers this and fails without the change: the toolchain was fetched for riscv64 and `go build` ran with GOARCH unset. Measured on xcat-master-ub, jammy riscv64: 380s, against a 9000s budget the emulated build exhausted. The deb carries statically linked RISC-V binaries that run under binfmt. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- goconserver/sbuild.pl | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/goconserver/sbuild.pl b/goconserver/sbuild.pl index 6c77988..06e36e8 100755 --- a/goconserver/sbuild.pl +++ b/goconserver/sbuild.pl @@ -46,7 +46,12 @@ $build_timestamp = time() unless defined $build_timestamp; # (its .deb(s) land in the copied package dir, which the collector picks up). # The build script below is lifted by t/goconserver_cross_build.t and run with the commands it # calls shadowed. Keep the marker: the test dies when it can no longer find this region. -my $build = <<'BUILD'; +my $host_deb_arch = `dpkg --print-architecture 2>/dev/null`; +chomp $host_deb_arch; +die "FATAL: cannot read the build host architecture from dpkg\n" + unless $host_deb_arch =~ /^[a-z0-9]+$/; + +my $build = "HOST_DEB_ARCH=$host_deb_arch\n" . <<'BUILD'; set -e VERSION=0.3.3 REPO=https://github.com/xcat2/goconserver.git @@ -54,13 +59,23 @@ REF=6166fe5ec1c5b3c20475e322a9f0e8e93c87e45f GO_PIN=1.25.12 # pinned modern Go toolchain (static CGO-free build, portable across codenames; reproducible compiler) -go_arch=$(dpkg --print-architecture); [ "$go_arch" = ppc64el ] && go_arch=ppc64le -echo "installing pinned go${GO_PIN} (${go_arch}) for the goconserver build" -rm -rf /usr/local/go -curl -fsSL "https://go.dev/dl/go${GO_PIN}.linux-${go_arch}.tar.gz" | tar -C /usr/local -xz -export PATH=/usr/local/go/bin:$PATH +# The toolchain is the BUILD HOST's and the target comes from GOARCH, because riscv64 has no build +# host: its chroot runs under qemu-user, and a riscv64 `go build` there parks in futex_wait and never +# returns. Go cross-compiles a CGO-free binary, and a Go toolchain is statically linked, so the host +# one runs inside the foreign chroot at native speed. HOST_DEB_ARCH is stamped in by sbuild.pl: it +# cannot be read here, because qemu makes the chroot's dpkg and uname both answer for the target. +deb_to_goarch() { case "$1" in ppc64el) echo ppc64le;; *) echo "$1";; esac; } +go_host_arch=$(deb_to_goarch "$HOST_DEB_ARCH") +go_target_arch=$(deb_to_goarch "$(dpkg --print-architecture)") +echo "installing pinned go${GO_PIN} (${go_host_arch}) to compile for ${go_target_arch}" +gotoolchain="$PWD/.gotoolchain" +mkdir -p "$gotoolchain" +curl -fsSL "https://go.dev/dl/go${GO_PIN}.linux-${go_host_arch}.tar.gz" | tar -C "$gotoolchain" --strip-components=1 -xz +export PATH="$gotoolchain/bin:$PATH" export GOTOOLCHAIN=local # use exactly the pinned toolchain; never auto-download another +export GOOS=linux GOARCH="$go_target_arch" go version +go env GOHOSTARCH GOARCH if [ -n "${SOURCE_DATE_EPOCH:-}" ]; then SNAP_TS=$(date -d "@$SOURCE_DATE_EPOCH" --utc '+%Y%m%d%H%M')