From d2d98b72483695c1a8e37bc5d3991f460fcc3f04 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:56:56 -0300 Subject: [PATCH] fix(xcat-core): port CI buildrpms.pl (parallel builds, multi-arch merge) into tree The 3 EL CD pipelines (xcat-core-devel-cd, xcat-core-stable-cd, xcat-dep-el-cd) overlaid a pinned $CI/buildrpms.pl at build time because the tree's buildrpms.pl lacked the options they depend on: - a per-target flock guard alongside --mock-uniqueext, so concurrent same-target builds do not corrupt each other's /var/lib/mock chroot namespace; - --native-only (build only arch-native pkgs on the secondary arch) plus --merge-core-repos/--output-dir/--input-core-repos, replacing --finalize-core, to assemble one signed flat multi-arch core from per-arch build outputs; - sh_retry() to absorb transient mock/nspawn flakes; - a single --target guard and graceful mock cancellation (sweep_mock_mounts/abort_builds) that unmounts chroots on abort. The xCAT-release repository package (master/2.19 only) is preserved: its write_release_alias() is invoked from the new merge_core_repos() and per-target. Porting them in-tree lets CI drop the $CI/buildrpms.pl pin and run the three pipelines in parallel without the cross-job serialize lock. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- buildrpms.pl | 276 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 239 insertions(+), 37 deletions(-) diff --git a/buildrpms.pl b/buildrpms.pl index 0fcdad12d..c5dec9bf0 100755 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -42,6 +42,7 @@ use File::Path qw(make_path remove_tree); use File::Slurper qw(read_text write_text); use File::Temp qw(tempdir tempfile); use FindBin qw($Bin); +use Fcntl qw(:flock); # per-target build lock (concurrency guard; see main()) use Getopt::Long qw(GetOptions); use POSIX qw(strftime); use Parallel::ForkManager; @@ -137,6 +138,12 @@ my @PACKAGES = qw( xCAT-release ); +# The arch-native packages: their rpms carry the target arch. Everything else in @PACKAGES +# is noarch and byte-identical on every arch, so `--native-only` builds just these -- a +# secondary-arch build (e.g. ppc64le) then produces only what the x86_64 build cannot already +# provide, and the multi-arch merge has no duplicate noarch to reconcile. +my @NATIVE_PACKAGES = qw(xCAT xCATsn xCAT-genesis-scripts); + my @TARGETS = ( "$DISTRO+epel-8-$ARCH", "$DISTRO+epel-9-$ARCH", @@ -157,12 +164,13 @@ my %opts = ( packages => \@PACKAGES, release => "", repo_mode => "file", + repo_baseurl => "https://xcat.org/files/xcat/repos/yum/devel/xcat-core", targets => \@TARGETS, verbose => 0, xcat_dep_path => "$PWD/../xcat-dep/", ); -my @cli_packages; +my (@cli_packages, @cli_targets, @cli_input_core_repos); GetOptions( "configure_nginx" => \$opts{configure_nginx}, "force" => \$opts{force}, @@ -174,13 +182,17 @@ GetOptions( "nginx_port" => \$opts{nginx_port}, "nproc=i" => \$opts{nproc}, "package=s@" => \@cli_packages, + "native-only" => \$opts{native_only}, "release=s" => \$opts{release}, "repo-mode=s" => \$opts{repo_mode}, - "target=s@" => \$opts{targets}, + "target=s@" => \@cli_targets, "verbose" => \$opts{verbose}, "xcat_dep_path=s" => \$opts{xcat_dep_path}, "setup_local_repos" => \$opts{setup_local_repos}, - "finalize-core=s" => \$opts{finalize_core}, + "merge-core-repos" => \$opts{merge_core_repos}, + "output-dir=s" => \$opts{output_dir}, + "input-core-repos=s{1,}" => \@cli_input_core_repos, + "repo-baseurl=s" => \$opts{repo_baseurl}, ) or usage(); # --package REPLACES the default set (build exactly what was asked), so @@ -189,6 +201,30 @@ GetOptions( # arch produces a complete, self-contained xcat-core repo. $opts{packages} = \@cli_packages if @cli_packages; +# --native-only: build just the arch-native packages (@NATIVE_PACKAGES). Used on a +# secondary-arch builder (ppc64le) so the noarch packages get built only once, on x86_64. +$opts{packages} = [@NATIVE_PACKAGES] if $opts{native_only} && !@cli_packages; + +# --input-core-repos accepts one or more dirs (repeatable, or several after one flag); each is +# a per-arch build's dist//rpms tree that --merge-core-repos assembles into --output-dir. +$opts{input_core_repos} = [@cli_input_core_repos] if @cli_input_core_repos; + +# --target REPLACES the default (like --package), and exactly ONE target is built per +# invocation. The flat xcat-core is EL-agnostic, so a single +epel-10- +# build per arch is canonical; the multi-arch flat core is assembled separately via +# --merge-core-repos. Building several targets in one run is unsupported: Getopt used to +# bind --target directly to the pre-seeded 3-EL default and thus SILENTLY APPEND (so +# `--target X` built 4 targets). Collect into @cli_targets and reject >1 explicitly. +if (@cli_targets) { + usage(verbose => 0, + message => "only one --target may be given (got: @cli_targets); " + . "run buildrpms.pl once per target") + if @cli_targets > 1; + $opts{targets} = [@cli_targets]; +} else { + $opts{targets} = ["$DISTRO+epel-10-$ARCH"]; +} + # Release is derived from SOURCE_DATE_EPOCH (the git commit time), NOT wall-clock, # so identical sources -> identical Version-Release -> bit-reproducible packages # (a hard requirement for the content-addressed/Merkle-DAG CI). Override with @@ -216,6 +252,23 @@ sub sh { $? >> 8; } +# sh_retry: run $cmd, retrying up to $tries times on non-zero exit. Absorbs transient mock/nspawn +# flakes (e.g. the systemd-nspawn ENOMEDIUM cgroup race, dnf mirror hiccups) so one bad attempt does +# not silently drop a package from the core. Returns the last exit code (0 on eventual success). +sub sh_retry { + my ($cmd, $tries) = @_; + $tries ||= 3; + my $rc = 1; + for my $t (1 .. $tries) { + $rc = sh($cmd); + return 0 if $rc == 0; + warn "[buildrpms] build command failed (rc=$rc), attempt $t/$tries" + . ($t < $tries ? " -- retrying after backoff...\n" : " -- giving up.\n"); + sleep(5 * $t) if $t < $tries; # linear backoff + } + return $rc; +} + # sed { s/foo/bar/ } $filepath applies s/foo/bar/ to the file at $filepath sub sed (&$) { my ($block, $path) = @_; @@ -289,12 +342,19 @@ sub createmockconfig { cp "/etc/mock/$target.cfg", $cfgfile; my $contents = read_text($cfgfile); $contents =~ s/config_opts\['root'\]\s+=.*/config_opts['root'] = \"$chroot\"/; - if ($pkg eq "perl-xCAT") { - # perl-generators is required for having perl(xCAT::...) symbols - # exported by the RPM + if ($pkg eq "perl-xCAT" && $target !~ /suse|sles|leap/i) { + # perl-generators exports perl(xCAT::...) provides on RHEL/Fedora; it does not + # exist on openSUSE/SLES (rpm there generates perl provides itself), so injecting + # it into a SUSE chroot aborts chroot setup. Suppress it for SUSE targets. $contents .= "config_opts['chroot_additional_packages'] = 'perl-generators'\n"; } $contents .= "config_opts['environment']['SOURCE_DATE_EPOCH'] = '$SOURCE_DATE_EPOCH'\n"; + # Avoid systemd-nspawn: it INTERMITTENTLY fails chroot setup with + # "Failed to determine whether the unified cgroups hierarchy is used: No medium found" + # (ENOMEDIUM), which drops that package from the (still-signed) core -> an incomplete build that + # only surfaces later as a confusing MN install failure. 'simple' isolation is a plain chroot -- + # reliable for these RPM builds -- and sidesteps the nspawn cgroup race entirely. + $contents .= "config_opts['isolation'] = 'simple'\n"; write_text($cfgfile, $contents); } @@ -423,7 +483,7 @@ sub buildspkgs { say "Building $diskcache"; - sh(<<"EOF"); + sh_retry(<<"EOF"); mock -r $chroot \\ -N \\ @{[ join " ", @opts ]} \\ @@ -446,17 +506,11 @@ sub buildpkgs { my $ext = $opts{mock_uniqueext} ? "-$opts{mock_uniqueext}" : ""; my $chroot = "$pkg-$target$ext"; - my @native_pkgs = qw( - xCAT - xCATsn - xCAT-genesis-scripts - ); - # get x86_64 from alma+epel-9-x86_64 my $targetarch = targetarch_from_target($target); # xCAT genesis packages include the translated target arch in their file names. - my $arch = is_in($pkg, @native_pkgs) ? $targetarch : "noarch"; + my $arch = is_in($pkg, @NATIVE_PACKAGES) ? $targetarch : "noarch"; my $genesis_tarch = genesis_tarch_from_targetarch($targetarch); my $diskcache = ( @@ -480,7 +534,7 @@ sub buildpkgs { say "Building $pkg $diskcache"; - sh(<<"EOF"); + sh_retry(<<"EOF"); mock -r $chroot \\ -N \\ @{[ join " ", @opts ]} \\ @@ -646,8 +700,7 @@ sub index_repo { sub update_repo { my ($target) = @_; - my $repodir = "dist/$target/rpms"; - index_repo($repodir); + index_repo("dist/$target/rpms"); } sub write_release_alias { @@ -719,9 +772,9 @@ sub write_repo_metadata_dir { my ($repodir) = @_; return unless -d $repodir; - # Shipped baseurl points at xcat.org; mklocalrepo.sh rewrites baseurl/gpgkey to - # file:// at deploy time for local use. - my $baseurl = "https://xcat.org/files/xcat/repos/yum/devel/xcat-core"; + # Shipped baseurl points at xcat.org (--repo-baseurl overrides it per family, e.g. the + # sles/apt layout); mklocalrepo.sh rewrites baseurl/gpgkey to file:// at deploy time. + my $baseurl = $opts{repo_baseurl}; my $gpgcheck = $opts{gpg_sign} ? 1 : 0; my $gpgkey_line = $opts{gpg_sign} ? "gpgkey=$baseurl/repodata/repomd.xml.key" @@ -772,24 +825,103 @@ COMMIT_ID_LONG=$GITINFO EOF } -# Turn an already-populated core dir into a signed repo in the upstream xcat.org -# layout, reusing the same index/sign/metadata code as a per-target build. Used to -# assemble the flat MULTI-ARCH core: the caller rsyncs each arch's dist//rpms/ -# (excluding repodata/) into first, then this does the single final -# createrepo_c + repomd signing so no packages are moved by hand. -sub finalize_core { - my $dir = $opts{finalize_core}; - die "FATAL: --finalize-core dir '$dir' does not exist\n" unless -d $dir; - index_repo($dir); +# Assemble the flat MULTI-ARCH core from per-arch build outputs and sign it, in the upstream +# xcat.org layout, reusing the same index/sign/metadata code as a per-target build. Given +# --output-dir OUT and one or more --input-core-repos IN (each a per-arch dist//rpms +# tree), this wipes OUT, rsyncs every IN into it (excluding each arch's own repodata/; noarch +# packages dedup), then does the single final createrepo_c + repomd signing -- no packages are +# moved by hand. This absorbs the wipe+rsync assembly that used to live in the CI caller. +# +# Start CLEAN (wipe OUT first): snap-versioned rpms carry a per-build timestamp in their NVR, so +# merging into a dirty OUT would PILE UP stale versions from prior builds and make the flat core +# unresolvable (e.g. an old noarch against a fresh ppc build). +sub merge_core_repos { + my $out = $opts{output_dir} + or die "FATAL: --merge-core-repos requires --output-dir\n"; + my @ins = @{ $opts{input_core_repos} || [] }; + die "FATAL: --merge-core-repos requires at least one --input-core-repos dir\n" unless @ins; + -d $_ or die "FATAL: --input-core-repos dir '$_' does not exist\n" for @ins; + + sh(qq(rm -rf "$out")) and die "Failed to clean output dir '$out'\n"; + make_path($out); + for my $in (@ins) { + sh(qq(rsync -a --exclude 'repodata/' "$in/" "$out/")) + and die "Failed to rsync '$in' into '$out'\n"; + } + + index_repo($out); if ($opts{gpg_sign}) { $ENV{GNUPGHOME} = $opts{gpg_home} if $opts{gpg_home}; - sign_repo_dir($dir, $opts{gpg_key_name}); + sign_repo_dir($out, $opts{gpg_key_name}); } - write_repo_metadata_dir($dir); - write_release_alias($dir); + write_repo_metadata_dir($out); + write_release_alias($out); return 0; } +# --- graceful mock cancellation -------------------------------------------------- +# A mock build killed mid-flight would leave its chroot bind-mounts (proc/sys/dev and the +# -bootstrap chroot's dnf/yum cache mounts) and orphaned rpmbuild/dnf processes behind, +# breaking the next run. Verified out-of-band: when the *mock* process itself receives +# SIGTERM it runs orphansKill, unmounts every chroot it created, releases its buildroot +# flock, and exits within a couple of seconds -- mock cleans up after itself. Builds run +# as Parallel::ForkManager children (main -> child -> mock -> rpmbuild), and the trap here +# sees SIGINT/SIGTERM before those mock grandchildren, so it just forwards the signal to +# the in-flight mock processes and WAITS for mock to finish that cleanup. Only if a mock +# is wedged do we escalate to SIGKILL and lazy-unmount by hand. (The 0-byte buildroot.lock +# file and the cached chroot dirs left behind are normal mock state, not leaks.) +my %MOCK_INFLIGHT; # ForkManager child pid => mock chroot (-r) name it is building +my $ABORTING = 0; + +# PIDs of running mock processes whose `-r ` matches one of @chroots. +sub mock_pids { + my %want = map { (" -r $_ " => 1) } @_; + my @pids; + for my $proc (glob '/proc/[0-9]*') { + my ($pid) = $proc =~ m{/(\d+)\z} or next; + open my $fh, '<', "$proc/cmdline" or next; + local $/; my $cmd = <$fh>; close $fh; + next unless defined $cmd; + $cmd =~ tr/\0/ /; # NUL-separated argv -> spaces + next unless $cmd =~ m{(?:^|/)mock } && index($cmd, ' -r ') >= 0; + push @pids, $pid if grep { index($cmd, $_) >= 0 } keys %want; + } + return @pids; +} + +sub sweep_mock_mounts { + # Fallback only (after SIGKILL): lazy-unmount every bind still under /var/lib/mock. + open my $f, '<', '/proc/mounts' or return; + my @mp = grep { m{^/var/lib/mock/} } map { (split ' ')[1] } <$f>; + close $f; + system('umount', '-l', $_) for sort { length($b) <=> length($a) } @mp; +} + +sub abort_builds { + my ($sig) = @_; + return if $ABORTING; + $ABORTING = 1; + warn "\n[buildrpms] caught SIG$sig: aborting -- signalling mock to self-clean...\n"; + my @chroots = values %MOCK_INFLIGHT; + kill 'TERM', mock_pids(@chroots); # mock unmounts + orphanKills itself + kill 'TERM', keys %MOCK_INFLIGHT; # unwind the ForkManager builders too + my @mock; + for (1 .. 30) { # wait for mock to finish its own cleanup + @mock = mock_pids(@chroots); + last unless @mock; + select undef, undef, undef, 1; + } + if (@mock) { # wedged mock -> force it, then clean by hand + warn "[buildrpms] mock still running after 30s; SIGKILL + unmount sweep\n"; + kill 'KILL', @mock, keys %MOCK_INFLIGHT; + select undef, undef, undef, 2; + sweep_mock_mounts(); + } + warn "[buildrpms] abort cleanup done\n"; + $SIG{$sig} = 'DEFAULT'; + kill $sig, $$; # re-raise for the correct exit status +} + sub main { usage(verbose => 2, exitval => 0) if $opts{help}; my $mode = repo_mode(); @@ -798,17 +930,52 @@ sub main { return exit(configure_nginx()) if $opts{configure_nginx}; return exit(setup_local_repos()) if $opts{setup_local_repos}; - return exit(finalize_core()) if $opts{finalize_core}; + return exit(merge_core_repos()) if $opts{merge_core_repos}; prepare_xcat_probe_source_tar() if grep { $_ eq "xCAT-probe" } $opts{packages}->@*; + # ---- concurrency guard (mirrors cluster-test.pl's per-cluster lock) -------------------------- + # Every per-package mock chroot/config for this run shares the "-" namespace: + # /etc/mock/.cfg, /var/lib/mock// and its buildroot.lock. Two builds of the SAME + # target(+uniqueext) therefore CLOBBER each other's mock config (SOURCE_DATE_EPOCH -> wrong NVR) + # and race the shared chroot -- and abort_builds' fallback lazy-unmounts EVERY /var/lib/mock bind, + # which would rip out a peer build's live chroot too. So refuse to run a second build of the same + # target(+ext) concurrently. Distinct targets / --mock-uniqueext are independent and never conflict. + # (Held for the process lifetime via a never-closed, intentionally leaked filehandle.) + { + my $key = join('-', $opts{targets}->@*) + . ($opts{mock_uniqueext} ? "-$opts{mock_uniqueext}" : ""); + $key =~ s/[^A-Za-z0-9._-]/-/g; + my $lock = "/var/lock/buildrpms.$key.lock"; + if (open(my $blk, '>', $lock)) { + unless (flock($blk, LOCK_EX | LOCK_NB)) { + die "FATAL: another buildrpms.pl is already building target '@{$opts{targets}}'" + . ($opts{mock_uniqueext} ? " (uniqueext=$opts{mock_uniqueext})" : "") . ".\n" + . " ($lock is held). Concurrent builds of the same target collide on the shared\n" + . " /etc/mock + /var/lib/mock chroot namespace (wrong NVR + a killed peer's\n" + . " cleanup unmounts this build's chroot). Serialize them, or pass a distinct\n" + . " --mock-uniqueext per build.\n"; + } + # intentionally leaked: the lock is released only when this process exits. + } + } + my @rpms = product($opts{packages}, $opts{targets}); my $pm = Parallel::ForkManager->new($opts{nproc}); + # Track which mock chroot each live child is building so abort_builds can scrub it. + local $SIG{INT} = \&abort_builds; + local $SIG{TERM} = \&abort_builds; + $pm->run_on_start(sub { my ($pid, $chroot) = @_; $MOCK_INFLIGHT{$pid} = $chroot if defined $chroot; }); + $pm->run_on_finish(sub { my ($pid) = @_; delete $MOCK_INFLIGHT{$pid}; }); + for my $pair (@rpms) { my ($pkg, $target) = $pair->@*; - $pm->start and next; + my $ext = $opts{mock_uniqueext} ? "-$opts{mock_uniqueext}" : ""; + my $chroot = "$pkg-$target$ext"; # matches buildspkgs/buildpkgs `-r` + $pm->start($chroot) and next; + $SIG{INT} = $SIG{TERM} = 'DEFAULT'; # child: die on signal; the parent cleans up buildall($pkg, $target); @@ -818,7 +985,8 @@ sub main { $pm->wait_all_children; for my $target ($opts{targets}->@*) { - $pm->start and next; + $pm->start and next; # no chroot ident: update_repo runs no mock + $SIG{INT} = $SIG{TERM} = 'DEFAULT'; update_repo($target); @@ -882,13 +1050,24 @@ This option is handled before normal option parsing. =item B<--target>=I -Build for the specified target. Repeatable. Example: -C. +Build for the specified mock target, e.g. C. Exactly ONE target +is built per invocation: passing more than one C<--target> is an error (run the script +once per target). When omitted, the default is a single C<< +epel-10- >> +derived from the host. The multi-arch flat core is assembled from per-arch builds via +C<--merge-core-repos>. =item B<--package>=I Build only selected package(s). Repeatable. +=item B<--native-only> + +Build only the arch-native packages (C, C, C) -- the +ones whose rpms carry the target arch. Everything else in the default set is C and +identical on every arch, so a secondary-arch builder (e.g. ppc64le) uses this to avoid +rebuilding the noarch packages that the x86_64 builder already produces. Ignored if +C<--package> is given. + =item B<--nproc>=I Number of parallel workers used by C. @@ -962,6 +1141,29 @@ If not specified, uses the default GPG keyring. Name of the GPG key to use for signing. Default: C. +=item B<--merge-core-repos> + +Assemble the flat multi-arch C from per-arch build outputs and sign it, then exit. +Requires C<--output-dir> and one or more C<--input-core-repos>. Wipes the output dir, rsyncs +every input into it (excluding each arch's own C; C packages dedup), then +runs a single C plus (with C<--gpg-sign>) C signing. + +=item B<--output-dir>=I + +Destination for C<--merge-core-repos>: the assembled, signed flat multi-arch core. +Wiped and recreated on each run. + +=item B<--input-core-repos>=I... + +Input dirs for C<--merge-core-repos>: one or more per-arch C/rpms> trees to merge +into C<--output-dir>. Repeatable, and also accepts several dirs after a single flag. + +=item B<--repo-baseurl>=I + +Base URL written into the generated C (and its C line). Defaults to the +yum/devel path; override per family, e.g. C +for SUSE. Applies to both per-target builds and C<--merge-core-repos>. + =back =head1 DEFAULT FLOW