From 4e6ab23d09171e3aac1ab92fa57b462ac790f5a1 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:49:44 -0300 Subject: [PATCH 1/5] fix(xcat-core): port CI buildrpms.pl (parallel builds, multi-arch merge) into tree The 3 EL CD pipelines (xcat-core-devel-cd, xcat-core-stable-cd, xcat-dep-el-cd) overlaid a pinned $CI/buildrpms.pl at build time because the tree's buildrpms.pl lacked the options they depend on: - a per-target flock guard alongside --mock-uniqueext, so concurrent same-target builds do not corrupt each other's /var/lib/mock chroot namespace; - --native-only (build only arch-native pkgs on the secondary arch) plus --merge-core-repos/--output-dir/--input-core-repos, replacing --finalize-core, to assemble one signed flat multi-arch core from per-arch build outputs; - sh_retry() to absorb transient mock/nspawn flakes; - a single --target guard and graceful mock cancellation (sweep_mock_mounts/abort_builds) that unmounts chroots on abort. Porting them in-tree lets CI drop the $CI/buildrpms.pl pin and run the three pipelines in parallel without the cross-job serialize lock. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- buildrpms.pl | 271 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 237 insertions(+), 34 deletions(-) diff --git a/buildrpms.pl b/buildrpms.pl index b4461ad4a..b3909ed7a 100755 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -42,6 +42,7 @@ use File::Path qw(make_path remove_tree); use File::Slurper qw(read_text write_text); use File::Temp qw(tempdir tempfile); use FindBin qw($Bin); +use Fcntl qw(:flock); # per-target build lock (concurrency guard; see main()) use Getopt::Long qw(GetOptions); use POSIX qw(strftime); use Parallel::ForkManager; @@ -136,6 +137,12 @@ my @PACKAGES = qw( xCAT-vlan ); +# The arch-native packages: their rpms carry the target arch. Everything else in @PACKAGES +# is noarch and byte-identical on every arch, so `--native-only` builds just these -- a +# secondary-arch build (e.g. ppc64le) then produces only what the x86_64 build cannot already +# provide, and the multi-arch merge has no duplicate noarch to reconcile. +my @NATIVE_PACKAGES = qw(xCAT xCATsn xCAT-genesis-scripts); + my @TARGETS = ( "$DISTRO+epel-8-$ARCH", "$DISTRO+epel-9-$ARCH", @@ -156,12 +163,13 @@ my %opts = ( packages => \@PACKAGES, release => "", repo_mode => "file", + repo_baseurl => "https://xcat.org/files/xcat/repos/yum/devel/xcat-core", targets => \@TARGETS, verbose => 0, xcat_dep_path => "$PWD/../xcat-dep/", ); -my @cli_packages; +my (@cli_packages, @cli_targets, @cli_input_core_repos); GetOptions( "configure_nginx" => \$opts{configure_nginx}, "force" => \$opts{force}, @@ -173,13 +181,17 @@ GetOptions( "nginx_port" => \$opts{nginx_port}, "nproc=i" => \$opts{nproc}, "package=s@" => \@cli_packages, + "native-only" => \$opts{native_only}, "release=s" => \$opts{release}, "repo-mode=s" => \$opts{repo_mode}, - "target=s@" => \$opts{targets}, + "target=s@" => \@cli_targets, "verbose" => \$opts{verbose}, "xcat_dep_path=s" => \$opts{xcat_dep_path}, "setup_local_repos" => \$opts{setup_local_repos}, - "finalize-core=s" => \$opts{finalize_core}, + "merge-core-repos" => \$opts{merge_core_repos}, + "output-dir=s" => \$opts{output_dir}, + "input-core-repos=s{1,}" => \@cli_input_core_repos, + "repo-baseurl=s" => \$opts{repo_baseurl}, ) or usage(); # --package REPLACES the default set (build exactly what was asked), so @@ -188,6 +200,30 @@ GetOptions( # arch produces a complete, self-contained xcat-core repo. $opts{packages} = \@cli_packages if @cli_packages; +# --native-only: build just the arch-native packages (@NATIVE_PACKAGES). Used on a +# secondary-arch builder (ppc64le) so the noarch packages get built only once, on x86_64. +$opts{packages} = [@NATIVE_PACKAGES] if $opts{native_only} && !@cli_packages; + +# --input-core-repos accepts one or more dirs (repeatable, or several after one flag); each is +# a per-arch build's dist//rpms tree that --merge-core-repos assembles into --output-dir. +$opts{input_core_repos} = [@cli_input_core_repos] if @cli_input_core_repos; + +# --target REPLACES the default (like --package), and exactly ONE target is built per +# invocation. The flat xcat-core is EL-agnostic, so a single +epel-10- +# build per arch is canonical; the multi-arch flat core is assembled separately via +# --merge-core-repos. Building several targets in one run is unsupported: Getopt used to +# bind --target directly to the pre-seeded 3-EL default and thus SILENTLY APPEND (so +# `--target X` built 4 targets). Collect into @cli_targets and reject >1 explicitly. +if (@cli_targets) { + usage(verbose => 0, + message => "only one --target may be given (got: @cli_targets); " + . "run buildrpms.pl once per target") + if @cli_targets > 1; + $opts{targets} = [@cli_targets]; +} else { + $opts{targets} = ["$DISTRO+epel-10-$ARCH"]; +} + # Release is derived from SOURCE_DATE_EPOCH (the git commit time), NOT wall-clock, # so identical sources -> identical Version-Release -> bit-reproducible packages # (a hard requirement for the content-addressed/Merkle-DAG CI). Override with @@ -215,6 +251,23 @@ sub sh { $? >> 8; } +# sh_retry: run $cmd, retrying up to $tries times on non-zero exit. Absorbs transient mock/nspawn +# flakes (e.g. the systemd-nspawn ENOMEDIUM cgroup race, dnf mirror hiccups) so one bad attempt does +# not silently drop a package from the core. Returns the last exit code (0 on eventual success). +sub sh_retry { + my ($cmd, $tries) = @_; + $tries ||= 3; + my $rc = 1; + for my $t (1 .. $tries) { + $rc = sh($cmd); + return 0 if $rc == 0; + warn "[buildrpms] build command failed (rc=$rc), attempt $t/$tries" + . ($t < $tries ? " -- retrying after backoff...\n" : " -- giving up.\n"); + sleep(5 * $t) if $t < $tries; # linear backoff + } + return $rc; +} + # sed { s/foo/bar/ } $filepath applies s/foo/bar/ to the file at $filepath sub sed (&$) { my ($block, $path) = @_; @@ -288,12 +341,19 @@ sub createmockconfig { cp "/etc/mock/$target.cfg", $cfgfile; my $contents = read_text($cfgfile); $contents =~ s/config_opts\['root'\]\s+=.*/config_opts['root'] = \"$chroot\"/; - if ($pkg eq "perl-xCAT") { - # perl-generators is required for having perl(xCAT::...) symbols - # exported by the RPM + if ($pkg eq "perl-xCAT" && $target !~ /suse|sles|leap/i) { + # perl-generators exports perl(xCAT::...) provides on RHEL/Fedora; it does not + # exist on openSUSE/SLES (rpm there generates perl provides itself), so injecting + # it into a SUSE chroot aborts chroot setup. Suppress it for SUSE targets. $contents .= "config_opts['chroot_additional_packages'] = 'perl-generators'\n"; } $contents .= "config_opts['environment']['SOURCE_DATE_EPOCH'] = '$SOURCE_DATE_EPOCH'\n"; + # Avoid systemd-nspawn: it INTERMITTENTLY fails chroot setup with + # "Failed to determine whether the unified cgroups hierarchy is used: No medium found" + # (ENOMEDIUM), which drops that package from the (still-signed) core -> an incomplete build that + # only surfaces later as a confusing MN install failure. 'simple' isolation is a plain chroot -- + # reliable for these RPM builds -- and sidesteps the nspawn cgroup race entirely. + $contents .= "config_opts['isolation'] = 'simple'\n"; write_text($cfgfile, $contents); } @@ -422,7 +482,7 @@ sub buildspkgs { say "Building $diskcache"; - sh(<<"EOF"); + sh_retry(<<"EOF"); mock -r $chroot \\ -N \\ @{[ join " ", @opts ]} \\ @@ -445,17 +505,11 @@ sub buildpkgs { my $ext = $opts{mock_uniqueext} ? "-$opts{mock_uniqueext}" : ""; my $chroot = "$pkg-$target$ext"; - my @native_pkgs = qw( - xCAT - xCATsn - xCAT-genesis-scripts - ); - # get x86_64 from alma+epel-9-x86_64 my $targetarch = targetarch_from_target($target); # xCAT genesis packages include the translated target arch in their file names. - my $arch = is_in($pkg, @native_pkgs) ? $targetarch : "noarch"; + my $arch = is_in($pkg, @NATIVE_PACKAGES) ? $targetarch : "noarch"; my $genesis_tarch = genesis_tarch_from_targetarch($targetarch); my $diskcache = ( @@ -479,7 +533,7 @@ sub buildpkgs { say "Building $pkg $diskcache"; - sh(<<"EOF"); + sh_retry(<<"EOF"); mock -r $chroot \\ -N \\ @{[ join " ", @opts ]} \\ @@ -697,9 +751,9 @@ sub write_repo_metadata_dir { my ($repodir) = @_; return unless -d $repodir; - # Shipped baseurl points at xcat.org; mklocalrepo.sh rewrites baseurl/gpgkey to - # file:// at deploy time for local use. - my $baseurl = "https://xcat.org/files/xcat/repos/yum/devel/xcat-core"; + # Shipped baseurl points at xcat.org (--repo-baseurl overrides it per family, e.g. the + # sles/apt layout); mklocalrepo.sh rewrites baseurl/gpgkey to file:// at deploy time. + my $baseurl = $opts{repo_baseurl}; my $gpgcheck = $opts{gpg_sign} ? 1 : 0; my $gpgkey_line = $opts{gpg_sign} ? "gpgkey=$baseurl/repodata/repomd.xml.key" @@ -750,23 +804,102 @@ COMMIT_ID_LONG=$GITINFO EOF } -# Turn an already-populated core dir into a signed repo in the upstream xcat.org -# layout, reusing the same index/sign/metadata code as a per-target build. Used to -# assemble the flat MULTI-ARCH core: the caller rsyncs each arch's dist//rpms/ -# (excluding repodata/) into first, then this does the single final -# createrepo_c + repomd signing so no packages are moved by hand. -sub finalize_core { - my $dir = $opts{finalize_core}; - die "FATAL: --finalize-core dir '$dir' does not exist\n" unless -d $dir; - index_repo($dir); +# Assemble the flat MULTI-ARCH core from per-arch build outputs and sign it, in the upstream +# xcat.org layout, reusing the same index/sign/metadata code as a per-target build. Given +# --output-dir OUT and one or more --input-core-repos IN (each a per-arch dist//rpms +# tree), this wipes OUT, rsyncs every IN into it (excluding each arch's own repodata/; noarch +# packages dedup), then does the single final createrepo_c + repomd signing -- no packages are +# moved by hand. This absorbs the wipe+rsync assembly that used to live in the CI caller. +# +# Start CLEAN (wipe OUT first): snap-versioned rpms carry a per-build timestamp in their NVR, so +# merging into a dirty OUT would PILE UP stale versions from prior builds and make the flat core +# unresolvable (e.g. an old noarch against a fresh ppc build). +sub merge_core_repos { + my $out = $opts{output_dir} + or die "FATAL: --merge-core-repos requires --output-dir\n"; + my @ins = @{ $opts{input_core_repos} || [] }; + die "FATAL: --merge-core-repos requires at least one --input-core-repos dir\n" unless @ins; + -d $_ or die "FATAL: --input-core-repos dir '$_' does not exist\n" for @ins; + + sh(qq(rm -rf "$out")) and die "Failed to clean output dir '$out'\n"; + make_path($out); + for my $in (@ins) { + sh(qq(rsync -a --exclude 'repodata/' "$in/" "$out/")) + and die "Failed to rsync '$in' into '$out'\n"; + } + + index_repo($out); if ($opts{gpg_sign}) { $ENV{GNUPGHOME} = $opts{gpg_home} if $opts{gpg_home}; - sign_repo_dir($dir, $opts{gpg_key_name}); + sign_repo_dir($out, $opts{gpg_key_name}); } - write_repo_metadata_dir($dir); + write_repo_metadata_dir($out); return 0; } +# --- graceful mock cancellation -------------------------------------------------- +# A mock build killed mid-flight would leave its chroot bind-mounts (proc/sys/dev and the +# -bootstrap chroot's dnf/yum cache mounts) and orphaned rpmbuild/dnf processes behind, +# breaking the next run. Verified out-of-band: when the *mock* process itself receives +# SIGTERM it runs orphansKill, unmounts every chroot it created, releases its buildroot +# flock, and exits within a couple of seconds -- mock cleans up after itself. Builds run +# as Parallel::ForkManager children (main -> child -> mock -> rpmbuild), and the trap here +# sees SIGINT/SIGTERM before those mock grandchildren, so it just forwards the signal to +# the in-flight mock processes and WAITS for mock to finish that cleanup. Only if a mock +# is wedged do we escalate to SIGKILL and lazy-unmount by hand. (The 0-byte buildroot.lock +# file and the cached chroot dirs left behind are normal mock state, not leaks.) +my %MOCK_INFLIGHT; # ForkManager child pid => mock chroot (-r) name it is building +my $ABORTING = 0; + +# PIDs of running mock processes whose `-r ` matches one of @chroots. +sub mock_pids { + my %want = map { (" -r $_ " => 1) } @_; + my @pids; + for my $proc (glob '/proc/[0-9]*') { + my ($pid) = $proc =~ m{/(\d+)\z} or next; + open my $fh, '<', "$proc/cmdline" or next; + local $/; my $cmd = <$fh>; close $fh; + next unless defined $cmd; + $cmd =~ tr/\0/ /; # NUL-separated argv -> spaces + next unless $cmd =~ m{(?:^|/)mock } && index($cmd, ' -r ') >= 0; + push @pids, $pid if grep { index($cmd, $_) >= 0 } keys %want; + } + return @pids; +} + +sub sweep_mock_mounts { + # Fallback only (after SIGKILL): lazy-unmount every bind still under /var/lib/mock. + open my $f, '<', '/proc/mounts' or return; + my @mp = grep { m{^/var/lib/mock/} } map { (split ' ')[1] } <$f>; + close $f; + system('umount', '-l', $_) for sort { length($b) <=> length($a) } @mp; +} + +sub abort_builds { + my ($sig) = @_; + return if $ABORTING; + $ABORTING = 1; + warn "\n[buildrpms] caught SIG$sig: aborting -- signalling mock to self-clean...\n"; + my @chroots = values %MOCK_INFLIGHT; + kill 'TERM', mock_pids(@chroots); # mock unmounts + orphanKills itself + kill 'TERM', keys %MOCK_INFLIGHT; # unwind the ForkManager builders too + my @mock; + for (1 .. 30) { # wait for mock to finish its own cleanup + @mock = mock_pids(@chroots); + last unless @mock; + select undef, undef, undef, 1; + } + if (@mock) { # wedged mock -> force it, then clean by hand + warn "[buildrpms] mock still running after 30s; SIGKILL + unmount sweep\n"; + kill 'KILL', @mock, keys %MOCK_INFLIGHT; + select undef, undef, undef, 2; + sweep_mock_mounts(); + } + warn "[buildrpms] abort cleanup done\n"; + $SIG{$sig} = 'DEFAULT'; + kill $sig, $$; # re-raise for the correct exit status +} + sub main { usage(verbose => 2, exitval => 0) if $opts{help}; my $mode = repo_mode(); @@ -775,17 +908,52 @@ sub main { return exit(configure_nginx()) if $opts{configure_nginx}; return exit(setup_local_repos()) if $opts{setup_local_repos}; - return exit(finalize_core()) if $opts{finalize_core}; + return exit(merge_core_repos()) if $opts{merge_core_repos}; prepare_xcat_probe_source_tar() if grep { $_ eq "xCAT-probe" } $opts{packages}->@*; + # ---- concurrency guard (mirrors cluster-test.pl's per-cluster lock) -------------------------- + # Every per-package mock chroot/config for this run shares the "-" namespace: + # /etc/mock/.cfg, /var/lib/mock// and its buildroot.lock. Two builds of the SAME + # target(+uniqueext) therefore CLOBBER each other's mock config (SOURCE_DATE_EPOCH -> wrong NVR) + # and race the shared chroot -- and abort_builds' fallback lazy-unmounts EVERY /var/lib/mock bind, + # which would rip out a peer build's live chroot too. So refuse to run a second build of the same + # target(+ext) concurrently. Distinct targets / --mock-uniqueext are independent and never conflict. + # (Held for the process lifetime via a never-closed, intentionally leaked filehandle.) + { + my $key = join('-', $opts{targets}->@*) + . ($opts{mock_uniqueext} ? "-$opts{mock_uniqueext}" : ""); + $key =~ s/[^A-Za-z0-9._-]/-/g; + my $lock = "/var/lock/buildrpms.$key.lock"; + if (open(my $blk, '>', $lock)) { + unless (flock($blk, LOCK_EX | LOCK_NB)) { + die "FATAL: another buildrpms.pl is already building target '@{$opts{targets}}'" + . ($opts{mock_uniqueext} ? " (uniqueext=$opts{mock_uniqueext})" : "") . ".\n" + . " ($lock is held). Concurrent builds of the same target collide on the shared\n" + . " /etc/mock + /var/lib/mock chroot namespace (wrong NVR + a killed peer's\n" + . " cleanup unmounts this build's chroot). Serialize them, or pass a distinct\n" + . " --mock-uniqueext per build.\n"; + } + # intentionally leaked: the lock is released only when this process exits. + } + } + my @rpms = product($opts{packages}, $opts{targets}); my $pm = Parallel::ForkManager->new($opts{nproc}); + # Track which mock chroot each live child is building so abort_builds can scrub it. + local $SIG{INT} = \&abort_builds; + local $SIG{TERM} = \&abort_builds; + $pm->run_on_start(sub { my ($pid, $chroot) = @_; $MOCK_INFLIGHT{$pid} = $chroot if defined $chroot; }); + $pm->run_on_finish(sub { my ($pid) = @_; delete $MOCK_INFLIGHT{$pid}; }); + for my $pair (@rpms) { my ($pkg, $target) = $pair->@*; - $pm->start and next; + my $ext = $opts{mock_uniqueext} ? "-$opts{mock_uniqueext}" : ""; + my $chroot = "$pkg-$target$ext"; # matches buildspkgs/buildpkgs `-r` + $pm->start($chroot) and next; + $SIG{INT} = $SIG{TERM} = 'DEFAULT'; # child: die on signal; the parent cleans up buildall($pkg, $target); @@ -795,7 +963,8 @@ sub main { $pm->wait_all_children; for my $target ($opts{targets}->@*) { - $pm->start and next; + $pm->start and next; # no chroot ident: update_repo runs no mock + $SIG{INT} = $SIG{TERM} = 'DEFAULT'; update_repo($target); @@ -853,13 +1022,24 @@ This option is handled before normal option parsing. =item B<--target>=I -Build for the specified target. Repeatable. Example: -C. +Build for the specified mock target, e.g. C. Exactly ONE target +is built per invocation: passing more than one C<--target> is an error (run the script +once per target). When omitted, the default is a single C<< +epel-10- >> +derived from the host. The multi-arch flat core is assembled from per-arch builds via +C<--merge-core-repos>. =item B<--package>=I Build only selected package(s). Repeatable. +=item B<--native-only> + +Build only the arch-native packages (C, C, C) -- the +ones whose rpms carry the target arch. Everything else in the default set is C and +identical on every arch, so a secondary-arch builder (e.g. ppc64le) uses this to avoid +rebuilding the noarch packages that the x86_64 builder already produces. Ignored if +C<--package> is given. + =item B<--nproc>=I Number of parallel workers used by C. @@ -933,6 +1113,29 @@ If not specified, uses the default GPG keyring. Name of the GPG key to use for signing. Default: C. +=item B<--merge-core-repos> + +Assemble the flat multi-arch C from per-arch build outputs and sign it, then exit. +Requires C<--output-dir> and one or more C<--input-core-repos>. Wipes the output dir, rsyncs +every input into it (excluding each arch's own C; C packages dedup), then +runs a single C plus (with C<--gpg-sign>) C signing. + +=item B<--output-dir>=I + +Destination for C<--merge-core-repos>: the assembled, signed flat multi-arch core. +Wiped and recreated on each run. + +=item B<--input-core-repos>=I... + +Input dirs for C<--merge-core-repos>: one or more per-arch C/rpms> trees to merge +into C<--output-dir>. Repeatable, and also accepts several dirs after a single flag. + +=item B<--repo-baseurl>=I + +Base URL written into the generated C (and its C line). Defaults to the +yum/devel path; override per family, e.g. C +for SUSE. Applies to both per-target builds and C<--merge-core-repos>. + =back =head1 DEFAULT FLOW From 2e3eef2867d6de5697b8bd5a13135610e1a32f4d Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:51:26 -0300 Subject: [PATCH 2/5] fix(xcat-core): clean mock buildroot before each build (--init), not reuse-dirty buildrpms.pl built every package with mock -N (--no-clean), reusing the per-package - chroot across runs for flat disk. But a build aborted or killed mid-flight leaves that chroot half-initialised with a corrupt rpmdb; the NEXT run reused it and failed (cannot open Packages database .../usr/lib/sysimage/rpm), producing an incomplete core (e.g. missing xCAT-test) that fails the deploy-time completeness gate. Re-init the buildroot (mock --init) right before building each package, after the diskcache skip so it only runs when actually building. --init restores from mock's root-cache tarball (cheap) so disk stays flat and builds stay fast; -N is kept on the srpm/binary calls so they still reuse the freshly initialised root within the run. This stabilises builds against corrupt state left by any previous failed/aborted build. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- buildrpms.pl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/buildrpms.pl b/buildrpms.pl index b3909ed7a..264a4adba 100755 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -482,6 +482,16 @@ sub buildspkgs { say "Building $diskcache"; + # Clean-before-start: re-init the buildroot from mock's root cache so a corrupt or + # half-built chroot left behind by a PREVIOUS aborted/failed run cannot poison this + # build (this is what caused "cannot open Packages database ... /usr/lib/sysimage/rpm" + # -> missing rpm -> incomplete core). Cheap: --init restores from the cached root + # tarball rather than a full dnf bootstrap, and the -N below then reuses THIS freshly + # initialised root for both the srpm and the binary rebuild within this run. We reach + # here only when actually building (past the diskcache skip), so flat-disk reuse across + # runs is preserved -- we just guarantee a known-good starting point each time. + sh_retry(qq(mock -r $chroot @{[ join " ", @opts ]} --init)); + sh_retry(<<"EOF"); mock -r $chroot \\ -N \\ From 27ec50528fbd9c380e7ca3c6e5aa86f9d4a9ccc1 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:04:04 -0300 Subject: [PATCH 3/5] fix(xcat-core): default DNS TSIG to hmac-sha256 on new EL9+/Ubuntu installs (backport #7597) xCAT 2.18 defaults the DNS/DHCP OMAPI TSIG key to hmac-md5. On EL9/EL10 the newer bind/Net::DNS reject md5-signed dynamic updates (TSIG BADSIG), so makedns fails (FORMERR) and node DNS setup / install cases fail on a DEFAULT install, needing a manual 'chdef -t site dhcpomapialgorithm=hmac-sha256' workaround. Backport xcat2/xcat-core#7597: xcatconfig initDB detects a fresh install and, via OmapiPolicy::new_install_default_algorithm, seeds site.dhcpomapialgorithm=hmac-sha256 for EL9+/Ubuntu 20.04+ (EL8/older keep hmac-md5, which works there). Existing sites are untouched. Default installs on EL9/EL10 now work with no intervention. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- perl-xCAT/xCAT/DHCP/OmapiPolicy.pm | 18 ++++++++++++++++++ xCAT-server/sbin/xcatconfig | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/perl-xCAT/xCAT/DHCP/OmapiPolicy.pm b/perl-xCAT/xCAT/DHCP/OmapiPolicy.pm index 36b2b6d77..a3621a3ce 100644 --- a/perl-xCAT/xCAT/DHCP/OmapiPolicy.pm +++ b/perl-xCAT/xCAT/DHCP/OmapiPolicy.pm @@ -64,6 +64,24 @@ sub normalize_algorithm { return; } +sub new_install_default_algorithm { + my ( $class, %args ) = @_; + + my $platform = $args{platform}; + my $os = $args{os}; + + return unless $args{is_new_install}; + return 'hmac-sha256' + if defined($platform) && $platform =~ /^el(\d+)\b/i && $1 >= 9; + if ( defined($os) && $os =~ /^ubuntu,(\d+\.\d+(?:\.\d+)*)\b/i ) { + my $ubuntu_version = $1; + require xCAT::Utils; + return 'hmac-sha256' + if xCAT::Utils->version_cmp( $ubuntu_version, '20.04' ) >= 0; + } + return; +} + sub normalize_key_name { my ( $class, $key_name ) = @_; diff --git a/xCAT-server/sbin/xcatconfig b/xCAT-server/sbin/xcatconfig index 7eec28355..829116d93 100755 --- a/xCAT-server/sbin/xcatconfig +++ b/xCAT-server/sbin/xcatconfig @@ -25,6 +25,7 @@ use strict; use xCAT::Utils; use xCAT::SvrUtils; use xCAT::DHCP::Backend; +use xCAT::DHCP::OmapiPolicy; use xCAT::TLSPolicy qw(tls_setting_warnings); use xCAT::NetworkUtils; use Getopt::Long; @@ -134,6 +135,13 @@ else $::osname = 'Linux'; } +# Record whether this invocation is creating a new site before database setup +# runs. Reinitializing an existing site must preserve its key algorithm. +my $initializing_new_site = + $::INITIALINSTALL + && !-r "/etc/xcat/site.sqlite" + && !-r "/etc/xcat/cfgloc"; + # if on rhel6, check to see if perl-IO-Compress-Zlib* is installed if (($::INITIALINSTALL) || ($::UPDATEINSTALL)) { @@ -1253,6 +1261,16 @@ sub initDB $chtabcmds .= "$::XCATROOT/sbin/chtab key=vsftp site.value=n;"; $chtabcmds .= "$::XCATROOT/sbin/chtab key=cleanupxcatpost site.value=no;"; $chtabcmds .= "$::XCATROOT/sbin/chtab key=cleanupdiskfullxcatpost site.value=no;"; + my $omapi_algorithm = + xCAT::DHCP::OmapiPolicy->new_install_default_algorithm( + is_new_install => $initializing_new_site, + platform => xCAT::Utils->osver("platform"), + os => xCAT::Utils->osver("all"), + ); + if ($omapi_algorithm) { + $chtabcmds .= + "$::XCATROOT/sbin/chtab key=dhcpomapialgorithm site.value=$omapi_algorithm;"; + } $chtabcmds .= "$::XCATROOT/sbin/chtab key=dhcplease site.value=43200;"; $chtabcmds .= "$::XCATROOT/sbin/chtab key=auditnosyslog site.value=0;"; $chtabcmds .= "$::XCATROOT/sbin/chtab key=xcatsslversion site.value=;"; From dfa22e3650cdebe3c66de824429f5618f9459b8c Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:35:01 -0300 Subject: [PATCH 4/5] fix(xcat-core): make parallel buildrpms fail-closed and lock/cleanup-safe Addresses three concurrency-safety defects raised in review of the parallel buildrpms.pl work: 1. Per-target build lock was released immediately. The flock filehandle was a lexical (my $blk) scoped to the guard block, so it was destroyed -- and the lock dropped -- as soon as that block exited, before any worker forked. The "intentionally leaked" comment did not match the code. Hold the handle in a file-scoped $BUILD_LOCK_FH so the fd (and the lock) live for the whole process; forked children inherit the fd but their exits never release it. 2. Build failures were silently swallowed. buildspkgs()/buildpkgs() called sh_retry() in void context, so a mock build that failed all retries returned non-zero into the void; the child then exited 0 and the parent's run_on_finish ignored the exit code. The parent could therefore index and GPG-sign a repo that was missing packages and still exit 0. Now sh_retry failures die in the child, run_on_finish records any non-zero child, and the run aborts before update_repo and again before signing if any child failed -- never publishing a partial core. 3. Abort cleanup unmounted unrelated builds. sweep_mock_mounts() lazy-unmounted every bind under /var/lib/mock, which on a shared host tears out the live chroots of concurrent, unrelated builds. Scope it to this run's own chroots (each chroot dir plus its -bootstrap sibling), passed in from abort_builds. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- buildrpms.pl | 62 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/buildrpms.pl b/buildrpms.pl index 264a4adba..d8ec2acaf 100755 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -490,9 +490,10 @@ sub buildspkgs { # initialised root for both the srpm and the binary rebuild within this run. We reach # here only when actually building (past the diskcache skip), so flat-disk reuse across # runs is preserved -- we just guarantee a known-good starting point each time. - sh_retry(qq(mock -r $chroot @{[ join " ", @opts ]} --init)); + sh_retry(qq(mock -r $chroot @{[ join " ", @opts ]} --init)) == 0 + or die "FATAL: mock --init failed for $chroot ($pkg/$target)\n"; - sh_retry(<<"EOF"); + sh_retry(<<"EOF") == 0 or die "FATAL: srpm build failed for $pkg ($target)\n"; mock -r $chroot \\ -N \\ @{[ join " ", @opts ]} \\ @@ -543,7 +544,7 @@ sub buildpkgs { say "Building $pkg $diskcache"; - sh_retry(<<"EOF"); + sh_retry(<<"EOF") == 0 or die "FATAL: rpm rebuild failed for $pkg ($target)\n"; mock -r $chroot \\ -N \\ @{[ join " ", @opts ]} \\ @@ -860,6 +861,10 @@ sub merge_core_repos { # file and the cached chroot dirs left behind are normal mock state, not leaks.) my %MOCK_INFLIGHT; # ForkManager child pid => mock chroot (-r) name it is building my $ABORTING = 0; +my $BUILD_LOCK_FH; # per-target build flock; MUST stay file-scoped so the fd (and thus the + # lock) lives for the whole process. A lexical inside main()'s block would + # be DESTROYED at block exit -> lock released before any worker forks. +my @CHILD_FAILURES; # idents (chroot names) of ForkManager children that exited non-zero # PIDs of running mock processes whose `-r ` matches one of @chroots. sub mock_pids { @@ -878,9 +883,18 @@ sub mock_pids { } sub sweep_mock_mounts { - # Fallback only (after SIGKILL): lazy-unmount every bind still under /var/lib/mock. + # Fallback only (after SIGKILL): lazy-unmount binds left under THIS build's own chroots -- + # each chroot's dir plus its "-bootstrap" sibling. Scoped to @chroots so an unrelated, + # concurrent build's mounts under other /var/lib/mock chroots are never torn out. + my (@chroots) = @_; + return unless @chroots; + my %want = map { $_ => 1 } @chroots; open my $f, '<', '/proc/mounts' or return; - my @mp = grep { m{^/var/lib/mock/} } map { (split ' ')[1] } <$f>; + my @mp = grep { + my ($comp) = m{^/var/lib/mock/([^/]+)/}; + if (defined $comp) { (my $base = $comp) =~ s/-bootstrap$//; $want{$comp} || $want{$base} } + else { 0 } + } map { (split ' ')[1] } <$f>; close $f; system('umount', '-l', $_) for sort { length($b) <=> length($a) } @mp; } @@ -903,7 +917,7 @@ sub abort_builds { warn "[buildrpms] mock still running after 30s; SIGKILL + unmount sweep\n"; kill 'KILL', @mock, keys %MOCK_INFLIGHT; select undef, undef, undef, 2; - sweep_mock_mounts(); + sweep_mock_mounts(@chroots); } warn "[buildrpms] abort cleanup done\n"; $SIG{$sig} = 'DEFAULT'; @@ -927,17 +941,17 @@ sub main { # Every per-package mock chroot/config for this run shares the "-" namespace: # /etc/mock/.cfg, /var/lib/mock// and its buildroot.lock. Two builds of the SAME # target(+uniqueext) therefore CLOBBER each other's mock config (SOURCE_DATE_EPOCH -> wrong NVR) - # and race the shared chroot -- and abort_builds' fallback lazy-unmounts EVERY /var/lib/mock bind, - # which would rip out a peer build's live chroot too. So refuse to run a second build of the same - # target(+ext) concurrently. Distinct targets / --mock-uniqueext are independent and never conflict. - # (Held for the process lifetime via a never-closed, intentionally leaked filehandle.) + # and race the shared chroot -- and abort_builds' fallback lazy-unmounts this build's own chroot + # binds, which for the SAME target(+ext) are the very dirs a peer uses. So refuse to run a second + # build of the same target(+ext) concurrently. Distinct targets / --mock-uniqueext are independent + # and never conflict. (Held for the process lifetime via $BUILD_LOCK_FH, a file-scoped handle.) { my $key = join('-', $opts{targets}->@*) . ($opts{mock_uniqueext} ? "-$opts{mock_uniqueext}" : ""); $key =~ s/[^A-Za-z0-9._-]/-/g; my $lock = "/var/lock/buildrpms.$key.lock"; - if (open(my $blk, '>', $lock)) { - unless (flock($blk, LOCK_EX | LOCK_NB)) { + if (open($BUILD_LOCK_FH, '>', $lock)) { + unless (flock($BUILD_LOCK_FH, LOCK_EX | LOCK_NB)) { die "FATAL: another buildrpms.pl is already building target '@{$opts{targets}}'" . ($opts{mock_uniqueext} ? " (uniqueext=$opts{mock_uniqueext})" : "") . ".\n" . " ($lock is held). Concurrent builds of the same target collide on the shared\n" @@ -945,7 +959,9 @@ sub main { . " cleanup unmounts this build's chroot). Serialize them, or pass a distinct\n" . " --mock-uniqueext per build.\n"; } - # intentionally leaked: the lock is released only when this process exits. + # $BUILD_LOCK_FH is file-scoped, so the fd stays open (lock held) until this process + # exits. Child forks inherit the fd but their exits never release it (the parent's + # still-open fd keeps the lock), which is exactly what we want. } } @@ -956,7 +972,13 @@ sub main { local $SIG{INT} = \&abort_builds; local $SIG{TERM} = \&abort_builds; $pm->run_on_start(sub { my ($pid, $chroot) = @_; $MOCK_INFLIGHT{$pid} = $chroot if defined $chroot; }); - $pm->run_on_finish(sub { my ($pid) = @_; delete $MOCK_INFLIGHT{$pid}; }); + $pm->run_on_finish(sub { + my ($pid, $exit_code, $ident) = @_; + delete $MOCK_INFLIGHT{$pid}; + # A build/update child that die()s (e.g. a failed sh_retry) exits non-zero; record it so + # we never index or sign a partial repository (would ship a core missing packages). + push @CHILD_FAILURES, ($ident // "pid=$pid") if $exit_code; + }); for my $pair (@rpms) { my ($pkg, $target) = $pair->@*; @@ -972,6 +994,12 @@ sub main { $pm->wait_all_children; + # Gate: if any package build failed, stop here -- do NOT update_repo/sign a partial core. + if (@CHILD_FAILURES) { + die "FATAL: build failed for: @CHILD_FAILURES\n" + . " refusing to index/sign a partial repository.\n"; + } + for my $target ($opts{targets}->@*) { $pm->start and next; # no chroot ident: update_repo runs no mock $SIG{INT} = $SIG{TERM} = 'DEFAULT'; @@ -982,6 +1010,12 @@ sub main { } $pm->wait_all_children; + # Gate: a failed repo index (update_repo die) must not be signed as if complete. + if (@CHILD_FAILURES) { + die "FATAL: repo update failed for: @CHILD_FAILURES\n" + . " refusing to sign an incomplete repository.\n"; + } + if ($opts{gpg_sign}) { $ENV{GNUPGHOME} = $opts{gpg_home} if $opts{gpg_home}; for my $target ($opts{targets}->@*) { From 3152ae12a9d986aa9bc41427def7102572546321 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 30 Jul 2026 19:02:54 -0300 Subject: [PATCH 5/5] fix(xcat-core): treat signal-killed build workers as failures too The parallel-build failure gate only inspected $exit_code in run_on_finish. A ForkManager child killed by a signal -- SIGKILL, or the OOM-killer under the concurrent build load -- is reaped with $exit_code == 0 but $exit_signal != 0 (and possibly $core_dump). Such a worker therefore was NOT recorded as a failure, so the parent could still index and GPG-sign a repository that is missing the package that worker was building -- the exact partial-repo hazard the gate was added to prevent, via a path it did not cover. Capture $exit_signal and $core_dump from the run_on_finish callback and fail the build when any of $exit_code, $exit_signal, or $core_dump is set. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- buildrpms.pl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/buildrpms.pl b/buildrpms.pl index d8ec2acaf..645add289 100755 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -973,11 +973,13 @@ sub main { local $SIG{TERM} = \&abort_builds; $pm->run_on_start(sub { my ($pid, $chroot) = @_; $MOCK_INFLIGHT{$pid} = $chroot if defined $chroot; }); $pm->run_on_finish(sub { - my ($pid, $exit_code, $ident) = @_; + my ($pid, $exit_code, $ident, $exit_signal, $core_dump) = @_; delete $MOCK_INFLIGHT{$pid}; - # A build/update child that die()s (e.g. a failed sh_retry) exits non-zero; record it so - # we never index or sign a partial repository (would ship a core missing packages). - push @CHILD_FAILURES, ($ident // "pid=$pid") if $exit_code; + # A child that die()s exits non-zero; one killed by a SIGNAL (SIGKILL / OOM-killer) is reaped + # with $exit_code==0 but $exit_signal!=0 (and maybe $core_dump). Checking $exit_code alone + # would let an OOM-killed worker through and the parent would index+sign a partial repository + # (a core missing packages). Treat a non-zero exit, a killing signal, OR a core dump as failure. + push @CHILD_FAILURES, ($ident // "pid=$pid") if $exit_code || $exit_signal || $core_dump; }); for my $pair (@rpms) {