xcat-dep/mockbuild-all.pl

#!/usr/bin/perl

use strict;
use warnings;

use Cwd qw(abs_path cwd);
use File::Basename qw(dirname basename);
use File::Copy qw(copy);
use File::Find qw(find);
use File::Path qw(make_path);
use Getopt::Long qw(GetOptions);
use Parallel::ForkManager;
use POSIX qw(strftime);

my $script_dir = abs_path(dirname(__FILE__));
my $repo_root  = abs_path($script_dir);
my $xcat_src   = "$repo_root/../xcat-core";
# Single knob for all NFS-shared output; --output-root/--repo-dep derive from it below
# unless explicitly overridden. Empty means "not set on the command line".
my $output     = '';
my $output_root = '';
my $target     = '';
my $nproc      = 1;
my $parallel_builds;
my $parallel_targets = 1;   # 1 = serial (default; safe). 0/auto = all EL targets at once; N = cap.
                            # NOTE: parallel targets need every per-package mockbuild.pl to avoid
                            # shared-path writes (repo tarballs, $HOME/rpmbuild); serial is safe today.
my $max_parallel = 0;       # 0/auto = host nproc: global cap on concurrent mock builds (all targets)
my $run_id     = '';
my $build_timestamp;
# CD version bump: when set, every xcat-dep package spec's Release gets a
# ".snap<YYYYMMDDHHMM>.<build_number>" suffix so each pipeline run publishes a
# fresh, monotonic NVR (deploy's additive rsync is a no-op otherwise). NOT applied
# to xCAT-genesis-base (built from xcat-core, kept in lockstep with genesis-scripts).
my $build_number;
my $skip_install = 0;
my $skip_build = 0;
my $skip_xcat_dep = 0;
my $skip_perl = 0;
my $skip_genesis = 0;
my $skip_createrepo = 0;
my $skip_tarball = 0;
my $scrub_all_chroots = 0;
my $keep_buildroots = 0;   # keep per-step mock chroots after build (default: --scrub=chroot each)
my $dry_run = 0;
my @extra_collect_dirs;
my $repo_dep = '';
my $gpg_sign = 0;
my $gpg_key_name = 'xCAT Signing Key';
my $gpg_home = '';
my $force_unlock = 0;
# --finalize-xcat-dep: post-build cross-arch genesis provisioning (issue #7610). Takes the two
# per-arch repo roots and cross-populates the noarch xCAT-genesis-base between them.
my $finalize_xcat_dep = 0;
my $x86_64_repo = '';
my $ppc64le_repo = '';
my $HELD_LOCK;        # path of the output lock this process owns (for cleanup on exit)
my $LOCK_OWNER_PID;   # pid that created the lock; forked children must NOT remove it

GetOptions(
    'repo-root=s'       => \$repo_root,
    'xcat-source=s'     => \$xcat_src,
    'output=s'          => \$output,
    'output-root=s'     => \$output_root,
    'repo-dep=s'        => \$repo_dep,
    'gpg-sign!'         => \$gpg_sign,
    'gpg-key-name=s'    => \$gpg_key_name,
    'gpg-home=s'        => \$gpg_home,
    'force-unlock!'     => \$force_unlock,
    'finalize-xcat-dep!' => \$finalize_xcat_dep,
    'x86_64-repo=s'     => \$x86_64_repo,
    'ppc64le-repo=s'    => \$ppc64le_repo,
    'target=s'          => \$target,
    'nproc=i'           => \$nproc,
    'parallel-builds=i' => \$parallel_builds,
    'parallel-targets=i' => \$parallel_targets,
    'max-parallel=i'    => \$max_parallel,
    'run-id=s'          => \$run_id,
    'build-timestamp=i' => \$build_timestamp,
    'build-number=i'    => \$build_number,
    'skip-install!'     => \$skip_install,
    'skip-build!'       => \$skip_build,
    'skip-xcat-dep!'    => \$skip_xcat_dep,
    'skip-perl!'        => \$skip_perl,
    'skip-genesis!'     => \$skip_genesis,
    'skip-createrepo!'  => \$skip_createrepo,
    'skip-tarball!'     => \$skip_tarball,
    'scrub-all-chroots!' => \$scrub_all_chroots,
    'keep-buildroots!'  => \$keep_buildroots,
    'collect-dir=s@'    => \@extra_collect_dirs,
    'dry-run!'          => \$dry_run,
) or die usage();

die "Run as root (uid=$>)\n" if $> != 0 && !$finalize_xcat_dep;
die "--parallel-builds must be >= 1\n"
    if defined($parallel_builds) && $parallel_builds < 1;

$repo_root = abs_path($repo_root);

my $SOURCE_DATE_EPOCH;
$SOURCE_DATE_EPOCH = $build_timestamp if defined $build_timestamp;
if (!$SOURCE_DATE_EPOCH && -f "$repo_root/Gitepoch") {
    $SOURCE_DATE_EPOCH = slurp_chomp("$repo_root/Gitepoch");
}
unless ($SOURCE_DATE_EPOCH && $SOURCE_DATE_EPOCH =~ /^\d+$/) {
    $SOURCE_DATE_EPOCH = `git -C \Q$repo_root\E log -1 --format=%ct HEAD 2>/dev/null`;
    chomp $SOURCE_DATE_EPOCH;
}
$SOURCE_DATE_EPOCH = time() unless $SOURCE_DATE_EPOCH =~ /^\d+$/;
$ENV{SOURCE_DATE_EPOCH} = $SOURCE_DATE_EPOCH;

if ($run_id eq '') {
    $run_id = strftime('%Y%m%d-%H%M%S', gmtime($SOURCE_DATE_EPOCH));
}

# --finalize-xcat-dep: a distinct, build-free mode. After BOTH arch build hosts have
# produced their per-EL repos (each carrying only its own xCAT-genesis-base), the x86_64
# repo must ALSO ship the noarch xCAT-genesis-base-ppc64 (so an x86_64 MN can netboot ppc
# nodes) and the ppc64le repo must ship xCAT-genesis-base-x86_64 -- the 2.17 behaviour
# that issue #7610 regressed. This mode ONLY cross-copies the genesis-base rpm(s) between
# the two repos (dropping any stale foreign-arch genesis) and re-indexes + re-signs the
# affected repomd; it builds nothing and holds no output lock.
if ($finalize_xcat_dep) {
    die "--finalize-xcat-dep requires --x86_64-repo and --ppc64le-repo\n"
        if $x86_64_repo eq '' || $ppc64le_repo eq '';
    require_command('createrepo_c');
    require_command('rpm');
    require_command('rpmsign') if $gpg_sign;
    require_command('gpg')     if $gpg_sign;
    my $x86 = abs_path($x86_64_repo) or die "--x86_64-repo '$x86_64_repo' not found\n";
    my $ppc = abs_path($ppc64le_repo) or die "--ppc64le-repo '$ppc64le_repo' not found\n";
    die "--x86_64-repo '$x86' is not a directory\n" if !-d $x86;
    die "--ppc64le-repo '$ppc' is not a directory\n" if !-d $ppc;
    finalize_xcat_dep($x86, $ppc);
    exit 0;
}

# CD version bump. Rewrite every xcat-dep package spec's Release line in this
# (freshly-checked-out, git-clean) tree so the built rpms carry a fresh, monotonic
# NVR each run. Runs BEFORE any child builder is invoked. genesis-base lives under
# xcat-core, not $repo_root, so it is untouched (stays in lockstep with the deployed
# core's genesis-scripts).
my $RELEASE_BUMP = '';
if (defined $build_number) {
    die "--build-number must be a non-negative integer\n" if $build_number < 0;
    $RELEASE_BUMP = strftime('.snap%Y%m%d%H%M', gmtime($SOURCE_DATE_EPOCH)) . ".$build_number";
    bump_dep_release_suffix($repo_root, $RELEASE_BUMP);
}

# Single output base for every NFS-shared write. Two hosts build in parallel on one NFS by
# passing distinct --output paths. --output-root/--repo-dep, if given, override the derived
# values. Default keeps the historical layout so existing callers are unaffected.
# Create the dir first, THEN abs_path -- abs_path() on a not-yet-existing path returns undef,
# and abs_path(undef) silently resolves to cwd, which would misdirect all output.
my $output_base = $output ne '' ? $output : "$repo_root/build-output";
make_path($output_base) if !-d $output_base;
$output_base = abs_path($output_base)
    or die "Cannot resolve --output base directory\n";
$output_root = "$output_base/mockbuild-all" if $output_root eq '';
# Deployable per-EL xcat-dep repo root (rh8/rh9/rh10/<arch> assembled here).
$repo_dep = "$output_base/xcat-dep" if $repo_dep eq '';

# Fail-fast lock on the output base so a second run against the same --output aborts instead of
# racing on the shared NFS tree. Held for the whole invocation; released by the exit handlers.
acquire_output_lock($output_base, $force_unlock);

$xcat_src  = resolve_xcat_source($xcat_src, $repo_root);

my $arch = capture('uname -m');
my %os = read_os_release('/etc/os-release');
my $os_id = $os{ID} // '';
my $version_id = $os{VERSION_ID} // '';
my ($rel) = $version_id =~ /^(\d+)/;

die "Could not resolve ID from /etc/os-release\n" if $os_id eq '';
die "Could not resolve major release from VERSION_ID='$version_id' in /etc/os-release\n"
    if !defined($rel) || $rel eq '';

for my $bin (qw(perl uname createrepo_c tar find rpm)) {
    require_command($bin);
}
require_command('mock') if $scrub_all_chroots;
require_command('rpmsign') if $gpg_sign;
require_command('gpg')     if $gpg_sign;

# An explicit --target builds just that target; otherwise build the current host
# arch across rh8/rh9/rh10 into a deployable per-EL xcat-dep repo. This script builds
# ONLY the host arch (uname -m) -- the other arch is produced on its own build host.
my @build_targets = $target
    ? ($target)
    : map { resolve_mock_cfg($os_id, $_, $arch) } (8, 9, 10);

# NOTE: no dhcp- packages are built here. DHCP backend selection is an install-time
# rich dep in xCAT.spec (kea if system-release>=10 else /usr/sbin/dhcpd), so there is
# nothing arch/EL-specific to build or to exclude for el10.

print_step('Targets to build');
print "  $_\n" for @build_targets;
print "output_base:      $output_base\n";
print "deploy repo-dep:  $repo_dep\n";
print "lock:             $output_base/.lock (held)\n";
print "gpg_sign:         $gpg_sign\n";
print "gpg_key_name:     $gpg_key_name\n" if $gpg_sign;
print "gpg_home:         " . ($gpg_home ne '' ? $gpg_home : '(default keyring)') . "\n" if $gpg_sign;

# Build (and deploy) EL targets concurrently. Each target is fully isolated -- distinct run_id
# (target-folded), mock --uniqueext, /tmp work dir, xcat_src/dist/<target>, and deploy dir
# rh<rel>/<arch> -- so there is no cross-target contention. 0/auto = all at once; 1 = serial.
my $tgt_workers = $parallel_targets > 0 ? $parallel_targets : scalar(@build_targets);
$tgt_workers = scalar(@build_targets) if $tgt_workers > scalar(@build_targets);
# Global cap on concurrent mock builds so parallel targets don't oversubscribe the host. Each
# mock build already gets a unique --uniqueext (separate chroot), so the only limit needed is
# hardware: total concurrent builds across all targets stays <= $cap (default host nproc). The
# per-target build-step concurrency is therefore the cap divided across the active targets.
my $cap = $max_parallel > 0 ? $max_parallel : (capture('nproc') || 4);
my $per_target_builds = defined($parallel_builds) ? $parallel_builds : int($cap / $tgt_workers);
$per_target_builds = 1 if $per_target_builds < 1;
print "parallel_targets: " . ($parallel_targets > 0 ? $parallel_targets : "auto($tgt_workers)") . "\n";
print "max_parallel:     $cap (per-target build workers: $per_target_builds)\n";
my $tgt_pm = Parallel::ForkManager->new($tgt_workers <= 1 ? 0 : $tgt_workers);
my $tgt_fail = 0;
$tgt_pm->run_on_finish(sub {
    my ($pid, $exit) = @_;
    $tgt_fail++ if $exit;
});
for my $tgt (@build_targets) {
    $tgt_pm->start and next;
    my $rc = 0;
    eval {
        my $info = build_one_target($tgt, $run_id, $per_target_builds);
        deploy_target($tgt, $info);
        1;
    } or do { warn "ERROR: target $tgt failed: $@"; $rc = 1; };
    $tgt_pm->finish($rc);
}
$tgt_pm->wait_all_children;
die "FATAL: $tgt_fail target(s) failed\n" if $tgt_fail;

print_step('All targets completed');
exit 0;

# Append $suffix (e.g. ".snap202607161200.57") to the Release: line of every xcat-dep
# package spec under $root, so the CD build stamps a fresh, monotonic NVR. Idempotent:
# a spec already carrying this exact suffix is left alone (so a re-run in the same tree
# does not double-stamp). Preserves any %{?dist}/%{?distver} macro already on the line.
sub bump_dep_release_suffix {
    my ($root, $suffix) = @_;
    my $qs = quotemeta($suffix);
    my @specs;
    find(sub { push @specs, $File::Find::name if /\.spec$/ && -f $_ }, $root);
    my ($with_release, $bumped, $already) = (0, 0, 0);
    for my $spec (sort @specs) {
        open my $in, '<', $spec or die "open $spec: $!\n";
        my @lines = <$in>;
        close $in;
        my ($has_release, $changed) = (0, 0);
        for my $line (@lines) {
            # case-insensitive: some specs (e.g. Sys-Virt.spec) use a lowercase `release:`
            next unless $line =~ /^Release:\s*\S/i;
            $has_release = 1;
            if ($line =~ /$qs\s*$/) { last }     # already stamped (idempotent / concurrent arch)
            $line =~ s/(^Release:\s*\S+)/$1$suffix/i;
            $changed = 1;
            last;                                 # only the first Release: line
        }
        $with_release++ if $has_release;
        $already++      if $has_release && !$changed;
        next unless $changed;
        # atomic write (temp + rename) so a concurrent per-arch build on the shared NFS tree never
        # sees a torn spec; identical suffix -> identical content, so last-writer-wins is safe.
        my $tmp = "$spec.bump.$$";
        open my $out, '>', $tmp or die "open> $tmp: $!\n";
        print {$out} @lines;
        close $out;
        rename $tmp, $spec or die "rename $tmp -> $spec: $!\n";
        $bumped++;
    }
    print "Release bump '$suffix': $bumped newly stamped, $already already stamped, of $with_release spec(s) with a Release line under $root\n";
    # Only a genuine "no dep specs at all" is fatal. All-already-stamped is the expected idempotent
    # case (re-run in the same tree, or the other arch bumped first) -- NOT an error.
    die "FATAL: --build-number given but NO spec carried a Release: line under $root (wrong tree?)\n"
        if $with_release == 0;
}

# Build a single target into its own build-output/<target-runid> tree and return
# { repo_dir, rel }. Everything below through the summary is per-target work.
sub build_one_target {
    my ($target, $run_id, $max_build_workers) = @_;
    # The build output identity must be per-target (os version + arch). SOURCE_DATE_EPOCH
    # is the same across targets for a given commit, so a timestamp-only run_id makes
    # different targets (e.g. alma+epel-8 vs -9) share build-output/<run_id> and
    # cross-contaminate. Fold the target into run_id so each target gets its own tree.
    $run_id = "$target-$run_id" unless index($run_id, $target) >= 0;
    my ($rel) = $target =~ /epel-(\d+)-/;
    die "Could not parse EL release from target '$target'\n" unless defined $rel;

my $run_root     = "$output_root/$run_id";
my $build_root   = "$run_root/build-results";
my $log_root     = "$run_root/build-logs";
my $repo_dir     = "$run_root/repo/$arch";
my $summary_file = "$run_root/summary.txt";
my $tarball      = "$output_root/mockbuild-all-$target-$run_id.tar.gz";
my $srpm_repo_dir = "$run_root/repo-src";
my $srpm_tarball  = "$output_root/mockbuild-all-$target-$run_id-srpm.tar.gz";

# All dep builders run natively on every arch. xnba-undi and grub2-xcat are noarch packagings of
# committed artifacts (an x86 UNDI ROM / the grub2 resource tarball) with no arch-specific build
# step, so ppc builds them the same as x86 -- no cross-arch import.
my @dep_builders = (
    { name => 'elilo-xcat',  script => "$repo_root/elilo/mockbuild.pl" },
    { name => 'grub2-xcat',  script => "$repo_root/grub2-xcat/mockbuild.pl" },
    { name => 'ipmitool-xcat', script => "$repo_root/ipmitool/mockbuild.pl" },
    { name => 'syslinux-xcat', script => "$repo_root/syslinux/mockbuild.pl" },
    { name => 'goconserver', script => "$repo_root/goconserver/mockbuild.pl" },
    { name => 'conserver-xcat', script => "$repo_root/conserver/mockbuild.pl" },
    { name => 'xnba-undi',   script => "$repo_root/xnba/mockbuild.pl" },
);

my $perl_builder = "$repo_root/mockbuild-perl-packages.pl";

# buildrpms.pl (in xcat-core) is only needed for the OS-dependent xCAT-genesis-base
# build below; the full xCAT core is built separately by the xcat-core pipeline.
die "Missing xCAT build script: $xcat_src/buildrpms.pl\n"
    if !$skip_genesis && !-f "$xcat_src/buildrpms.pl";

my @active_dep_builders;
for my $b (@dep_builders) {
    if (-f $b->{script}) {
        push @active_dep_builders, $b;
        next;
    }
    print "WARN: missing dep builder script, skipping: $b->{script}\n";
}
die "Missing perl builder script: $perl_builder\n"
    if !$skip_perl && !$perl_builder;

if (!$dry_run) {
    make_path($build_root, $log_root, $repo_dir, $srpm_repo_dir);
}

print_step("Configuration");
print "repo_root:        $repo_root\n";
print "xcat_source:      $xcat_src\n";
print "output_root:      $output_root\n";
print "run_root:         $run_root\n";
print "arch:             $arch\n";
print "os_id:            $os_id\n";
print "version_id:       $version_id\n";
print "rel:              $rel\n";
print "target:           $target\n";
print "nproc:            $nproc\n";
print "parallel_builds:  " . (defined($parallel_builds) ? $parallel_builds : 'auto') . "\n";
print "skip_build:       $skip_build\n";
print "skip_xcat_dep:    $skip_xcat_dep\n";
print "skip_perl:        $skip_perl\n";
print "skip_genesis:     $skip_genesis\n";
print "skip_install:     $skip_install\n";
print "skip_createrepo:  $skip_createrepo\n";
print "skip_tarball:     $skip_tarball\n";
print "scrub_all_chroots:$scrub_all_chroots\n";
print "keep_buildroots:  $keep_buildroots\n";
print "dry_run:          $dry_run\n";
print "perl_builder:     $perl_builder\n";
print "tarball:          $tarball\n";
print "srpm_repo_dir:    $srpm_repo_dir\n";
print "srpm_tarball:     $srpm_tarball\n";

my @collect_roots;

if ($scrub_all_chroots) {
    run_step(
        step => "Scrub all chroots for target $target",
        cmd  => "mock -r " . sh_quote($target) . " --scrub=all",
        log  => "$log_root/scrub-all-chroots.log",
    );
}

if (!$skip_build) {
    my @build_steps;
    my $build_step_seq = 0;

    if (!$skip_xcat_dep) {
        for my $builder (@active_dep_builders) {
            my $name = $builder->{name};
            my $script = $builder->{script};
            my $step_result = "$build_root/$name";
            my $step_log    = "$log_root/$name";
            my $step_uniqueext = build_mock_uniqueext($run_id, ++$build_step_seq, $name);
            my $cmd = join(' ',
                'perl', sh_quote($script),
                '--mock-cfg', sh_quote($target),
                '--mock-uniqueext', sh_quote($step_uniqueext),
                '--result-dir', sh_quote($step_result),
                '--log-dir', sh_quote($step_log),
                # host-local, run-scoped work dir so /tmp doesn't collide between runs
                '--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/$name"),
                '--build-timestamp', $SOURCE_DATE_EPOCH,
                ($skip_install ? '--skip-install' : ()),
            );
            push @build_steps, {
                id   => "xcat-dep:$name",
                step => "Build xcat-dep: $name",
                cmd  => $cmd,
                log  => "$log_root/$name/run.log",
                scrub_cfg       => $target,
                scrub_uniqueext => $step_uniqueext,
            };
            push @collect_roots, $step_result;
        }
    }

    if (!$skip_perl) {
        my $perl_result = "$build_root/perl/$arch";
        my $perl_log    = "$log_root/perl/$arch";
        my $perl_uniqueext = build_mock_uniqueext($run_id, ++$build_step_seq, 'perl-list6');
        # Bound the perl builder's OWN internal parallelism to this target's budget; otherwise it
        # forks one mock build per perl package (~7), which -- multiplied by parallel EL targets --
        # oversubscribes the host.
        my $cmd = join(' ',
            'perl', sh_quote($perl_builder),
            '--mock-cfg', sh_quote($target),
            '--mock-uniqueext', sh_quote($perl_uniqueext),
            '--result-dir', sh_quote($perl_result),
            '--log-dir', sh_quote($perl_log),
            '--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/perl-list6"),
            (($max_build_workers && $max_build_workers >= 1) ? ('--jobs', $max_build_workers) : ()),
            '--build-timestamp', $SOURCE_DATE_EPOCH,
            # CD bump: the in-tree spec Release bump above only reaches the spec-mode perl
            # packages; the srpm-mode ones (HTML-Form, IO-Stty, Net-Telnet) build from a
            # committed .src.rpm, so hand the suffix down for the builder to re-stamp them.
            ($RELEASE_BUMP ne '' ? ('--release-suffix', sh_quote($RELEASE_BUMP)) : ()),
            ($skip_install ? '--skip-install' : ()),
            ($keep_buildroots ? '--keep-buildroots' : ()),
        );
        push @build_steps, {
            id   => 'perl',
            step => 'Build perl xcat-dep packages',
            cmd  => $cmd,
            log  => "$log_root/perl-build.log",
        };
        push @collect_roots, $perl_result;
    }

    # NOTE: this script builds ONLY xcat-dep (its dep packages, the perl packages, and
    # the OS-dependent xCAT-genesis-base below). The full xCAT core is built separately
    # by the xcat-core pipeline -- mockbuild-all no longer has a monolithic core-build path.

    # xCAT-genesis-base is OS-dependent (its initramfs bundles the build chroot's
    # kernel + glibc/busybox/perl), so it is built here, per target, and shipped
    # in this per-EL xcat-dep repo rather than in the flat xcat-core. buildrpms.pl
    # (run in the xcat-core dir) derives the same snapYYYYMMDDHHMM Release from
    # xcat-core's Gitepoch, so it matches xCAT-genesis-scripts (built in core) and
    # the exact-version dependency genesis-scripts -> genesis-base resolves.
    if (!$skip_genesis) {
        # buildrpms.pl stages sources in $HOME/rpmbuild (via rpmdev-setuptree). Give each
        # per-target genesis build its own HOME so parallel EL targets don't race on the shared
        # /root/rpmbuild tree (that race is what made concurrent genesis builds fail).
        my $genesis_home = "/tmp/mockbuild-all-$run_id/genesis-home";
        # buildrpms.pl's rpmdev-setuptree only runs during env setup, not per build, so create the
        # rpmbuild tree ourselves for this per-target HOME (else $HOME/rpmbuild/SOURCES is missing).
        my $mktree = join(' ', map { sh_quote("$genesis_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS));
        my $cmd = "mkdir -p $mktree && HOME=" . sh_quote($genesis_home) . ' ' . join(' ',
            'perl', sh_quote("$xcat_src/buildrpms.pl"),
            '--package', 'xCAT-genesis-base',
            '--target', sh_quote($target),
            '--nproc', int($nproc),
            '--force',
            '--verbose',
            '--xcat_dep_path', sh_quote($repo_root),
        );
        push @build_steps, {
            id   => 'genesis',
            step => 'Build xCAT-genesis-base (per-target, OS-dependent)',
            cmd  => $cmd,
            cwd  => $xcat_src,
            log  => "$log_root/genesis-build.log",
            scrub_cfg => "xCAT-genesis-base-$target",
        };
    }

    if (@build_steps) {
        # Prefer the caller-supplied cap (global budget / active targets). Fall back to the old
        # behaviour (all steps at once) only when unset.
        my $effective_parallel_builds =
              ($max_build_workers && $max_build_workers >= 1) ? $max_build_workers
            : defined($parallel_builds)                       ? $parallel_builds
            :                                                   scalar(@build_steps);
        run_build_steps_parallel(
            steps         => \@build_steps,
            max_processes => $effective_parallel_builds,
        );

        # Reclaim each build step's mock chroot now that the step copied its RPMs/logs out to
        # its --result-dir (collect_rpms reads those, never /var/lib/mock). mock's own cleanup
        # leaves these chroots behind -- and keeps them entirely on failure -- so /var/lib/mock
        # grows ~15-17G per run until the host fills and every dnf transaction fails for lack of
        # space. Scrub each via `mock --scrub=chroot --scrub=bootstrap` (never rm): it takes the
        # chroot lock, so a chroot still used by a concurrent build is refused and safely skipped.
        # Both the build chroot and its per-uniqueext bootstrap are removed; the root cache stays
        # for fast rebuilds. Perl packages are scrubbed inside mockbuild-perl-packages.pl (it
        # derives its own per-package uniqueexts).
        unless ($keep_buildroots) {
            for my $s (@build_steps) {
                next unless defined $s->{scrub_cfg};
                (my $slug = $s->{id}) =~ s/[^\w.-]+/-/g;
                scrub_buildroot($s->{scrub_cfg}, $s->{scrub_uniqueext}, "$log_root/scrub-$slug.log");
            }
        }
    }
}

# The xCAT core is built by the xcat-core pipeline, NOT here -- so we deliberately do
# NOT collect the xCAT dist tree. Only the OS-dependent xCAT-genesis-base rpm (built by
# the genesis step above) is pulled out of it, individually, further below.
my $xcat_rpms_dir = "$xcat_src/dist/$target/rpms";

if ($skip_build) {
    push @collect_roots,
        "$repo_root/build-output/list3/elilo-xcat",
        "$repo_root/build-output/list3/grub2-xcat",
        "$repo_root/build-output/list3/ipmitool-xcat",
        "$repo_root/build-output/list3/syslinux-xcat",
        "$repo_root/build-output/list3/xnba-undi",
        "$repo_root/build-output/list5/goconserver/$arch",
        "$repo_root/goconserver-build-$arch/results/rpm",
        "$repo_root/build-output/list6/perl/$arch",
        "$repo_root/perl-list6/$arch";
}

push @collect_roots, @extra_collect_dirs;
@collect_roots = uniq(@collect_roots);
my @srpm_collect_roots = uniq(@collect_roots);

print_step('Collect RPM artifacts');
print "collection roots:\n";
print "  $_\n" for @collect_roots;

my ($copied, $skipped_src, $missing_roots) = collect_rpms(
    roots    => \@collect_roots,
    dest_dir => $repo_dir,
    dry_run  => $dry_run,
);

if (!$dry_run && $copied == 0) {
    die "No binary RPMs were collected. Check build logs and collection roots.\n";
}

# Ensure the OS-dependent xCAT-genesis-base rpm (built by the genesis step above)
# lands in the dep repo -- pull it individually out of the xcat-core dist tree (the
# rest of that tree, the full xCAT core, is built + published by the xcat-core pipeline).
if (!$skip_genesis && !$dry_run) {
    for my $g (glob("$xcat_rpms_dir/xCAT-genesis-base-*.rpm")) {
        next if $g =~ /\.src\.rpm$/;
        copy($g, "$repo_dir/" . basename($g))
            or die "Failed to copy genesis-base $g -> $repo_dir: $!\n";
        $copied++;
    }
}

print_step('Collect source RPM artifacts');
print "source collection roots:\n";
print "  $_\n" for @srpm_collect_roots;

my ($copied_srpms, $skipped_non_src, $missing_srpm_roots) = collect_srpms(
    roots    => \@srpm_collect_roots,
    dest_dir => $srpm_repo_dir,
    dry_run  => $dry_run,
);

if (!$dry_run && $copied_srpms == 0) {
    print "WARN: No source RPMs were collected. SRPM repo and tarball may be empty.\n";
}

if (!$skip_createrepo) {
    run_step(
        step => 'Run createrepo',
        cmd  => createrepo_c_cmd($repo_dir),
        log  => "$log_root/createrepo.log",
    );
    run_step(
        step => 'Run createrepo for SRPM repo',
        cmd  => createrepo_c_cmd($srpm_repo_dir),
        log  => "$log_root/createrepo-srpm.log",
    );
}

if (!$skip_tarball) {
    my $cmd = join(' ',
        'tar', '--sort=name', '--owner=0', '--group=0',
        "--mtime=\@$SOURCE_DATE_EPOCH",
        '-C', sh_quote($run_root),
        '-czf', sh_quote($tarball),
        'repo'
    );
    run_step(
        step => 'Create tarball',
        cmd  => $cmd,
        log  => "$log_root/tarball.log",
    );
    my $srpm_cmd = join(' ',
        'tar', '--sort=name', '--owner=0', '--group=0',
        "--mtime=\@$SOURCE_DATE_EPOCH",
        '-C', sh_quote($run_root),
        '-czf', sh_quote($srpm_tarball),
        'repo-src'
    );
    run_step(
        step => 'Create SRPM tarball',
        cmd  => $srpm_cmd,
        log  => "$log_root/tarball-srpm.log",
    );
}

if (!$dry_run) {
    open my $sfh, '>', $summary_file or die "Cannot write $summary_file: $!\n";
    print {$sfh} "run_root=$run_root\n";
    print {$sfh} "repo_dir=$repo_dir\n";
    print {$sfh} "target=$target\n";
    print {$sfh} "arch=$arch\n";
    print {$sfh} "os_id=$os_id\n";
    print {$sfh} "version_id=$version_id\n";
    print {$sfh} "rel=$rel\n";
    print {$sfh} "copied_rpms=$copied\n";
    print {$sfh} "skipped_src_rpms=$skipped_src\n";
    print {$sfh} "missing_collection_roots=$missing_roots\n";
    print {$sfh} "srpm_repo_dir=$srpm_repo_dir\n";
    print {$sfh} "copied_srpms=$copied_srpms\n";
    print {$sfh} "skipped_non_src_rpms=$skipped_non_src\n";
    print {$sfh} "missing_srpm_collection_roots=$missing_srpm_roots\n";
    print {$sfh} "tarball=$tarball\n" if !$skip_tarball;
    print {$sfh} "srpm_tarball=$srpm_tarball\n" if !$skip_tarball;
    close $sfh;
}

print_step('Completed');
print "Collected binary RPMs: $copied\n";
print "Skipped source RPMs:   $skipped_src\n";
print "Missing roots:         $missing_roots\n";
print "Repo dir:              $repo_dir\n";
print "Collected source RPMs: $copied_srpms\n";
print "Skipped non-src RPMs:  $skipped_non_src\n";
print "Missing source roots:  $missing_srpm_roots\n";
print "SRPM repo dir:         $srpm_repo_dir\n";
print "Summary:               $summary_file\n" if !$dry_run;
print "Tarball:               $tarball\n" if !$skip_tarball;
print "SRPM Tarball:          $srpm_tarball\n" if !$skip_tarball;

    return { repo_dir => $repo_dir, rel => $rel };
}

# Assemble the built per-target repo into the deployable, signed per-EL layout
# <repo-dep>/rh<rel>/<arch>: copy the binary rpms, sign, createrepo, and drop the
# xcat-dep.repo / mklocalrepo.sh / buildinfo.txt (ready to push to xcat.org).
sub deploy_target {
    my ($tgt, $info) = @_;
    my $rel  = $info->{rel};
    my $src  = $info->{repo_dir};
    my $dest = "$repo_dep/rh$rel/$arch";
    print_step("Deploy $tgt -> $dest");
    return if $dry_run;
    make_path($dest);
    for my $rpm (glob("$src/*.rpm")) {
        next if $rpm =~ /\.src\.rpm$/;
        copy($rpm, "$dest/" . basename($rpm))
            or die "Failed to copy $rpm -> $dest: $!\n";
    }
    assert_required_deps($dest);
    sign_and_index_repo($dest);
    write_dep_repo_metadata($dest, $rel);
    my $n = scalar(grep { !/\.src\.rpm$/ } glob("$dest/*.rpm"));
    print "Deployed rh$rel/$arch: $n rpms\n";
}

# createrepo_c command with upstream-matching, deterministic metadata. The tool's
# defaults emit primary/filelists/other as *.xml.zst plus *.sqlite.bz2 (--database),
# exactly the upstream shape; --set-timestamp-to-revision pins repomd to SOURCE_DATE_EPOCH.
sub createrepo_c_cmd {
    my ($dir) = @_;
    return 'createrepo_c --update --database '
        . '--revision ' . sh_quote($SOURCE_DATE_EPOCH) . ' --set-timestamp-to-revision '
        . sh_quote($dir);
}

sub sign_and_index_repo {
    my ($dir) = @_;
    my @rpms = grep { !/\.src\.rpm$/ } glob("$dir/*.rpm");
    if ($gpg_sign && @rpms) {
        local $ENV{GNUPGHOME} = $gpg_home if $gpg_home;
        run_simple(qq(rpmsign --define "%_gpg_name $gpg_key_name" --addsign )
            . join(' ', map { sh_quote($_) } @rpms));
    }
    run_simple(createrepo_c_cmd($dir));
    if ($gpg_sign) {
        local $ENV{GNUPGHOME} = $gpg_home if $gpg_home;
        my $repomd = "$dir/repodata/repomd.xml";
        unlink "$repomd.asc" if -f "$repomd.asc";
        run_simple(qq(gpg -a --detach-sign --default-key "$gpg_key_name" ) . sh_quote($repomd));
        run_simple(qq(gpg -a --export "$gpg_key_name" > ) . sh_quote("$repomd.key"));
    }
}

sub write_dep_repo_metadata {
    my ($dir, $rel) = @_;
    my $baseurl = "https://xcat.org/files/xcat/repos/yum/devel/xcat-dep/rh$rel/$arch";
    my $gpgcheck = $gpg_sign ? 1 : 0;
    my $gpgkey_line = $gpg_sign ? "gpgkey=$baseurl/repodata/repomd.xml.key" : "# gpgkey=";
    open my $r, '>', "$dir/xcat-dep.repo" or die "Cannot write $dir/xcat-dep.repo: $!\n";
    print {$r} <<"EOF";
[xcat-dep]
name=xCAT 2 dependencies (rh$rel $arch)
baseurl=$baseurl
enabled=1
gpgcheck=$gpgcheck
$gpgkey_line
EOF
    close $r;

    open my $m, '>', "$dir/mklocalrepo.sh" or die "Cannot write $dir/mklocalrepo.sh: $!\n";
    print {$m} <<'EOS';
#!/bin/sh
cd `dirname $0`
REPOFILE=`basename xcat-*.repo`
if [[ $REPOFILE == "xcat-*.repo" ]]; then
    echo "ERROR: For xcat-dep, please execute $0 in the correct <os>/<arch> subdirectory"
    exit 1
fi
DIRECTORY="/etc/yum.repos.d"
if [ ! -d "$DIRECTORY" ]; then
    DIRECTORY="/etc/zypp/repos.d"
fi
sed -e 's|baseurl=.*|baseurl=file://'"`pwd`"'|' $REPOFILE | sed -e 's|gpgkey=.*|gpgkey=file://'"`pwd`"'/repodata/repomd.xml.key|' > "$DIRECTORY/$REPOFILE"
cd -
EOS
    close $m;
    chmod 0775, "$dir/mklocalrepo.sh";

    my $build_time = strftime("%a %b %e %H:%M:%S %Z %Y", gmtime($SOURCE_DATE_EPOCH));
    my $build_machine = `hostname`; chomp $build_machine;
    my $commit = `git -C "$repo_root" rev-parse HEAD 2>/dev/null`; chomp $commit;
    $commit ||= 'unknown';
    my $commit_short = substr($commit, 0, 7);
    my $release = strftime('snap%Y%m%d%H%M', gmtime($SOURCE_DATE_EPOCH));
    open my $b, '>', "$dir/buildinfo.txt" or die "Cannot write $dir/buildinfo.txt: $!\n";
    print {$b} <<"EOF";
TARGET=rh$rel/$arch
RELEASE=$release
BUILD_TIME=$build_time
BUILD_MACHINE=$build_machine
COMMIT_ID=$commit_short
COMMIT_ID_LONG=$commit
SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH
EOF
    close $b;
}

# --finalize-xcat-dep: cross-populate the noarch xCAT-genesis-base between each matching
# <os>/x86_64 and <os>/ppc64le repo pair, then re-index + re-sign the repos that changed.
# <x86_64-repo>/<ppc64le-repo> are the two per-arch repo roots (each holding rh8/rh9/rh10/<arch>).
# They may be the same path (both arches built into one tree) or two separate trees (one per
# build host); either way pairs are matched by <os> subdir.
sub finalize_xcat_dep {
    my ($x86_64_repo, $ppc64le_repo) = @_;
    print_step('Finalize xcat-dep: cross-arch genesis-base provisioning (issue #7610)');
    print "x86_64-repo:  $x86_64_repo\n";
    print "ppc64le-repo: $ppc64le_repo\n";
    # Every OS-release dir under the x86_64 repo that actually has an x86_64 sub-repo.
    my @osdirs = grep { -d "$_/x86_64" } glob("$x86_64_repo/*");
    my $pairs = 0;
    for my $p (sort @osdirs) {
        my $osdir  = basename($p);
        my $x86dir = "$x86_64_repo/$osdir/x86_64";
        my $ppcdir = "$ppc64le_repo/$osdir/ppc64le";
        if (!-d $ppcdir) {
            print "[finalize] $osdir: no ppc64le peer at $ppcdir -- skipping\n";
            next;
        }
        # xCAT collapses ppc/ppc64/ppc64le into tarch=ppc64, so the ppc genesis rpm is
        # named xCAT-genesis-base-ppc64-*. Cross-copy both directions.
        my $to_x86 = cross_copy_genesis($ppcdir, $x86dir, 'ppc64');
        my $to_ppc = cross_copy_genesis($x86dir, $ppcdir, 'x86_64');
        reindex_and_sign_repo($x86dir) if $to_x86;
        reindex_and_sign_repo($ppcdir) if $to_ppc;
        printf "[finalize] %s: %d ppc64 genesis -> x86_64, %d x86_64 genesis -> ppc64le\n",
            $osdir, $to_x86, $to_ppc;
        $pairs++;
    }
    die "FATAL: --finalize-xcat-dep found no <os>/x86_64 + <os>/ppc64le repo pair under\n"
      . "  --x86_64-repo '$x86_64_repo'\n  --ppc64le-repo '$ppc64le_repo'\n" if $pairs == 0;
    print_step('Finalize complete');
}

# Cross-copy the noarch xCAT-genesis-base-<tarch>-*.rpm from $from into $to. Drops any
# stale foreign-arch genesis already in $to (e.g. issue #7610's 2.16.3 ppc leftover) so
# the repo ends with exactly the fresh set. Returns the number of rpms newly copied
# (0 = already up to date, so the caller can skip re-indexing). Idempotent.
sub cross_copy_genesis {
    my ($from, $to, $tarch) = @_;
    my @src = grep { !/\.src\.rpm$/ } glob("$from/xCAT-genesis-base-$tarch-*.rpm");
    return 0 if !@src;
    my %want = map { basename($_) => $_ } @src;
    my @existing = grep { !/\.src\.rpm$/ } glob("$to/xCAT-genesis-base-$tarch-*.rpm");
    my %have = map { basename($_) => 1 } @existing;
    # Already exactly the fresh set (same basenames)? idempotent no-op.
    if (scalar(keys %want) == scalar(keys %have) && !grep { !$have{$_} } keys %want) {
        return 0;
    }
    for my $old (@existing) {
        unlink $old or die "Failed to remove stale genesis $old: $!\n";
        print "[finalize]   - " . basename($old) . " (stale foreign-arch, removed from $to)\n";
    }
    my $copied = 0;
    for my $base (sort keys %want) {
        copy($want{$base}, "$to/$base")
            or die "Failed to cross-copy genesis $want{$base} -> $to: $!\n";
        print "[finalize]   + $base  ($from -> $to)\n";
        # The source rpm is already signed by the build, but re-assert it under --gpg-sign
        # so the deploy signing gate never sees an unsigned cross-copied rpm.
        if ($gpg_sign) {
            local $ENV{GNUPGHOME} = $gpg_home if $gpg_home;
            run_simple(qq(rpmsign --define "%_gpg_name $gpg_key_name" --addsign )
                . sh_quote("$to/$base"));
        }
        $copied++;
    }
    return $copied;
}

# Re-run createrepo_c on a repo whose rpm set changed, and (under --gpg-sign) re-sign +
# re-export repomd. Does NOT re-sign the rpms (cross_copy_genesis already did the copied
# one; the rest keep their build-time signatures).
sub reindex_and_sign_repo {
    my ($dir) = @_;
    run_simple(createrepo_c_cmd($dir));
    if ($gpg_sign) {
        local $ENV{GNUPGHOME} = $gpg_home if $gpg_home;
        my $repomd = "$dir/repodata/repomd.xml";
        unlink "$repomd.asc" if -f "$repomd.asc";
        run_simple(qq(gpg -a --detach-sign --default-key "$gpg_key_name" ) . sh_quote($repomd));
        run_simple(qq(gpg -a --export "$gpg_key_name" > ) . sh_quote("$repomd.key"));
    }
}

sub usage {
    return <<"USAGE";
Usage: $0 [options]

Build xcat-dep RPMs (dep packages, perl packages, and the OS-dependent xCAT-genesis-base),
consolidate binary/source artifacts, run createrepo, and create tarballs. The full xCAT core
is built separately by the xcat-core pipeline, not here.

Options:
  --repo-root PATH        xcat-dep repository root (default: script directory)
  --xcat-source PATH      xCAT source root with buildrpms.pl (default: <repo-root>/../xcat-core)
  --output PATH           Single base for ALL output; --output-root and --repo-dep derive from
                          it. A fail-fast lock is held at <PATH>/.lock, so pass distinct paths
                          to run on two hosts in parallel on one NFS (default: <repo-root>/build-output)
  --output-root PATH      Override the derived build tree root (default: <output>/mockbuild-all)
  --repo-dep PATH         Override the derived deployable per-EL output root; rh8/rh9/rh10/<arch>
                          are assembled + signed here (default: <output>/xcat-dep)
  --force-unlock          Remove a stale <output>/.lock before acquiring it
  --finalize-xcat-dep     Post-build cross-arch genesis mode (builds nothing). Requires
                          --x86_64-repo and --ppc64le-repo. For each matching <os>/x86_64 and
                          <os>/ppc64le repo pair, copies the noarch xCAT-genesis-base-ppc64
                          (the ppc64le genesis; xCAT names it -ppc64 via tarch, no big-endian
                          code) into the x86_64 repo and xCAT-genesis-base-x86_64 into the
                          ppc64le repo (dropping any stale foreign-arch genesis), then
                          re-indexes + re-signs. Restores the 2.17 cross-arch genesis
                          (issue #7610). Honors --gpg-sign/--gpg-key-name/--gpg-home. Use alone.
  --x86_64-repo PATH      (finalize) x86_64 repo root holding <os>/x86_64 (e.g. rh9/x86_64)
  --ppc64le-repo PATH     (finalize) ppc64le repo root holding <os>/ppc64le
  --gpg-sign              Sign rpms + repomd.xml of each per-EL repo
  --gpg-key-name NAME     GPG key name (default: "xCAT Signing Key")
  --gpg-home PATH         GNUPGHOME for signing (default: system keyring)
  --target NAME           Build only this target (<ID>+epel-<REL>-<ARCH>); default is
                          the host arch across rh8, rh9 and rh10
  --nproc N               Parallel jobs for buildrpms.pl (default: 1)
  --parallel-builds N     Max concurrent top-level build steps within one EL target (default: auto)
  --parallel-targets N    Concurrent EL targets (rh8/rh9/rh10). 0/auto = all at once, 1 = serial,
                          N = cap at N. Each target is fully output-isolated (default: auto)
  --max-parallel N        Global cap on concurrent mock builds across ALL targets, to avoid
                          oversubscribing the host. Split evenly across active targets.
                          0/auto = host nproc (default: auto)
  --run-id ID             Run identifier suffix (default: derived from build timestamp)
  --build-timestamp EPOCH Unix epoch for deterministic builds (default: Gitepoch or git log)
  --skip-install          Skip install/smoke tests in child builder scripts
  --skip-build            Skip all build steps and only collect/create repo/tarballs
  --skip-xcat-dep         Skip xcat-dep mockbuild.pl package steps
  --skip-perl             Skip perl package build step
  --skip-genesis          Skip the xCAT-genesis-base build step
  --skip-createrepo       Skip createrepo
  --skip-tarball          Skip binary/SRPM tarball creation
  --scrub-all-chroots     Run mock -r <target> --scrub=all before build/collect
  --collect-dir PATH      Additional directory to scan recursively for RPMs (repeatable)
  --dry-run               Print planned commands without executing

Notes:
  - Run this script as root on the build host.
  - ARCH is derived from: uname -m
  - Top-level parallel queue includes xcat-dep mockbuild.pl steps, the perl builder,
    and the xCAT-genesis-base build (../xcat-core/buildrpms.pl --package xCAT-genesis-base).
  - Child mockbuild scripts are invoked with per-step mock --uniqueext values
    to avoid lock collisions on the same mock config.
  - If --target is omitted, it is deduced from /etc/os-release:
      ID + epel + int(VERSION_ID) + ARCH
USAGE
}

sub print_step {
    my ($msg) = @_;
    print "\n== $msg ==\n";
}

sub require_command {
    my ($cmd) = @_;
    run_simple("command -v " . sh_quote($cmd) . " >/dev/null 2>&1");
}

sub run_simple {
    my ($cmd) = @_;
    my $rc = system($cmd);
    if ($rc != 0) {
        my $exit = $rc == -1 ? 255 : ($rc >> 8);
        die "Command failed (rc=$exit): $cmd\n";
    }
}

sub capture {
    my ($cmd) = @_;
    my $out = `$cmd`;
    my $rc = $?;
    if ($rc != 0) {
        my $exit = $rc == -1 ? 255 : ($rc >> 8);
        die "Command failed (rc=$exit): $cmd\n$out\n";
    }
    chomp $out;
    return $out;
}

sub run_step {
    my (%args) = @_;
    my $step = $args{step} // 'Run command';
    my $cmd  = $args{cmd}  // die "run_step missing cmd\n";
    my $cwd  = $args{cwd};
    my $log  = $args{log};

    print_step($step);
    print "+ $cmd\n";
    if ($cwd) {
        print "  (cwd: $cwd)\n";
    }
    if ($log) {
        print "  (log: $log)\n";
    }

    return if $dry_run;

    my $full_cmd = $cmd;
    if ($cwd) {
        $full_cmd = "cd " . sh_quote($cwd) . " && $cmd";
    }
    if ($log) {
        my $log_dir = dirname($log);
        make_path($log_dir) if !-d $log_dir;
        $full_cmd .= " > " . sh_quote($log) . " 2>&1";
    }

    my $rc = system($full_cmd);
    if ($rc != 0) {
        my $exit = $rc == -1 ? 255 : ($rc >> 8);
        die "Step failed (rc=$exit): $step\nCommand: $cmd\n";
    }
}

# Scrub a single mock buildroot via mock's own lock-safe --scrub. Never rm: if a concurrent build
# still holds the chroot lock, mock refuses and we skip it. Failures (already scrubbed, locked, or
# config missing) are tolerated -- a cleanup hiccup must never fail the build. Scrubs both the
# build chroot and its per-uniqueext bootstrap chroot (each build step gets its own bootstrap, so
# both must go or /var/lib/mock still leaks). The shared root cache under /var/cache/mock is kept,
# so rebuilds stay fast. $uniqueext is optional (genesis has none).
sub scrub_buildroot {
    my ($cfg, $uniqueext, $log) = @_;
    return if !defined $cfg || $cfg eq '';
    my $ext = (defined $uniqueext && $uniqueext ne '')
        ? ' --uniqueext ' . sh_quote($uniqueext) : '';
    eval {
        run_step(
            step => "Scrub chroot $cfg$ext",
            cmd  => "mock -r " . sh_quote($cfg) . $ext . " --scrub=chroot --scrub=bootstrap",
            log  => $log,
        );
        1;
    } or do {
        warn "WARN: chroot scrub failed (tolerated) for $cfg$ext: $@";
    };
}

sub run_build_steps_parallel {
    my (%args) = @_;
    my $steps = $args{steps} // [];
    my $max_processes = $args{max_processes} // 1;
    return if !@{$steps};

    # Individual dep-builder failures here are TOLERATED only so one flaky builder does not abort
    # the others. This is load-bearing, NOT laziness: some builders are expected to fail on a given
    # arch/el (e.g. perl-Sys-Virt on el8 -- not a required dep), and some REQUIRED builders "fail"
    # cosmetically while still producing their rpm (xCAT-genesis-base: xcat-core buildrpms.pl exits
    # non-zero on an unrelated post-build xCAT-release-latest cp, yet the genesis rpm is built). So
    # correctness is enforced by RESULT, not exit code: assert_required_deps runs after collection
    # and fails the whole run if any REQUIRED rpm is missing -- caught at assert time, not swept
    # under the rug. (A blanket "die on any builder failure" reddens the build on these non-issues.)
    if ($dry_run || $max_processes <= 1 || @{$steps} == 1) {
        for my $step (@{$steps}) {
            my $ok = eval { run_step(%{$step}); 1 };
            warn "WARN: build step failed (tolerated): $step->{step}\n" . ($@ // '') unless $ok;
        }
        return;
    }

    my $workers = $max_processes;
    $workers = scalar(@{$steps}) if $workers > scalar(@{$steps});

    print_step('Run build steps in parallel');
    print "max_processes: $workers\n";
    print "queued steps:\n";
    print "  - $_->{step}\n" for @{$steps};

    my %failed;
    my $pm = Parallel::ForkManager->new($workers);
    $pm->run_on_finish(
        sub {
            my ($pid, $exit_code, $ident, $signal, $core_dump) = @_;
            return if $exit_code == 0 && $signal == 0 && !$core_dump;
            my $key = defined($ident) ? $ident : "pid:$pid";
            $failed{$key} = {
                exit      => $exit_code,
                signal    => $signal,
                core_dump => $core_dump ? 1 : 0,
            };
        }
    );

    for my $step (@{$steps}) {
        my %step_copy = %{$step};
        my $ident = delete $step_copy{id};
        $ident = $step_copy{step} if !defined($ident) || $ident eq '';

        my $pid = $pm->start($ident);
        next if $pid;

        my $ok = eval {
            run_step(%step_copy);
            1;
        };
        if (!$ok) {
            my $err = $@;
            $err = "unknown error\n" if !defined($err) || $err eq '';
            print STDERR "ERROR [$ident] $err";
            $pm->finish(1);
        }
        $pm->finish(0);
    }
    $pm->wait_all_children;

    if (%failed) {
        my @lines;
        for my $id (sort keys %failed) {
            my $f = $failed{$id};
            push @lines,
                "$id (exit=$f->{exit}, signal=$f->{signal}, core_dump=$f->{core_dump})";
        }
        # Tolerated: warn, don't die. The REQUIRED set is asserted after collection/deploy.
        warn "WARN: some build steps failed (tolerated; required deps asserted after deploy):\n  "
            . join("\n  ", @lines) . "\n";
    }
}

# have_rpm: is there a non-src rpm named <name>-... under $dir?
sub have_rpm {
    my ($dir, $name) = @_;
    my @m = grep { !/\.src\.rpm$/ } glob("$dir/${name}-*.rpm");
    return scalar(@m) > 0;
}

# assert_required_deps: the per-EL dep repo is unusable without these, so a MISSING one is
# fatal even though individual builder failures are tolerated above. genesis-base is required
# unless --skip-genesis.
sub assert_required_deps {
    my ($dir) = @_;
    # xCAT Requires all of these on every arch, and every one of them builds natively on every
    # arch (the noarch deps -- grub2-xcat, xnba-undi -- just repackage committed artifacts), so
    # a self-sufficient per-arch build produces the whole set with no cross-arch import.
    # elilo-xcat is noarch but xCAT hard-requires it (Requires: elilo-xcat >= 3.14-6) on EVERY arch,
    # so a missing elilo makes the whole dep repo uninstallable -- it MUST be required here, not
    # silently tolerated (it builds from a tracked prebuilt on ppc64le/EL8, compiled elsewhere).
    my @req = qw(elilo-xcat ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi
                 perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB);
    push @req, 'xCAT-genesis-base' unless $skip_genesis;
    my @missing = grep { !have_rpm($dir, $_) } @req;
    die "FATAL: required deps missing from $dir: @missing\n" if @missing;
    print "[deps] required set present in $dir: @req\n";
}

sub collect_rpms {
    my (%args) = @_;
    my $roots = $args{roots} // [];
    my $dest  = $args{dest_dir} // die "collect_rpms missing dest_dir\n";
    my $is_dry = $args{dry_run} ? 1 : 0;

    my %seen;
    my $copied = 0;
    my $skipped_src = 0;
    my $missing_roots = 0;

    for my $root (@{$roots}) {
        if (!$root || !-d $root) {
            $missing_roots++;
            print "WARN: missing collection root: $root\n";
            next;
        }
        my @rpms;
        find(
            sub {
                return if !-f $_;
                return if $_ !~ /\.rpm$/;
                push @rpms, $File::Find::name;
            },
            $root,
        );
        @rpms = sort uniq(@rpms);
        for my $rpm (@rpms) {
            next if !-f $rpm;
            if ($rpm =~ /\.src\.rpm$/) {
                $skipped_src++;
                next;
            }
            my $base = basename($rpm);
            next if $seen{$base}++;
            if ($is_dry) {
                print "DRY-RUN copy: $rpm -> $dest/$base\n";
                $copied++;
                next;
            }
            copy($rpm, "$dest/$base")
                or die "Failed to copy $rpm to $dest/$base: $!\n";
            $copied++;
        }
    }

    return ($copied, $skipped_src, $missing_roots);
}

sub collect_srpms {
    my (%args) = @_;
    my $roots = $args{roots} // [];
    my $dest  = $args{dest_dir} // die "collect_srpms missing dest_dir\n";
    my $is_dry = $args{dry_run} ? 1 : 0;

    my %seen;
    my $copied = 0;
    my $skipped_non_src = 0;
    my $missing_roots = 0;

    for my $root (@{$roots}) {
        if (!$root || !-d $root) {
            $missing_roots++;
            print "WARN: missing source collection root: $root\n";
            next;
        }
        my @rpms;
        find(
            sub {
                return if !-f $_;
                return if $_ !~ /\.rpm$/;
                push @rpms, $File::Find::name;
            },
            $root,
        );
        @rpms = sort uniq(@rpms);
        for my $rpm (@rpms) {
            next if !-f $rpm;
            if ($rpm !~ /\.src\.rpm$/) {
                $skipped_non_src++;
                next;
            }
            my $base = basename($rpm);
            next if $seen{$base}++;
            if ($is_dry) {
                print "DRY-RUN copy source: $rpm -> $dest/$base\n";
                $copied++;
                next;
            }
            copy($rpm, "$dest/$base")
                or die "Failed to copy $rpm to $dest/$base: $!\n";
            $copied++;
        }
    }

    return ($copied, $skipped_non_src, $missing_roots);
}

sub resolve_mock_cfg {
    my ($os_id, $rel, $arch) = @_;
    my %short_forms = (
        almalinux      => 'alma',
        'centos-stream' => 'centos-stream',
        rocky          => 'rocky',
    );
    # Resolve by CONFIG-FILE existence, not by running `mock --print-root-path`: the latter can fail
    # transiently (bootstrap chroot setup, a concurrent mock holding a lock) and made el10 flakily
    # "resolve" to the long form that has no .cfg. Checking /etc/mock/<cfg>.cfg is deterministic.
    for my $id ($os_id, (exists $short_forms{$os_id} ? ($short_forms{$os_id}) : ())) {
        my $candidate = "${id}+epel-${rel}-${arch}";
        if (-f "/etc/mock/${candidate}.cfg") {
            print "Mock config resolved: $candidate\n" if $id ne $os_id;
            return $candidate;
        }
    }
    my $short = $short_forms{$os_id} // $os_id;
    die "Could not find mock config for ${os_id}+epel-${rel}-${arch} "
      . "(tried /etc/mock/${os_id}+epel-${rel}-${arch}.cfg and /etc/mock/${short}+epel-${rel}-${arch}.cfg)\n";
}

sub build_mock_uniqueext {
    my ($run, $seq, $label) = @_;

    my $run_part = defined($run) ? $run : 'run';
    $run_part =~ s/[^A-Za-z0-9_.-]+/-/g;
    $run_part =~ s/^-+|-+$//g;
    $run_part = 'run' if $run_part eq '';
    $run_part = substr($run_part, -24) if length($run_part) > 24;

    my $label_part = defined($label) ? $label : 'step';
    $label_part =~ s/[^A-Za-z0-9_.-]+/-/g;
    $label_part =~ s/^-+|-+$//g;
    $label_part = 'step' if $label_part eq '';
    $label_part = substr($label_part, 0, 20) if length($label_part) > 20;

    my $idx = defined($seq) ? int($seq) : 0;
    $idx = 0 if $idx < 0;

    return sprintf("mba-%02d-%s-%s", $idx, $run_part, $label_part);
}

sub resolve_xcat_source {
    my ($requested, $root) = @_;
    # Prefer the sibling ../xcat-core (the real layout: source/xcat-core beside source/xcat-dep)
    # before the legacy xcat-source-code location.
    my @candidates = (
        $requested,
        "$root/../xcat-core",
        "$root/xcat-source-code",
    );
    for my $c (@candidates) {
        next if !defined($c) || $c eq '';
        my $abs = eval { abs_path($c) };
        next if !$abs;
        return $abs if -f "$abs/buildrpms.pl";
    }
    return eval { abs_path($requested) } || $requested;
}

# Fail-fast advisory lock on the output base. Uses an atomic mkdir (portable and reliable over
# NFS, unlike flock) of "<base>/.lock". A second run against the same --output dies immediately
# rather than racing on the shared tree. Only the process that created the lock removes it.
sub acquire_output_lock {
    my ($base, $force) = @_;
    my $lock = "$base/.lock";
    if ($force && -d $lock) {
        print "force-unlock: removing stale lock $lock\n";
        _rmdir_lock($lock);
    }
    if (mkdir $lock) {
        $HELD_LOCK = $lock;
        $LOCK_OWNER_PID = $$;
        my $host = capture('uname -n') || 'unknown';
        if (open my $fh, '>', "$lock/owner") {
            print {$fh} "host=$host\npid=$$\nepoch=" . time() . "\n";
            close $fh;
        }
        return;
    }
    # mkdir failed: either it already exists (locked) or a real error.
    if (-d $lock) {
        my $info = '';
        if (open my $fh, '<', "$lock/owner") { local $/; $info = <$fh>; close $fh; }
        $info =~ s/\s+/ /g;
        die "output $base is locked ($lock): $info\n"
          . "another mockbuild-all run owns it; use a different --output or --force-unlock if stale.\n";
    }
    die "Cannot create lock $lock: $!\n";
}

sub _rmdir_lock {
    my ($lock) = @_;
    unlink "$lock/owner";
    rmdir $lock;
}

# Release the lock on any exit path (normal, die, or signal) -- but ONLY in the process that
# created it. Forked children (per-builder and per-target ForkManager workers) inherit
# $HELD_LOCK; without the pid guard their exit would delete the parent's lock mid-run.
sub _release_lock_if_owner {
    return unless $HELD_LOCK && defined $LOCK_OWNER_PID && $$ == $LOCK_OWNER_PID;
    _rmdir_lock($HELD_LOCK) if -d $HELD_LOCK;
}
END { _release_lock_if_owner(); }
for my $sig (qw(INT TERM HUP)) {
    $SIG{$sig} = sub { _release_lock_if_owner(); exit 1; };
}

sub read_os_release {
    my ($path) = @_;
    my %vals;
    open my $fh, '<', $path or die "Cannot open $path: $!\n";
    while (my $line = <$fh>) {
        chomp $line;
        next if $line =~ /^\s*#/;
        next if $line !~ /=/;
        my ($k, $v) = split /=/, $line, 2;
        $v =~ s/^"(.*)"$/$1/;
        $v =~ s/^'(.*)'$/$1/;
        $vals{$k} = $v;
    }
    close $fh;
    return %vals;
}

sub uniq {
    my %seen;
    return grep { defined($_) && !$seen{$_}++ } @_;
}

sub slurp_chomp {
    my ($path) = @_;
    open my $fh, '<', $path or die "Cannot read $path: $!\n";
    my $line = <$fh>;
    close $fh;
    chomp $line if defined $line;
    return $line // '';
}

sub sh_quote {
    my ($s) = @_;
    $s = '' if !defined $s;
    $s =~ s/'/'"'"'/g;
    return "'$s'";
}