mirror of
https://github.com/xcat2/xcat-dep.git
synced 2026-09-30 23:05:17 +00:00
Vinícius Ferrão
2539 lines
120 KiB
Perl
Executable File
2539 lines
120 KiB
Perl
Executable File
#!/usr/bin/perl
|
|
|
|
use strict;
|
|
use warnings;
|
|
|
|
use Cwd qw(abs_path cwd);
|
|
use File::Basename qw(dirname basename);
|
|
use File::Copy qw(copy);
|
|
use File::Find qw(find);
|
|
use File::Glob qw(bsd_glob);
|
|
use File::Path qw(make_path remove_tree);
|
|
use File::Spec;
|
|
use File::Temp qw(tempdir tempfile);
|
|
use Getopt::Long qw(GetOptions);
|
|
use Parallel::ForkManager;
|
|
use POSIX qw(strftime);
|
|
use JSON::PP;
|
|
use FindBin qw($RealBin);
|
|
use lib $RealBin, "$RealBin/lib";
|
|
use XCAT::NFSLock ();
|
|
use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs rpm_in_cell resolve_mock_cfg
|
|
carry_over_rpms rpm_name rpm_arch rpm_source_rpm rpm_digests_ok
|
|
install_deps_packages install_deps_command missing_perl_modules
|
|
read_manifest derive_target_from_repo_path
|
|
verify_repo_packages verify_repo_signature verify_rpm_signatures
|
|
rpm_version rpm_release rpm_sigmd5 restamp_release_line
|
|
cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix
|
|
build_mock_uniqueext rpmkeys_checksig_problem
|
|
openeuler_build_target openeuler_repo_subdir);
|
|
# print_step and sh_quote come from MockBuildUtils above; XCAT::BuildUtils carries the same
|
|
# print_step, so it is deliberately NOT imported here (one definition, no redefinition warning).
|
|
use XCAT::BuildUtils qw(
|
|
capture_command
|
|
digest_file
|
|
every_step_failed
|
|
forward_signals_to_workers
|
|
block_handled_signals
|
|
restore_signal_mask
|
|
hashes_equal
|
|
read_lines
|
|
emulated_build_timeout
|
|
require_command
|
|
run_bounded
|
|
run_command
|
|
shell_quote
|
|
);
|
|
use XCAT::GenesisRelease qw(
|
|
architectures
|
|
rpm_package_prefix
|
|
rpm_package_name
|
|
validate_repository_packages
|
|
validated_release_checksums
|
|
verify_release_file
|
|
);
|
|
use XCAT::NativeInputs qw(load_inputs stage_inputs verify_input rpm_identity validate_outputs publisher_trust);
|
|
|
|
# --- Mount-namespace isolation: guard the host cgroup against mock teardown propagation ----------
|
|
# mock mounts /sys/fs/cgroup into every build chroot. On these systemd build hosts every mount is
|
|
# `shared`, so the chroot's cgroup joins the HOST's cgroup peer group. When mock tears a chroot down
|
|
# -- its post-build --scrub, or an aborted build's cleanup -- the unmount PROPAGATES back through the
|
|
# shared peer group and unmounts the HOST's /sys/fs/cgroup, after which every later mock (and even new
|
|
# login sessions) dies with "Failed to determine whether the unified cgroups hierarchy is used: No
|
|
# medium found". This bit ppc hardest (it leaks corpse chroot mounts on abort) but x86 shares the same
|
|
# shared-cgroup exposure. Re-exec inside a private mount namespace made rslave
|
|
# (`unshare --mount --propagation slave`): the namespace still sees host mounts (slave = one-way), but
|
|
# nothing mock mounts/unmounts can propagate OUT to the host. As a bonus the namespace tears down every
|
|
# mount mock leaks when we exit, so an aborted build can no longer leave corpse mounts under
|
|
# /var/lib/mock. Best-effort: only as root (needs CAP_SYS_ADMIN) and only if `unshare` exists;
|
|
# otherwise warn loudly and continue unisolated. MOCKBUILD_ALL_MOUNTNS guards against a re-exec loop.
|
|
# Build-free modes (--verify-repo, --finalize-xcat-dep) run no mock and are documented no-root, so they
|
|
# skip the re-exec entirely -- no cgroup exposure, and no spurious non-root warning.
|
|
my $mountns_build_free = grep { /^--(?:verify-repo|finalize-xcat-dep|install-deps)(?:=|$)/ } @ARGV;
|
|
unless ($ENV{MOCKBUILD_ALL_MOUNTNS} || $mountns_build_free) {
|
|
if ($> != 0) {
|
|
warn "WARN: not root -- skipping mount-namespace isolation (host-cgroup propagation guard); "
|
|
. "run as root in CI so mock chroot teardown cannot unmount the host /sys/fs/cgroup\n";
|
|
} elsif (system('sh', '-c', 'command -v unshare >/dev/null 2>&1') != 0) {
|
|
warn "WARN: 'unshare' not found -- skipping mount-namespace isolation; mock chroot teardown "
|
|
. "may unmount the host /sys/fs/cgroup on a shared-propagation host\n";
|
|
} else {
|
|
$ENV{MOCKBUILD_ALL_MOUNTNS} = 1;
|
|
# Absolute path to self (resolved against cwd, which unshare preserves) so the re-exec'd perl
|
|
# finds a relative $0. A bare $PATH-only $0 isn't resolved (abs_path doesn't search $PATH), but
|
|
# a shell invocation yields a full $0 there anyway.
|
|
my $self = abs_path($0) // $0;
|
|
my @reexec = ('unshare', '--mount', '--propagation', 'slave', '--', $^X, $self, @ARGV);
|
|
exec { $reexec[0] } @reexec;
|
|
# exec only returns on failure -- fall through and run unisolated rather than abort the build.
|
|
warn "WARN: exec unshare failed ($!) -- continuing without mount-namespace isolation\n";
|
|
delete $ENV{MOCKBUILD_ALL_MOUNTNS};
|
|
}
|
|
}
|
|
|
|
my $script_dir = abs_path(dirname(__FILE__));
|
|
my $repo_root = abs_path($script_dir);
|
|
my $xcat_src = "$repo_root/../xcat-core";
|
|
# Single knob for all NFS-shared output; --output-root/--repo-dep derive from it below
|
|
# unless explicitly overridden. Empty means "not set on the command line".
|
|
my $output = '';
|
|
my $output_root = '';
|
|
my $target = '';
|
|
my $nproc = 1;
|
|
my $parallel_builds;
|
|
my $parallel_targets = 1; # 1 = serial (default; safe). 0/auto = all EL targets at once; N = cap.
|
|
# NOTE: parallel targets need every per-package mockbuild.pl to avoid
|
|
# shared-path writes (repo tarballs, $HOME/rpmbuild); serial is safe today.
|
|
my $max_parallel = 0; # 0/auto = host nproc: global cap on concurrent mock builds (all targets)
|
|
# Per-build-step wall-clock bound for the dep and perl steps. undef = derived from the target arch
|
|
# (a forcearch target is cross-built through qemu-user); an explicit 0 removes the bound.
|
|
my $build_timeout;
|
|
my $run_id = '';
|
|
my $build_timestamp;
|
|
# CD version bump: when set, every xcat-dep package spec's Release gets a
|
|
# ".snap<YYYYMMDDHHMM>.<build_number>" suffix so each pipeline run publishes a
|
|
# fresh, monotonic NVR (deploy's additive rsync is a no-op otherwise). NOT applied
|
|
# to xCAT-genesis-base (built from xcat-core, kept in lockstep with genesis-scripts).
|
|
my $build_number;
|
|
# Pinned goconserver upstream commit (xcat2/goconserver). goconserver 0.3.3 is unreleased (the
|
|
# newest tag is v0.3.2), so it exists only on master -- pin an immutable SHA instead of the moving
|
|
# branch so the build is reproducible. Bump this deliberately when uptaking a new goconserver.
|
|
my $GOCONSERVER_REF = '6166fe5ec1c5b3c20475e322a9f0e8e93c87e45f';
|
|
my $skip_build = 0;
|
|
my $skip_xcat_dep = 0;
|
|
my $skip_perl = 0;
|
|
my $install_deps = 0;
|
|
my $skip_genesis = 0;
|
|
my $skip_createrepo = 0;
|
|
my $skip_tarball = 0;
|
|
my $genesis_release = '';
|
|
my $genesis_release_checksums;
|
|
my $scrub_all_chroots = 0;
|
|
my $keep_buildroots = 0; # keep per-step mock chroots after build (default: --scrub=chroot each)
|
|
my $dry_run = 0;
|
|
my @extra_collect_dirs;
|
|
my $repo_dep = '';
|
|
my $gpg_sign = 0;
|
|
my $gpg_key_name = 'xCAT Signing Key';
|
|
my $gpg_home = '';
|
|
my $gpg_program = '';
|
|
# Seconds to wait for a lock that a live process holds. 0 fails at once.
|
|
my $try_unlock_timeout = 0;
|
|
# --finalize-xcat-dep: post-build cross-arch genesis provisioning (issue #7610). Takes the two
|
|
# per-arch repo roots and cross-populates the noarch xCAT-genesis-base between them.
|
|
my $finalize_xcat_dep = 0;
|
|
my @finalize_arch;
|
|
my $x86_64_repo = '';
|
|
my $ppc64le_repo = '';
|
|
# --verify-repo=<repo>: standalone, build-free completeness + signature gate over one already-built
|
|
# per-target repo (see verify_target_repo). Empty means "not in standalone verify mode". The target
|
|
# is derived from the repo path (.../rh<N>/<arch> -> alma+epel-<N>-<arch>) or taken from --target.
|
|
my $verify_repo = '';
|
|
# --no-verify-repo suppresses the AUTOMATIC post-build gate deploy_target runs after each target is
|
|
# finalized+signed (for iteration/debug). Verification is ON by default.
|
|
my $no_verify_repo = 0;
|
|
my @HELD_LOCKS;
|
|
my $LOCK_OWNER_PID;
|
|
my ($COMMON_STAGE, $COMMON_DESTINATION, $COMMON_BACKUP);
|
|
my %NATIVE_PLANS;
|
|
for my $sig (qw(INT TERM HUP)) {
|
|
$SIG{$sig} = sub { exit 1; };
|
|
}
|
|
|
|
GetOptions(
|
|
'repo-root=s' => \$repo_root,
|
|
'xcat-source=s' => \$xcat_src,
|
|
'output=s' => \$output,
|
|
'output-root=s' => \$output_root,
|
|
'repo-dep=s' => \$repo_dep,
|
|
'gpg-sign!' => \$gpg_sign,
|
|
'gpg-key-name=s' => \$gpg_key_name,
|
|
'gpg-home=s' => \$gpg_home,
|
|
'try-unlock-timeout=i' => \$try_unlock_timeout,
|
|
'finalize-xcat-dep!' => \$finalize_xcat_dep,
|
|
'finalize-arch=s' => \@finalize_arch,
|
|
'x86_64-repo=s' => \$x86_64_repo,
|
|
'ppc64le-repo=s' => \$ppc64le_repo,
|
|
'verify-repo=s' => \$verify_repo,
|
|
'no-verify-repo!' => \$no_verify_repo,
|
|
'target=s' => \$target,
|
|
'nproc=i' => \$nproc,
|
|
'parallel-builds=i' => \$parallel_builds,
|
|
'parallel-targets=i' => \$parallel_targets,
|
|
'max-parallel=i' => \$max_parallel,
|
|
'build-timeout=i' => \$build_timeout,
|
|
'run-id=s' => \$run_id,
|
|
'build-timestamp=i' => \$build_timestamp,
|
|
'build-number=i' => \$build_number,
|
|
'skip-build!' => \$skip_build,
|
|
'skip-xcat-dep!' => \$skip_xcat_dep,
|
|
'skip-perl!' => \$skip_perl,
|
|
'install-deps!' => \$install_deps,
|
|
'skip-genesis!' => \$skip_genesis,
|
|
'skip-createrepo!' => \$skip_createrepo,
|
|
'skip-tarball!' => \$skip_tarball,
|
|
'genesis-release=s' => \$genesis_release,
|
|
'scrub-all-chroots!' => \$scrub_all_chroots,
|
|
'keep-buildroots!' => \$keep_buildroots,
|
|
'collect-dir=s@' => \@extra_collect_dirs,
|
|
'dry-run!' => \$dry_run,
|
|
) or die usage();
|
|
|
|
die "Run as root (uid=$>)\n" if $> != 0 && !$finalize_xcat_dep && !$verify_repo;
|
|
die "--try-unlock-timeout must be >= 0\n" if $try_unlock_timeout < 0;
|
|
# --skip-build collects a prior build's artifacts from that build's per-target tree, so it must
|
|
# know the target. Without --target the default is "all three EL targets", and each would collect
|
|
# the same artifacts and cross-publish them into every repo (foreign-EL / foreign-arch rpms).
|
|
die "--skip-build requires an explicit --target (collection is per-target)\n"
|
|
if $skip_build && $target eq '';
|
|
die "--parallel-builds must be >= 1\n"
|
|
if defined($parallel_builds) && $parallel_builds < 1;
|
|
|
|
$repo_root = abs_path($repo_root);
|
|
|
|
my $SOURCE_DATE_EPOCH;
|
|
$SOURCE_DATE_EPOCH = $build_timestamp if defined $build_timestamp;
|
|
if (!$SOURCE_DATE_EPOCH && -f "$repo_root/Gitepoch") {
|
|
my @gitepoch = read_lines("$repo_root/Gitepoch");
|
|
$SOURCE_DATE_EPOCH = $gitepoch[0] // '';
|
|
}
|
|
unless ($SOURCE_DATE_EPOCH && $SOURCE_DATE_EPOCH =~ /^\d+$/) {
|
|
$SOURCE_DATE_EPOCH = `git -C \Q$repo_root\E log -1 --format=%ct HEAD 2>/dev/null`;
|
|
chomp $SOURCE_DATE_EPOCH;
|
|
}
|
|
$SOURCE_DATE_EPOCH = time() unless $SOURCE_DATE_EPOCH =~ /^\d+$/;
|
|
$ENV{SOURCE_DATE_EPOCH} = $SOURCE_DATE_EPOCH;
|
|
|
|
if ($run_id eq '') {
|
|
$run_id = strftime('%Y%m%d-%H%M%S', gmtime($SOURCE_DATE_EPOCH));
|
|
}
|
|
|
|
# --verify-repo=<repo>: a distinct, build-free completeness + signature gate over ONE already-built
|
|
# per-target repo. The value is just the repo dir; the manifest comes from the script's existing
|
|
# resolution (repo_root/packages-manifest.conf) and the gpg key/home from --gpg-key-name/--gpg-home.
|
|
# The target is derived from the repo path (.../rh<N>/<arch> -> alma+epel-<N>-<arch>) unless --target
|
|
# is given. Delegates the whole check to verify_target_repo (the SAME gate the auto-run uses), so it
|
|
# exits 0 when complete or dies listing every problem. Runs alone -- no build, no lock, no root.
|
|
if ($verify_repo ne '') {
|
|
require_command('rpm');
|
|
my $rdir = abs_path($verify_repo) or die "--verify-repo repo '$verify_repo' not found\n";
|
|
die "--verify-repo repo '$rdir' is not a directory\n" if !-d $rdir;
|
|
my $tgt = $target ne '' ? $target : derive_target_from_repo_path($rdir);
|
|
die "--verify-repo: cannot derive a target from repo path '$rdir'; pass --target\n"
|
|
if !defined($tgt) || $tgt eq '';
|
|
# sig_required=1: a standalone verify MUST assert the repomd signature (its documented contract),
|
|
# never silently skip it when no gpg key/home is configured (that would be a false PASS on sigs).
|
|
verify_target_repo($rdir, $tgt, undef, 1); # manifest defaults to repo_root/packages-manifest.conf
|
|
exit 0;
|
|
}
|
|
|
|
# --finalize-xcat-dep: a distinct, build-free mode. After BOTH arch build hosts have
|
|
# produced their per-EL repos (each carrying only its own xCAT-genesis-base), the x86_64
|
|
# repo must ALSO ship the noarch xCAT-genesis-base-ppc64 (so an x86_64 MN can netboot ppc
|
|
# nodes) and the ppc64le repo must ship xCAT-genesis-base-x86_64 -- the 2.17 behaviour
|
|
# that issue #7610 regressed. This mode ONLY cross-copies the genesis-base rpm(s) between
|
|
# the two repos (dropping any stale foreign-arch genesis) and re-indexes + re-signs the
|
|
# affected repomd; it builds nothing and holds no output lock.
|
|
if ($finalize_xcat_dep) {
|
|
die "--finalize-xcat-dep requires --x86_64-repo and --ppc64le-repo\n"
|
|
if $x86_64_repo eq '' || $ppc64le_repo eq '';
|
|
require_command('createrepo_c');
|
|
require_command('rpm');
|
|
require_command('rpmsign') if $gpg_sign;
|
|
require_command('gpg') if $gpg_sign;
|
|
my $x86 = abs_path($x86_64_repo) or die "--x86_64-repo '$x86_64_repo' not found\n";
|
|
my $ppc = abs_path($ppc64le_repo) or die "--ppc64le-repo '$ppc64le_repo' not found\n";
|
|
die "--x86_64-repo '$x86' is not a directory\n" if !-d $x86;
|
|
die "--ppc64le-repo '$ppc' is not a directory\n" if !-d $ppc;
|
|
# finalize rewrites the per-arch cells a build deploys, so it takes the same cell locks. With
|
|
# --finalize-arch it writes, and locks, only the cells of those arches.
|
|
@finalize_arch = map { split /[\s,]+/ } @finalize_arch;
|
|
@finalize_arch = qw(x86_64 ppc64le) unless @finalize_arch;
|
|
my %cell;
|
|
for my $root ($x86, $ppc) {
|
|
my @os = grep { -d && basename($_) !~ /^openeuler/ } glob("$root/*");
|
|
for my $os (@os) {
|
|
$cell{ abs_path($_) } = 1 for grep { -d } map { "$os/$_" } @finalize_arch;
|
|
}
|
|
}
|
|
take_lock(cell_lock_path($_), 'repository cell lock') for sort keys %cell;
|
|
# Inject the per-rpm gpg re-sign and the repo re-index as callbacks so the finalize logic in
|
|
# MockBuildUtils stays free of this script's gpg/createrepo state.
|
|
finalize_xcat_dep($x86, $ppc,
|
|
only => \@finalize_arch,
|
|
sign => ($gpg_sign ? sub {
|
|
my ($rpm) = @_;
|
|
local $ENV{GNUPGHOME} = $gpg_home if $gpg_home;
|
|
run_simple("rpmsign --define " . sh_quote("%_gpg_name $gpg_key_name") . " --addsign " . sh_quote($rpm));
|
|
} : undef),
|
|
reindex => \&reindex_and_sign_repo,
|
|
);
|
|
|
|
# finalize just RE-INDEXED + RE-SIGNED each per-EL repo and cross-copied the foreign-arch genesis
|
|
# in -- i.e. it produced the FINAL shipped state, which the per-target gate in deploy_target (run
|
|
# earlier, pre-finalize) never saw. So run the SAME manifest completeness + signature gate here, on
|
|
# every finalized cell, so the build script verifies its own final output by default (no external
|
|
# --verify-repo needed). Suppressible with --no-verify-repo.
|
|
unless ($no_verify_repo) {
|
|
my %seen;
|
|
for my $root ($x86, $ppc) {
|
|
my @cells = map { glob("$root/rh*/$_") } @finalize_arch;
|
|
for my $d (sort @cells) {
|
|
next unless -d $d;
|
|
my $abs = abs_path($d);
|
|
next if $seen{$abs}++;
|
|
my $tgt = derive_target_from_repo_path($abs)
|
|
or die "FATAL: finalize verify -- cannot derive target from '$abs'\n";
|
|
verify_target_repo($abs, $tgt);
|
|
}
|
|
}
|
|
}
|
|
exit 0;
|
|
}
|
|
|
|
# CD version bump. Rewrite every xcat-dep package spec's Release line in this
|
|
# (freshly-checked-out, git-clean) tree so the built rpms carry a fresh, monotonic
|
|
# NVR each run. Runs BEFORE any child builder is invoked. genesis-base lives under
|
|
# xcat-core, not $repo_root, so it is untouched (stays in lockstep with the deployed
|
|
# core's genesis-scripts).
|
|
my $RELEASE_BUMP = '';
|
|
if (defined $build_number) {
|
|
die "--build-number must be a non-negative integer\n" if $build_number < 0;
|
|
$RELEASE_BUMP = strftime('.snap%Y%m%d%H%M', gmtime($SOURCE_DATE_EPOCH)) . ".$build_number";
|
|
# A dry run must not touch the tree. Report what would be stamped and leave the specs alone;
|
|
# $RELEASE_BUMP is still set so the rest of the (no-op) dry-run plan reflects it.
|
|
if ($dry_run) {
|
|
print "[dry-run] would stamp Release suffix '$RELEASE_BUMP' on xcat-dep specs under $repo_root (no files written)\n";
|
|
} else {
|
|
bump_dep_release_suffix($repo_root, $RELEASE_BUMP);
|
|
}
|
|
}
|
|
|
|
# Single output base for every NFS-shared write. Two hosts build in parallel on one NFS by
|
|
# passing distinct --output paths. --output-root/--repo-dep, if given, override the derived
|
|
# values. Default keeps the historical layout so existing callers are unaffected.
|
|
# Create the dir first, THEN abs_path -- abs_path() on a not-yet-existing path returns undef,
|
|
# and abs_path(undef) silently resolves to cwd, which would misdirect all output.
|
|
my $output_base = $output ne '' ? $output : "$repo_root/build-output";
|
|
make_path($output_base) if !-d $output_base;
|
|
$output_base = abs_path($output_base)
|
|
or die "Cannot resolve --output base directory\n";
|
|
$output_root = "$output_base/mockbuild-all" if $output_root eq '';
|
|
# Deployable per-EL xcat-dep repo root (rh8/rh9/rh10/<arch> assembled here).
|
|
$repo_dep = "$output_base/xcat-dep" if $repo_dep eq '';
|
|
make_path($repo_dep) if !-d $repo_dep;
|
|
$repo_dep = abs_path($repo_dep)
|
|
or die "Cannot resolve --repo-dep directory\n";
|
|
|
|
# Locks are taken in one order: the output tree, then each repository cell in name order, then
|
|
# common. The exit handlers release them.
|
|
take_lock("$output_base/.lock", 'output lock');
|
|
MockBuildUtils::recover_common_repository($repo_dep) unless $dry_run;
|
|
|
|
$xcat_src = resolve_xcat_source($xcat_src, $repo_root);
|
|
|
|
my $host_arch = capture_command('uname', '-m');
|
|
# Arch of the target being built: the host arch, except for the forcearch targets below, which
|
|
# cross-build another arch on this host. Set per target in build_one_target; the deploy and
|
|
# the repo metadata take the arch from the target profile instead of this variable.
|
|
my $arch = $host_arch;
|
|
my %os = read_os_release('/etc/os-release');
|
|
my $os_id = $os{ID} // '';
|
|
my $version_id = $os{VERSION_ID} // '';
|
|
my ($rel) = $version_id =~ /^(\d+)/;
|
|
|
|
die "Could not resolve ID from /etc/os-release\n" if $os_id eq '';
|
|
die "Could not resolve major release from VERSION_ID='$version_id' in /etc/os-release\n"
|
|
if !defined($rel) || $rel eq '';
|
|
|
|
# --install-deps: make THIS host able to run the script, then exit. It has to come before the
|
|
# require_command checks below -- those are the very things it installs, and a host that lacks them
|
|
# would die here with no way to fix itself. Run once per build host, as root.
|
|
#
|
|
# The perl modules are re-checked by LOADING them afterwards rather than trusting the package
|
|
# manager: a module that is still missing is exactly the failure this mode exists to prevent, and it
|
|
# aborted CD runs mid-build twice (perl-File-Slurper on xcat-master-ub, perl-IPC-Cmd on
|
|
# xcat-master-ppc), each time as a compile-time error inside XCAT::BuildUtils.
|
|
if ($install_deps) {
|
|
die "--install-deps must run as root (uid=$>)\n" if $> != 0;
|
|
my @cmd = install_deps_command($os_id);
|
|
print_step("Install build prerequisites ($os_id)");
|
|
print " " . join(' ', @cmd) . "\n";
|
|
run_command(@cmd);
|
|
my @modules = qw(File::Slurper IPC::Cmd Parallel::ForkManager Digest::SHA);
|
|
my @missing = missing_perl_modules(@modules);
|
|
die "FATAL: still missing after install: " . join(', ', @missing) . "\n" if @missing;
|
|
print " perl modules present: " . join(', ', @modules) . "\n";
|
|
print " host is ready\n";
|
|
if (lc($os_id) eq 'openeuler') {
|
|
install_mock_cfg(basename($_, '.cfg')) for glob("$repo_root/mock-configs/openeuler-*.cfg");
|
|
}
|
|
exit 0;
|
|
}
|
|
|
|
for my $bin (qw(perl uname createrepo_c tar find rpm)) {
|
|
require_command($bin);
|
|
}
|
|
require_command('mock') if $scrub_all_chroots;
|
|
require_command('rpmsign') if $gpg_sign;
|
|
$gpg_program = require_command('gpg') if $gpg_sign;
|
|
|
|
if ($genesis_release ne '') {
|
|
$genesis_release = abs_path($genesis_release)
|
|
or die "Cannot resolve --genesis-release directory\n";
|
|
die "Genesis release directory not found: $genesis_release\n"
|
|
unless -d $genesis_release;
|
|
my $verifier = "$script_dir/genesis-openembedded/verify-release";
|
|
die "Genesis release verifier not found: $verifier\n" unless -x $verifier;
|
|
common_repository_requirements();
|
|
# Checksum, verify, checksum again. The verifier reads the tree it validates, so a
|
|
# release rewritten together with its SHA256SUMS while the verifier runs would satisfy
|
|
# both the verifier and any single pass taken afterwards; comparing the pass taken
|
|
# before with the one taken after is what closes that window.
|
|
my $checksums_before = validated_release_checksums($genesis_release);
|
|
run_command($^X, $verifier, '--complete', '--format', 'rpm', $genesis_release);
|
|
my $checksums_after = validated_release_checksums($genesis_release);
|
|
die "Genesis release changed during verification\n"
|
|
unless hashes_equal($checksums_before, $checksums_after);
|
|
$genesis_release_checksums = $checksums_before;
|
|
}
|
|
|
|
# An explicit --target builds just that target; otherwise build the current host
|
|
# arch across rh8/rh9/rh10 into a deployable per-EL xcat-dep repo. This script builds
|
|
# ONLY the host arch (uname -m) -- the other arch is produced on its own build host --
|
|
# except for the forcearch targets (%forcearch_targets, --target only), which are
|
|
# cross-built here through qemu-user-static.
|
|
my $native_target = openeuler_build_target(\%os, $host_arch);
|
|
my @build_targets = $target
|
|
? ($target)
|
|
: defined($native_target) ? ($native_target)
|
|
: map { resolve_mock_cfg($os_id, $_, $host_arch) } (8, 9, 10);
|
|
|
|
die "openEuler repository publication requires --gpg-sign\n"
|
|
if !$dry_run && !$gpg_sign && grep { defined(openeuler_repo_subdir($_)) } @build_targets;
|
|
|
|
# What a target builds. The mock-core-configs targets (<os>+epel-<rel>-<arch>) build every
|
|
# dep natively on the host arch. The forcearch targets shipped in mock-configs/ cross-build
|
|
# another arch that has no EPEL: the EPEL-only perl deps of xCAT are built for it
|
|
# (mockbuild-perl-packages.pl --epel-gap), and the noarch deps, the x86 boot loaders among them,
|
|
# are built in the native, EPEL-free chroot of the same release (the rpms are identical for
|
|
# every arch and an emulated build is an order of magnitude slower). See BUILD.md ("riscv64").
|
|
my %forcearch_targets = (
|
|
'rocky-10-riscv64-xcat' => {
|
|
rel => 10,
|
|
arch => 'riscv64',
|
|
# x86_64 only, as the mock config admits: syslinux-xcat builds on x86 and ppc64le alone.
|
|
noarch_cfg => 'rocky-10-x86_64',
|
|
dep_builders => [qw(elilo-xcat grub2-xcat ipmitool-xcat syslinux-xcat goconserver conserver-xcat xnba-undi ipxe-xcat)],
|
|
required => [qw(ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi ipxe-xcat
|
|
perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB)],
|
|
},
|
|
);
|
|
|
|
# Lock each target's repository cell so architecture builds can share --repo-dep.
|
|
my @cell_locks = map {
|
|
my $cell = target_cell($_);
|
|
make_path(dirname($cell));
|
|
cell_lock_path($cell);
|
|
} @build_targets;
|
|
my %cell_lock_seen;
|
|
take_lock($_, 'repository cell lock') for grep { !$cell_lock_seen{$_}++ } sort @cell_locks;
|
|
|
|
# NOTE: no dhcp- packages are built here. DHCP backend selection is an install-time
|
|
# rich dep in xCAT.spec (kea if system-release>=10 else /usr/sbin/dhcpd), so there is
|
|
# nothing arch/EL-specific to build or to exclude for el10.
|
|
|
|
print_step('Targets to build');
|
|
print " $_\n" for @build_targets;
|
|
print "output_base: $output_base\n";
|
|
print "deploy repo-dep: $repo_dep\n";
|
|
print "locks held: $_\n" for map { $_->path } @HELD_LOCKS;
|
|
print "gpg_sign: $gpg_sign\n";
|
|
print "gpg_key_name: $gpg_key_name\n" if $gpg_sign;
|
|
print "gpg_home: " . ($gpg_home ne '' ? $gpg_home : '(default keyring)') . "\n" if $gpg_sign;
|
|
print "genesis_release: " . ($genesis_release || '(legacy builder)') . "\n";
|
|
|
|
# Build (and deploy) EL targets concurrently. Each target is fully isolated -- distinct run_id
|
|
# (target-folded), mock --uniqueext, /tmp work dir, xcat_src/dist/<target>, and deploy dir
|
|
# rh<rel>/<arch> -- so there is no cross-target contention. 0/auto = all at once; 1 = serial.
|
|
my $tgt_workers = $parallel_targets > 0 ? $parallel_targets : scalar(@build_targets);
|
|
$tgt_workers = scalar(@build_targets) if $tgt_workers > scalar(@build_targets);
|
|
# Global cap on concurrent mock builds so parallel targets don't oversubscribe the host. Each
|
|
# mock build already gets a unique --uniqueext (separate chroot), so the only limit needed is
|
|
# hardware: total concurrent builds across all targets stays <= $cap (default host nproc). The
|
|
# per-target build-step concurrency is therefore the cap divided across the active targets.
|
|
my $cap = $max_parallel > 0 ? $max_parallel : (capture_command('nproc') || 4);
|
|
my $per_target_builds = defined($parallel_builds) ? $parallel_builds : int($cap / $tgt_workers);
|
|
$per_target_builds = 1 if $per_target_builds < 1;
|
|
print "parallel_targets: " . ($parallel_targets > 0 ? $parallel_targets : "auto($tgt_workers)") . "\n";
|
|
print "max_parallel: $cap (per-target build workers: $per_target_builds)\n";
|
|
my $tgt_pm = Parallel::ForkManager->new($tgt_workers <= 1 ? 0 : $tgt_workers);
|
|
my $tgt_fail = 0;
|
|
# A signal to this process must reach the forked workers. Without forwarding, the orchestrator exits
|
|
# and releases its locks while the workers keep building and writing into the deploy tree.
|
|
my %tgt_kids;
|
|
$tgt_pm->run_on_start(sub { $tgt_kids{ $_[0] } = 1 });
|
|
$tgt_pm->run_on_finish(sub {
|
|
my ($pid, $exit, $ident, $signal, $core_dump) = @_;
|
|
delete $tgt_kids{$pid};
|
|
# A worker killed by a signal exits with code 0 in this callback, so reading the code alone
|
|
# reports a target that died mid-deploy as built.
|
|
$tgt_fail++ if $exit or $signal or $core_dump;
|
|
});
|
|
my $tgt_forward = forward_signals_to_workers(
|
|
pids => \%tgt_kids,
|
|
reap => sub { $tgt_pm->wait_all_children },
|
|
);
|
|
local $SIG{INT} = $tgt_forward;
|
|
local $SIG{TERM} = $tgt_forward;
|
|
local $SIG{HUP} = $tgt_forward;
|
|
for my $tgt (@build_targets) {
|
|
# ForkManager records the worker in run_on_start, which runs AFTER the fork, so the handler
|
|
# cannot signal a worker that arrives in between. Hold the signals across both -- but wait for
|
|
# a free slot FIRST, with them unblocked, or a cancellation would sit pending for as long as
|
|
# the pool stays full.
|
|
# max_procs 0 is ForkManager's no-fork mode, where asking for a slot is an error.
|
|
$tgt_pm->wait_for_available_procs(1) if $tgt_pm->max_procs;
|
|
my $orchestrator = $$;
|
|
my $previous = block_handled_signals();
|
|
if ($tgt_pm->start) { restore_signal_mask($previous); next; }
|
|
# With one worker ForkManager does not fork at all, and this is still the orchestrator: only a
|
|
# real child drops the forwarder, whose copy names siblings the parent already signals.
|
|
$SIG{$_} = 'DEFAULT' for ($$ == $orchestrator ? () : qw(INT TERM HUP));
|
|
restore_signal_mask($previous);
|
|
my $rc = 0;
|
|
eval {
|
|
my $info = build_one_target($tgt, $run_id, $per_target_builds);
|
|
deploy_target($tgt, $info);
|
|
1;
|
|
} or do { warn "ERROR: target $tgt failed: $@"; $rc = 1; };
|
|
$tgt_pm->finish($rc);
|
|
}
|
|
$tgt_pm->wait_all_children;
|
|
die "FATAL: $tgt_fail target(s) failed\n" if $tgt_fail;
|
|
|
|
publish_genesis_common_repo() if $genesis_release;
|
|
|
|
print_step('All targets completed');
|
|
exit 0;
|
|
|
|
# Build a single target into its own build-output/<target-runid> tree and return
|
|
# { repo_dir, rel }. Everything below through the summary is per-target work.
|
|
sub build_one_target {
|
|
my ($target, $run_id, $max_build_workers) = @_;
|
|
# The build output identity must be per-target (os version + arch). SOURCE_DATE_EPOCH
|
|
# is the same across targets for a given commit, so a timestamp-only run_id makes
|
|
# different targets (e.g. alma+epel-8 vs -9) share build-output/<run_id> and
|
|
# cross-contaminate. Fold the target into run_id so each target gets its own tree.
|
|
$run_id = "$target-$run_id" unless index($run_id, $target) >= 0;
|
|
my $profile = target_profile($target);
|
|
my $rel = $profile->{rel};
|
|
$arch = $profile->{arch};
|
|
|
|
# Per-target required set from packages-manifest.conf: build ONLY these packages, and fail the
|
|
# run if any of them fails. A package absent from this target's section is not built for it.
|
|
my %MANIFEST = read_manifest("$repo_root/packages-manifest.conf");
|
|
my %req = %{ $MANIFEST{$target} // {} };
|
|
die "FATAL: no manifest section for target '$target' in packages-manifest.conf\n"
|
|
if !%req;
|
|
my $native = $target eq 'openeuler-24.03-ppc64le' ? load_inputs($repo_root, \%req) : undef;
|
|
$NATIVE_PLANS{$target} = $native if $native;
|
|
if ($native) {
|
|
die "Native POWER collection requires signing and verification\n"
|
|
if !$gpg_sign || $no_verify_repo;
|
|
die "Native POWER inputs require a complete owner run\n"
|
|
if $skip_build || $skip_xcat_dep || $skip_perl || @extra_collect_dirs;
|
|
}
|
|
|
|
my $run_root = "$output_root/$run_id";
|
|
my $build_root = "$run_root/build-results";
|
|
my $log_root = "$run_root/build-logs";
|
|
my $repo_dir = "$run_root/repo/$arch";
|
|
my $summary_file = "$run_root/summary.txt";
|
|
my $tarball = "$output_root/mockbuild-all-$target-$run_id.tar.gz";
|
|
my $srpm_repo_dir = "$run_root/repo-src";
|
|
my $srpm_tarball = "$output_root/mockbuild-all-$target-$run_id-srpm.tar.gz";
|
|
|
|
# Each real build must start from a clean per-target tree. run_id is derived from the deterministic
|
|
# commit timestamp, so re-runs of the same commit resolve to the SAME $run_root -- without a wipe, a
|
|
# stale rpm or a stale perl status.txt from an earlier (possibly failed) run could be reused and mask
|
|
# a failure (see mockbuild-perl-packages.pl, which reads per-package status files back). --skip-build
|
|
# deliberately KEEPS the tree (it collects a prior build's artifacts); --dry-run writes nothing.
|
|
if (!$skip_build && !$dry_run && -d $run_root) {
|
|
print "Cleaning stale per-target tree before build: $run_root\n";
|
|
remove_tree($run_root);
|
|
}
|
|
|
|
# All dep builders run natively on every arch. xnba-undi, grub2-xcat and ipxe-xcat are noarch
|
|
# packagings of committed artifacts (an x86 UNDI ROM / the grub2 resource tarball / the iPXE release
|
|
# tree) with no arch-specific build step, so ppc builds them the same as x86 -- no cross-arch
|
|
# import. A forcearch target builds only the builders its profile lists; the noarch ones run in
|
|
# the profile's native chroot.
|
|
# syslinux-xcat is noarch too, and its spec builds on x86 and ppc64le only.
|
|
my @dep_builders = (
|
|
{ name => 'elilo-xcat', script => "$repo_root/elilo/mockbuild.pl", noarch => 1 },
|
|
{ name => 'grub2-xcat', script => "$repo_root/grub2-xcat/mockbuild.pl", noarch => 1 },
|
|
{ name => 'ipmitool-xcat', script => "$repo_root/ipmitool/mockbuild.pl" },
|
|
{ name => 'syslinux-xcat', script => "$repo_root/syslinux/mockbuild.pl", noarch => 1 },
|
|
{ name => 'goconserver', script => "$repo_root/goconserver/mockbuild.pl" },
|
|
{ name => 'conserver-xcat', script => "$repo_root/conserver/mockbuild.pl" },
|
|
{ name => 'xnba-undi', script => "$repo_root/xnba/mockbuild.pl", noarch => 1 },
|
|
{ name => 'python3-scp', srpm => "$repo_root/python-scp/python-scp-0.14.5-1.oe2403.src.rpm",
|
|
sha256 => '3461d2a3fe0122cac2893d8465ad1271ae21e5570a31d4402e3f887ef545a0e8' },
|
|
{ name => 'ipxe-xcat', script => "$repo_root/ipxe-xcat/mockbuild.pl", noarch => 1 },
|
|
);
|
|
my %profile_builds = map { $_ => 1 } @{ $profile->{dep_builders} };
|
|
|
|
my $perl_builder = "$repo_root/mockbuild-perl-packages.pl";
|
|
|
|
# buildrpms.pl (in xcat-core) is only needed for the OS-dependent xCAT-genesis-base
|
|
# build below; the full xCAT core is built separately by the xcat-core pipeline.
|
|
die "Missing xCAT build script: $xcat_src/buildrpms.pl\n"
|
|
if !$skip_genesis && !-f "$xcat_src/buildrpms.pl";
|
|
|
|
my @active_dep_builders;
|
|
for my $b (@dep_builders) {
|
|
next if !$profile_builds{$b->{name}};
|
|
if ($b->{srpm}) {
|
|
push @active_dep_builders, $b if $req{$b->{name}};
|
|
next;
|
|
}
|
|
if (-f $b->{script}) {
|
|
push @active_dep_builders, $b;
|
|
next;
|
|
}
|
|
print "WARN: missing dep builder script, skipping: $b->{script}\n";
|
|
die "Missing native owner script: $b->{script}\n" if $native && $req{$b->{name}};
|
|
}
|
|
die "Missing perl builder script: $perl_builder\n"
|
|
if !$skip_perl && !$perl_builder;
|
|
|
|
if (!$dry_run) {
|
|
make_path($build_root, $log_root, $repo_dir, $srpm_repo_dir);
|
|
# The staging repositories hold what THIS invocation produces. A reused --run-id, or a
|
|
# rerun after a failed invocation, otherwise leaves an earlier run's packages in them,
|
|
# where collection never sees them, createrepo indexes them and deploy_target publishes
|
|
# them as this run's output.
|
|
reset_staging_repo($repo_dir);
|
|
reset_staging_repo($srpm_repo_dir);
|
|
# Same for the builder results when this run is going to build them: a reused --run-id
|
|
# otherwise leaves a previous run's packages there, and a step that fails this time is
|
|
# collected from the last time it succeeded. --skip-build deliberately collects earlier
|
|
# output, from the repository-level build-output tree, and must keep what is there.
|
|
if (!$skip_build) {
|
|
remove_tree($build_root);
|
|
make_path($build_root);
|
|
}
|
|
}
|
|
|
|
print_step("Configuration");
|
|
print "repo_root: $repo_root\n";
|
|
print "xcat_source: $xcat_src\n";
|
|
print "output_root: $output_root\n";
|
|
print "run_root: $run_root\n";
|
|
print "arch: $arch\n";
|
|
print "host_arch: $host_arch\n";
|
|
print "forcearch: $profile->{forcearch}\n";
|
|
print "noarch_cfg: $profile->{noarch_cfg}\n";
|
|
print "epel: $profile->{epel}\n";
|
|
print "os_id: $os_id\n";
|
|
print "version_id: $version_id\n";
|
|
print "rel: $rel\n";
|
|
print "target: $target\n";
|
|
print "nproc: $nproc\n";
|
|
print "parallel_builds: " . (defined($parallel_builds) ? $parallel_builds : 'auto') . "\n";
|
|
print "skip_build: $skip_build\n";
|
|
print "skip_xcat_dep: $skip_xcat_dep\n";
|
|
print "skip_perl: $skip_perl\n";
|
|
print "skip_genesis: $skip_genesis\n";
|
|
print "skip_createrepo: $skip_createrepo\n";
|
|
print "skip_tarball: $skip_tarball\n";
|
|
print "scrub_all_chroots:$scrub_all_chroots\n";
|
|
print "keep_buildroots: $keep_buildroots\n";
|
|
print "dry_run: $dry_run\n";
|
|
print "perl_builder: $perl_builder\n";
|
|
print "tarball: $tarball\n";
|
|
print "srpm_repo_dir: $srpm_repo_dir\n";
|
|
print "srpm_tarball: $srpm_tarball\n";
|
|
|
|
my @collect_roots;
|
|
my @native_source_roots;
|
|
|
|
if ($native && !$dry_run) {
|
|
stage_inputs($native, "$run_root/native-inputs");
|
|
}
|
|
|
|
if (!$skip_build && !$skip_xcat_dep) {
|
|
for my $builder (grep { $_->{srpm} } @active_dep_builders) {
|
|
my $source = $builder->{srpm};
|
|
die "Missing source RPM: $source\n" unless -f $source;
|
|
die "Source RPM SHA256 mismatch: $source\n" unless digest_file($source) eq $builder->{sha256};
|
|
next if $dry_run;
|
|
my $stage_dir = "$run_root/source-rpms/$builder->{name}";
|
|
make_path($stage_dir);
|
|
my $staged = "$stage_dir/" . basename($source);
|
|
copy($source, $staged) or die "Cannot stage source RPM $source: $!\n";
|
|
die "Staged source RPM SHA256 mismatch: $staged\n" unless digest_file($staged) eq $builder->{sha256};
|
|
$builder->{srpm} = $staged;
|
|
}
|
|
}
|
|
|
|
install_mock_cfg($target);
|
|
|
|
if ($native) {
|
|
my ($runtime, $sources) = build_native_inputs($native, $target, \%req, $run_root, $log_root);
|
|
push @collect_roots, $runtime;
|
|
push @native_source_roots, @$sources;
|
|
}
|
|
|
|
if ($scrub_all_chroots) {
|
|
run_step(
|
|
step => "Scrub all chroots for target $target",
|
|
cmd => "mock -r " . shell_quote($target) . " --scrub=all",
|
|
log => "$log_root/scrub-all-chroots.log",
|
|
);
|
|
run_step(
|
|
step => "Scrub all chroots for noarch config $profile->{noarch_cfg}",
|
|
cmd => "mock -r " . shell_quote($profile->{noarch_cfg}) . " --scrub=all",
|
|
log => "$log_root/scrub-all-chroots-noarch.log",
|
|
) if $profile->{noarch_cfg} ne $target;
|
|
}
|
|
|
|
if (!$skip_build) {
|
|
my @build_steps;
|
|
my $build_step_seq = 0;
|
|
|
|
# Bound only what can run emulated. A forcearch target (rocky-10-riscv64-xcat) cross-builds every
|
|
# dep through qemu-user, where a deadlock burns no CPU and never returns. Native mock steps keep
|
|
# their present, unbounded behaviour: no measurement of them exists here, and a bound guessed for
|
|
# a step that legitimately runs long would turn a trusted cell red for no reason.
|
|
# --build-timeout overrides both; 0 removes the bound.
|
|
my $step_timeout = defined $build_timeout ? $build_timeout
|
|
: $profile->{forcearch} ? emulated_build_timeout($arch, $host_arch)
|
|
: 0;
|
|
|
|
if (!$skip_xcat_dep) {
|
|
for my $builder (@active_dep_builders) {
|
|
next unless $req{ $builder->{name} }; # manifest: build only required dep packages
|
|
my $name = $builder->{name};
|
|
my $script = $builder->{script};
|
|
my $step_result = "$build_root/$name";
|
|
my $step_log = "$log_root/$name";
|
|
my $step_uniqueext = build_mock_uniqueext($run_id, ++$build_step_seq, $name);
|
|
my $mock_cfg = $builder->{noarch} ? $profile->{noarch_cfg} : $target;
|
|
my $cmd = $builder->{srpm}
|
|
? source_rpm_build_command($builder, $target, $step_uniqueext, $step_result, $step_log,
|
|
"$run_root/source-rpms/$name")
|
|
: join(' ',
|
|
'perl', shell_quote($script),
|
|
'--mock-cfg', shell_quote($mock_cfg),
|
|
($profile->{forcearch} && !$builder->{noarch} ? ('--target-arch', shell_quote($arch)) : ()),
|
|
'--mock-uniqueext', shell_quote($step_uniqueext),
|
|
'--result-dir', shell_quote($step_result),
|
|
'--log-dir', shell_quote($step_log),
|
|
# host-local, run-scoped work dir so /tmp doesn't collide between runs
|
|
'--work-dir', shell_quote("/tmp/mockbuild-all-$run_id/$name"),
|
|
'--build-timestamp', $SOURCE_DATE_EPOCH,
|
|
($name eq 'goconserver' ? ('--go-ref', sh_quote($GOCONSERVER_REF)) : ()),
|
|
# Generated specs need the same release suffix as in-tree specs.
|
|
(($name eq 'goconserver' || $name eq 'xnba-undi') && $RELEASE_BUMP ne ''
|
|
? ('--release-suffix', sh_quote($RELEASE_BUMP))
|
|
: ()),
|
|
);
|
|
push @build_steps, {
|
|
id => "xcat-dep:$name",
|
|
step => "Build xcat-dep: $name",
|
|
cmd => $cmd,
|
|
timeout => $step_timeout,
|
|
log => "$log_root/$name/run.log",
|
|
scrub_cfg => $mock_cfg,
|
|
scrub_uniqueext => $step_uniqueext,
|
|
($native ? (native_results => {$name => $step_result}) : ()),
|
|
};
|
|
push @collect_roots, $step_result;
|
|
}
|
|
}
|
|
|
|
my @perl_pkgs = sort grep { /^perl-/ && (!$native || $native->{nodes}{$_}{type} eq 'owner') } keys %req;
|
|
if (!$skip_perl && @perl_pkgs) {
|
|
my $perl_result = "$build_root/perl/$arch";
|
|
my $perl_log = "$log_root/perl/$arch";
|
|
my $perl_uniqueext = build_mock_uniqueext($run_id, ++$build_step_seq, 'perl-list6');
|
|
# Bound the perl builder's OWN internal parallelism to this target's budget; otherwise it
|
|
# forks one mock build per perl package (~7), which -- multiplied by parallel EL targets --
|
|
# oversubscribes the host.
|
|
my $cmd = join(' ',
|
|
'perl', sh_quote($perl_builder),
|
|
'--mock-cfg', sh_quote($target),
|
|
($profile->{forcearch}
|
|
? ('--target-arch', sh_quote($arch), '--noarch-mock-cfg', sh_quote($profile->{noarch_cfg}))
|
|
: ()),
|
|
($profile->{epel} ? () : ('--epel-gap')),
|
|
'--mock-uniqueext', sh_quote($perl_uniqueext),
|
|
'--result-dir', sh_quote($perl_result),
|
|
'--log-dir', sh_quote($perl_log),
|
|
'--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/perl-list6"),
|
|
'--packages', sh_quote(join(',', @perl_pkgs)), # manifest: only required perl pkgs
|
|
(($max_build_workers && $max_build_workers >= 1) ? ('--jobs', $max_build_workers) : ()),
|
|
'--build-timestamp', $SOURCE_DATE_EPOCH,
|
|
# CD bump: the in-tree spec Release bump above only reaches the spec-mode perl
|
|
# packages; the srpm-mode ones (HTML-Form, IO-Stty, Net-Telnet) build from a
|
|
# committed .src.rpm, so hand the suffix down for the builder to re-stamp them.
|
|
($RELEASE_BUMP ne '' ? ('--release-suffix', sh_quote($RELEASE_BUMP)) : ()),
|
|
($keep_buildroots ? '--keep-buildroots' : ()),
|
|
);
|
|
push @build_steps, {
|
|
id => 'perl',
|
|
step => 'Build perl xcat-dep packages',
|
|
cmd => $cmd,
|
|
# The perl builder forks its own mock jobs, so under forcearch it is emulated too. Give it
|
|
# the per-package budget times the number of packages it builds serially per worker.
|
|
timeout => ($step_timeout ? $step_timeout * scalar(@perl_pkgs) : 0),
|
|
log => "$log_root/perl-build.log",
|
|
($native ? (native_results => {map { $_ => "$perl_result/$_" } @perl_pkgs}) : ()),
|
|
};
|
|
push @collect_roots, $perl_result;
|
|
}
|
|
|
|
# NOTE: this script builds ONLY xcat-dep (its dep packages, the perl packages, and
|
|
# the OS-dependent xCAT-genesis-base below). The full xCAT core is built separately
|
|
# by the xcat-core pipeline -- mockbuild-all no longer has a monolithic core-build path.
|
|
|
|
# xCAT-genesis-base is OS-dependent (its initramfs bundles the build chroot's
|
|
# kernel + glibc/busybox/perl), so it is built here, per target, and shipped
|
|
# in this per-EL xcat-dep repo rather than in the flat xcat-core. buildrpms.pl
|
|
# (run in the xcat-core dir) derives the same snapYYYYMMDDHHMM Release from
|
|
# xcat-core's Gitepoch, so it matches xCAT-genesis-scripts (built in core) and
|
|
# the exact-version dependency genesis-scripts -> genesis-base resolves.
|
|
if (!$skip_genesis && $req{'xCAT-genesis-base'}) {
|
|
# buildrpms.pl stages sources in $HOME/rpmbuild (via rpmdev-setuptree). Give each
|
|
# per-target genesis build its own HOME so parallel EL targets don't race on the shared
|
|
# /root/rpmbuild tree (that race is what made concurrent genesis builds fail).
|
|
my $genesis_home = "/tmp/mockbuild-all-$run_id/genesis-home";
|
|
# buildrpms.pl's rpmdev-setuptree only runs during env setup, not per build, so create the
|
|
# rpmbuild tree ourselves for this per-target HOME (else $HOME/rpmbuild/SOURCES is missing).
|
|
my $mktree = join(' ', map { sh_quote("$genesis_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS));
|
|
# The genesis chroot (xCAT-genesis-base-<target>) is SHARED across runs -- buildrpms.pl builds it
|
|
# without a per-run --mock-uniqueext. mock's post-build scrub only runs on SUCCESS, so a killed
|
|
# or failFast-interrupted prior run leaves the chroot stunted (missing /bin/sh), and mock REUSES
|
|
# the corpse on the next run -> "FileNotFoundError: '/bin/sh'". Scrub it FIRST (best-effort: a
|
|
# no-op on the first run before the config exists) so mock recreates the chroot from the cached
|
|
# root; --scrub=bootstrap goes too (the genesis bootstrap chroot is part of the leak; genesis
|
|
# carries no --mock-uniqueext), so the fresh chroot re-bootstraps from the root cache. mock takes the buildroot lock for --scrub and BLOCKS (not
|
|
# skips) if a concurrent build holds it, but within a run the scrub is sequential before the
|
|
# build and the CD topology never runs a second same-target build at once; `timeout` bounds even
|
|
# a pathological wait so a stale lock can never hang the build.
|
|
my $genesis_scrub = "{ timeout 300 mock -r " . sh_quote("xCAT-genesis-base-$target")
|
|
. " --scrub=chroot --scrub=bootstrap >/dev/null 2>&1 || true; }";
|
|
my $cmd = "mkdir -p $mktree && $genesis_scrub && HOME=" . sh_quote($genesis_home) . ' ' . join(' ',
|
|
'perl', sh_quote("$xcat_src/buildrpms.pl"),
|
|
'--package', 'xCAT-genesis-base',
|
|
'--target', shell_quote($target),
|
|
'--nproc', int($nproc),
|
|
'--force',
|
|
'--verbose',
|
|
'--xcat_dep_path', shell_quote($repo_root),
|
|
(defined(openeuler_repo_subdir($target)) && $gpg_sign
|
|
? ('--gpg-sign', '--gpg-key-name', shell_quote($gpg_key_name),
|
|
'--gpg-home', shell_quote(File::Spec->rel2abs($gpg_home ne '' ? $gpg_home
|
|
: $ENV{GNUPGHOME} || "$ENV{HOME}/.gnupg"))) : ()),
|
|
);
|
|
$cmd = native_owner_command($native, $target, $cmd, 0) if $native;
|
|
push @build_steps, {
|
|
id => 'genesis',
|
|
step => 'Build xCAT-genesis-base (per-target, OS-dependent)',
|
|
cmd => $cmd,
|
|
cwd => $xcat_src,
|
|
log => "$log_root/genesis-build.log",
|
|
scrub_cfg => "xCAT-genesis-base-$target",
|
|
($native ? (native_results => {'xCAT-genesis-base' => "$xcat_src/dist/$target/rpms"}) : ()),
|
|
};
|
|
}
|
|
|
|
if (@build_steps) {
|
|
if ($native) {
|
|
for my $step (grep { $_->{id} ne 'genesis' } @build_steps) {
|
|
$step->{cmd} = native_owner_command($native, $target, $step->{cmd}, 1000);
|
|
}
|
|
}
|
|
# Prefer the caller-supplied cap (global budget / active targets). Fall back to the old
|
|
# behaviour (all steps at once) only when unset.
|
|
my $effective_parallel_builds =
|
|
($max_build_workers && $max_build_workers >= 1) ? $max_build_workers
|
|
: defined($parallel_builds) ? $parallel_builds
|
|
: scalar(@build_steps);
|
|
# Make --max-parallel a REAL cap. The perl builder is a single step that internally forks up
|
|
# to $effective_parallel_builds mock jobs of its own, so running it concurrently with the dep
|
|
# builders pushed live mock builds to ~2x the cap. Run it in its OWN phase, after the dep
|
|
# builders (which are quick) -- each phase then runs at most $effective_parallel_builds mock
|
|
# builds, so the cap holds, at a small bounded wall-clock cost. (The perl step sets no
|
|
# scrub_cfg and scrubs its own chroots; the scrub loop below still covers the dep/genesis steps.)
|
|
my @perl_steps = grep { $_->{id} eq 'perl' } @build_steps;
|
|
my @nonperl_steps = grep { $_->{id} ne 'perl' } @build_steps;
|
|
my @failed;
|
|
push @failed, run_build_steps_parallel(
|
|
steps => \@nonperl_steps, max_processes => $effective_parallel_builds,
|
|
) if @nonperl_steps;
|
|
push @failed, run_build_steps_parallel(
|
|
steps => \@perl_steps, max_processes => $effective_parallel_builds,
|
|
) if @perl_steps;
|
|
|
|
# Reclaim each build step's mock chroot now that the step copied its RPMs/logs out to
|
|
# its --result-dir (collect_rpms reads those, never /var/lib/mock). mock's own cleanup
|
|
# leaves these chroots behind -- and keeps them entirely on failure -- so /var/lib/mock
|
|
# grows ~15-17G per run until the host fills and every dnf transaction fails for lack of
|
|
# space. Scrub each via `mock --scrub=chroot --scrub=bootstrap` (never rm): it takes the
|
|
# chroot lock, so a chroot still used by a concurrent build is refused and safely skipped.
|
|
# Both the build chroot and its per-uniqueext bootstrap are removed; the root cache stays
|
|
# for fast rebuilds. Perl packages are scrubbed inside mockbuild-perl-packages.pl (it
|
|
# derives its own per-package uniqueexts).
|
|
unless ($keep_buildroots) {
|
|
for my $s (@build_steps) {
|
|
next unless defined $s->{scrub_cfg};
|
|
(my $slug = $s->{id}) =~ s/[^\w.-]+/-/g;
|
|
scrub_buildroot($s->{scrub_cfg}, $s->{scrub_uniqueext}, "$log_root/scrub-$slug.log");
|
|
}
|
|
}
|
|
|
|
# Zero-tolerance: any build step that failed fails the whole run -- genesis included.
|
|
# (xcat-core #7696 is merged: buildrpms.pl now exits 0 iff it actually produced the
|
|
# genesis rpm, so there is no cosmetic non-zero exit left to tolerate. The old workaround
|
|
# -- ignore a genesis failure when a matching rpm already exists in dist/ -- is gone; a
|
|
# stale artifact from a previous build must never mask a failed genesis build.)
|
|
die "FATAL: required build step(s) failed for $target: @failed\n" if @failed;
|
|
if ($native && !$dry_run) {
|
|
for my $step (@build_steps) {
|
|
for my $name (sort keys %{$step->{native_results} // {}}) {
|
|
my $directory = $step->{native_results}{$name};
|
|
die "Missing native owner result: $directory\n" unless -d $directory;
|
|
my @rpms;
|
|
find({no_chdir => 1, wanted => sub {
|
|
push @rpms, $File::Find::name if -f $_ && /\.rpm\z/ && !/\.src\.rpm\z/
|
|
&& ($name ne 'xCAT-genesis-base' || basename($_) =~ /^xCAT-genesis-base-/);
|
|
}}, $directory);
|
|
validate_outputs($native->{nodes}{$name}, \@rpms, 1);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# The xCAT core is built by the xcat-core pipeline, NOT here -- so we deliberately do
|
|
# NOT collect the xCAT dist tree. Only the OS-dependent xCAT-genesis-base rpm (built by
|
|
# the genesis step above) is pulled out of it, individually, further below.
|
|
my $xcat_rpms_dir = "$xcat_src/dist/$target/rpms";
|
|
|
|
if ($skip_build) {
|
|
# Collect THIS target's previously-built artifacts from its own per-target build tree -- the
|
|
# same $build_root a normal build populates (collect_rpms recurses). NOT the legacy EL-agnostic
|
|
# build-output/list* dirs: those are scoped only by $arch, so an el8 rpm left there would be
|
|
# pulled into an el9/el10 repo, and with --target omitted the same rpms would be published into
|
|
# every EL repo. (--target is now required for --skip-build, see the option check above.)
|
|
push @collect_roots, $build_root;
|
|
}
|
|
|
|
push @collect_roots, @extra_collect_dirs;
|
|
@collect_roots = uniq(@collect_roots);
|
|
my @srpm_collect_roots = uniq(@collect_roots);
|
|
push @srpm_collect_roots, @native_source_roots;
|
|
|
|
if ($genesis_release && !$dry_run) {
|
|
remove_genesis_packages($repo_dir, 0);
|
|
remove_genesis_packages($srpm_repo_dir, 1);
|
|
}
|
|
|
|
print_step('Collect RPM artifacts');
|
|
print "collection roots:\n";
|
|
print " $_\n" for @collect_roots;
|
|
|
|
my ($copied, $skipped_src, $missing_roots, $skipped_foreign) = collect_rpms(
|
|
roots => \@collect_roots,
|
|
dest_dir => $repo_dir,
|
|
arch => $arch,
|
|
dry_run => $dry_run,
|
|
);
|
|
print "skipped $skipped_foreign rpm(s) of another architecture\n" if $skipped_foreign;
|
|
|
|
# Assert on what this run BUILT, before the Genesis release is added: the release is
|
|
# installed from a verified directory rather than built here, so counting it first would
|
|
# let a run whose builders all failed reach createrepo and the deployable tree, and fail
|
|
# much later in the repo gate (verify_target_repo), naming missing packages instead of the
|
|
# failed builds.
|
|
if (!$dry_run && $copied == 0 && @collect_roots) {
|
|
die "No binary RPMs were collected. Check build logs and collection roots.\n";
|
|
}
|
|
|
|
# Ensure the OS-dependent xCAT-genesis-base rpm (built by the genesis step above)
|
|
# lands in the dep repo -- pull it individually out of the xcat-core dist tree (the
|
|
# rest of that tree, the full xCAT core, is built + published by the xcat-core pipeline).
|
|
if (!$skip_genesis && !$dry_run) {
|
|
for my $g (bsd_glob("$xcat_rpms_dir/xCAT-genesis-base-*.rpm")) {
|
|
next if $g =~ /\.src\.rpm$/;
|
|
copy($g, "$repo_dir/" . basename($g))
|
|
or die "Failed to copy genesis-base $g -> $repo_dir: $!\n";
|
|
$copied++;
|
|
}
|
|
}
|
|
|
|
# A skipped builder built nothing this run, so everything it published in the cell joins the run
|
|
# repository here, ahead of the bump check, createrepo, the tarballs and the deploy gate.
|
|
if (!$dry_run && ($skip_genesis || $skip_perl || $skip_xcat_dep)) {
|
|
my $published = target_cell($target);
|
|
if (-d $published) {
|
|
my %skipped = (genesis => $skip_genesis, perl => $skip_perl, dep => $skip_xcat_dep);
|
|
# Only an rpm the configured key signed, by signer id and by rpmkeys --checksig, may be
|
|
# re-signed and republished; an unsigned run still requires the digests to verify.
|
|
my $trusted = \&rpm_digests_ok;
|
|
if ($gpg_sign || $gpg_home ne '') {
|
|
my ($dbopt, $problem) = rpmkeys_keyring($gpg_key_name, $gpg_home);
|
|
die "FATAL: $problem\n" if $problem;
|
|
my $accept = gpg_key_ids($gpg_key_name, $gpg_home);
|
|
$trusted = sub {
|
|
my $id = rpm_signer_keyid($_[0]);
|
|
return (defined $id && $accept->{$id} && !rpm_checksig_problem($_[0], $dbopt)) ? 1 : 0;
|
|
};
|
|
}
|
|
for my $base (carry_over_rpms($published, $repo_dir, \%skipped, [sort keys %req],
|
|
\&rpm_name, \&rpm_source_rpm, $trusted, $arch, \&rpm_arch)) {
|
|
print "[collect] $base kept from the published cell $published\n";
|
|
}
|
|
}
|
|
}
|
|
|
|
# Repo completeness -- every required package present at its pinned version (a '*' pin accepts any),
|
|
# missing packages included -- is now gated ONCE, centrally, in deploy_target via verify_target_repo
|
|
# (the single consolidated gate; it also runs under --skip-build and validates the deployed repo).
|
|
# The only per-build check kept here is the CD --build-number bump: confirm it actually LANDED in the
|
|
# built rpms' Release, since validating %{VERSION} alone can't catch a silently un-bumped NVR (which
|
|
# deploy's additive rsync would then dedup away). Every built dep + perl package carries the suffix;
|
|
# xCAT-genesis-base is intentionally NOT bumped (kept in lockstep with xcat-core's genesis-scripts).
|
|
if (!$dry_run && $RELEASE_BUMP ne '') {
|
|
my @rmiss;
|
|
for my $pkg (required_pkgs([sort keys %req], $skip_genesis, $skip_perl, $skip_xcat_dep)) {
|
|
next if $pkg eq 'xCAT-genesis-base';
|
|
next if $native && $native->{nodes}{$pkg} && $native->{nodes}{$pkg}{type} eq 'publisher';
|
|
my $rel = rpm_release($repo_dir, $pkg);
|
|
next if !defined $rel; # a missing rpm is caught by the completeness gate in deploy_target
|
|
push @rmiss, "$pkg: Release '$rel' is missing the CD bump '$RELEASE_BUMP'"
|
|
if index($rel, $RELEASE_BUMP) < 0;
|
|
}
|
|
die "FATAL: --build-number bump '$RELEASE_BUMP' did not land in built rpm(s) for $target:\n "
|
|
. join("\n ", @rmiss) . "\n" if @rmiss;
|
|
print "[manifest] Release bump '$RELEASE_BUMP' present on all built dep rpms for $target\n";
|
|
}
|
|
|
|
print_step('Collect source RPM artifacts');
|
|
print "source collection roots:\n";
|
|
print " $_\n" for @srpm_collect_roots;
|
|
|
|
my ($copied_srpms, $skipped_non_src, $missing_srpm_roots) = collect_srpms(
|
|
roots => \@srpm_collect_roots,
|
|
dest_dir => $srpm_repo_dir,
|
|
dry_run => $dry_run,
|
|
);
|
|
|
|
if (!$dry_run && $copied_srpms == 0) {
|
|
print "WARN: No source RPMs were collected. SRPM repo and tarball may be empty.\n";
|
|
}
|
|
|
|
if (!$skip_createrepo) {
|
|
run_step(
|
|
step => 'Run createrepo',
|
|
cmd => createrepo_c_cmd($repo_dir),
|
|
log => "$log_root/createrepo.log",
|
|
);
|
|
run_step(
|
|
step => 'Run createrepo for SRPM repo',
|
|
cmd => createrepo_c_cmd($srpm_repo_dir),
|
|
log => "$log_root/createrepo-srpm.log",
|
|
);
|
|
}
|
|
|
|
if (!$skip_tarball) {
|
|
my $cmd = join(' ',
|
|
'tar', '--sort=name', '--owner=0', '--group=0',
|
|
"--mtime=\@$SOURCE_DATE_EPOCH",
|
|
'-C', shell_quote($run_root),
|
|
'-czf', shell_quote($tarball),
|
|
'repo'
|
|
);
|
|
run_step(
|
|
step => 'Create tarball',
|
|
cmd => $cmd,
|
|
log => "$log_root/tarball.log",
|
|
);
|
|
my $srpm_cmd = join(' ',
|
|
'tar', '--sort=name', '--owner=0', '--group=0',
|
|
"--mtime=\@$SOURCE_DATE_EPOCH",
|
|
'-C', shell_quote($run_root),
|
|
'-czf', shell_quote($srpm_tarball),
|
|
'repo-src'
|
|
);
|
|
run_step(
|
|
step => 'Create SRPM tarball',
|
|
cmd => $srpm_cmd,
|
|
log => "$log_root/tarball-srpm.log",
|
|
);
|
|
}
|
|
|
|
if (!$dry_run) {
|
|
open my $sfh, '>', $summary_file or die "Cannot write $summary_file: $!\n";
|
|
print {$sfh} "run_root=$run_root\n";
|
|
print {$sfh} "repo_dir=$repo_dir\n";
|
|
print {$sfh} "target=$target\n";
|
|
print {$sfh} "arch=$profile->{arch}\n";
|
|
print {$sfh} "os_id=$os_id\n";
|
|
print {$sfh} "version_id=$version_id\n";
|
|
print {$sfh} "rel=$rel\n";
|
|
print {$sfh} "copied_rpms=$copied\n";
|
|
print {$sfh} "skipped_src_rpms=$skipped_src\n";
|
|
print {$sfh} "missing_collection_roots=$missing_roots\n";
|
|
print {$sfh} "srpm_repo_dir=$srpm_repo_dir\n";
|
|
print {$sfh} "copied_srpms=$copied_srpms\n";
|
|
print {$sfh} "skipped_non_src_rpms=$skipped_non_src\n";
|
|
print {$sfh} "missing_srpm_collection_roots=$missing_srpm_roots\n";
|
|
print {$sfh} "tarball=$tarball\n" if !$skip_tarball;
|
|
print {$sfh} "srpm_tarball=$srpm_tarball\n" if !$skip_tarball;
|
|
close $sfh;
|
|
}
|
|
|
|
print_step('Completed');
|
|
print "Collected binary RPMs: $copied\n";
|
|
print "Skipped source RPMs: $skipped_src\n";
|
|
print "Missing roots: $missing_roots\n";
|
|
print "Repo dir: $repo_dir\n";
|
|
print "Collected source RPMs: $copied_srpms\n";
|
|
print "Skipped non-src RPMs: $skipped_non_src\n";
|
|
print "Missing source roots: $missing_srpm_roots\n";
|
|
print "SRPM repo dir: $srpm_repo_dir\n";
|
|
print "Summary: $summary_file\n" if !$dry_run;
|
|
print "Tarball: $tarball\n" if !$skip_tarball;
|
|
print "SRPM Tarball: $srpm_tarball\n" if !$skip_tarball;
|
|
|
|
return { repo_dir => $repo_dir, rel => $rel, profile => $profile };
|
|
}
|
|
|
|
# The build profile of a target: EL release, arch of the rpms, where its noarch deps are built,
|
|
# which dep builders run and which rpms the deployed repo must contain.
|
|
sub target_profile {
|
|
my ($target) = @_;
|
|
if (my $native = openeuler_repo_subdir($target)) {
|
|
my ($version, $arch) = $target =~ /\Aopeneuler-(.*)-([^-]+)\z/;
|
|
die "Native target '$target' requires a $arch build host, found $host_arch\n"
|
|
unless $arch eq $host_arch;
|
|
return {
|
|
rel => $version,
|
|
arch => $arch,
|
|
noarch_cfg => $target,
|
|
forcearch => 0,
|
|
epel => 0,
|
|
dep_builders => [qw(grub2-xcat ipmitool-xcat syslinux-xcat goconserver conserver-xcat xnba-undi python3-scp)],
|
|
required => [qw(ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi
|
|
perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB)],
|
|
};
|
|
}
|
|
if (my $fa = $forcearch_targets{$target}) {
|
|
return {
|
|
%{$fa},
|
|
forcearch => 1,
|
|
epel => 0,
|
|
};
|
|
}
|
|
my ($rel) = $target =~ /epel-(\d+)-/;
|
|
die "Could not parse EL release from target '$target'\n" unless defined $rel;
|
|
return {
|
|
rel => $rel,
|
|
arch => $host_arch,
|
|
noarch_cfg => $target,
|
|
forcearch => 0,
|
|
epel => 1,
|
|
dep_builders => [qw(elilo-xcat grub2-xcat ipmitool-xcat syslinux-xcat goconserver conserver-xcat xnba-undi ipxe-xcat)],
|
|
# xCAT Requires all of these on every arch, and every one of them builds natively on
|
|
# every arch (the noarch deps -- grub2-xcat, xnba-undi, ipxe-xcat -- just repackage committed
|
|
# artifacts), so a self-sufficient per-arch build produces the whole set.
|
|
required => [qw(ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi ipxe-xcat
|
|
perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB)],
|
|
};
|
|
}
|
|
|
|
# The forcearch targets are shipped in mock-configs/; mock, and the include('/etc/mock/<cfg>.cfg')
|
|
# overlays of the per-package builders, need them in /etc/mock. Install a missing one; never
|
|
# overwrite one the host already has.
|
|
sub install_mock_cfg {
|
|
my ($cfg) = @_;
|
|
install_mock_cfg('templates/openeuler-lts-xcat') if $cfg =~ /^openeuler-/;
|
|
my $extension = $cfg =~ m{^templates/} ? 'tpl' : 'cfg';
|
|
my $src = "$repo_root/mock-configs/$cfg.$extension";
|
|
return if !-f $src;
|
|
my $dst = "/etc/mock/$cfg.$extension";
|
|
if (-f $dst) {
|
|
die "$dst differs from $src: the build would not use the configuration shipped in this"
|
|
. " tree. Remove or update the host copy (it is never overwritten here) and rerun.\n"
|
|
if system("cmp -s " . shell_quote($src) . ' ' . shell_quote($dst)) != 0;
|
|
return;
|
|
}
|
|
print "Installing mock config $src -> $dst\n";
|
|
return if $dry_run;
|
|
copy($src, $dst) or die "Failed to install $src -> $dst: $!\n";
|
|
chmod 0644, $dst;
|
|
}
|
|
|
|
sub source_rpm_build_command {
|
|
my ($builder, $target, $uniqueext, $result, $log, $work) = @_;
|
|
my $cfg = "$work/mock-deterministic.cfg";
|
|
if (!$dry_run) {
|
|
make_path($work, $result);
|
|
open my $fh, '>', $cfg or die "Cannot write $cfg: $!\n";
|
|
print {$fh} "include('/etc/mock/$target.cfg')\n";
|
|
print {$fh} "config_opts['environment']['SOURCE_DATE_EPOCH'] = '$SOURCE_DATE_EPOCH'\n";
|
|
close $fh or die "Cannot close $cfg: $!\n";
|
|
}
|
|
my $mock = join(' ', 'mock', '-r', sh_quote($cfg), '--uniqueext', sh_quote($uniqueext),
|
|
'--define', sh_quote('use_source_date_epoch_as_buildtime 1'),
|
|
'--define', sh_quote('clamp_mtime_to_source_date_epoch 1'),
|
|
'--define', sh_quote('_buildhost xcat-build'));
|
|
$mock .= join('', map { ' --define ' . sh_quote($_) } @{$builder->{defines} // []});
|
|
my $srpm = sh_quote($builder->{srpm});
|
|
my $prefix = '';
|
|
if ($RELEASE_BUMP ne '' || @{$builder->{patches} // []} || @{$builder->{defines} // []}) {
|
|
my $top = "$work/restamp";
|
|
if (!$dry_run) {
|
|
make_path(map { "$top/$_" } qw(BUILD BUILDROOT RPMS SOURCES SPECS SRPMS));
|
|
}
|
|
run_step(step => "Unpack source RPM: $builder->{name}",
|
|
cmd => 'rpm -i --define ' . sh_quote("_topdir $top") . " $srpm",
|
|
log => "$log/srpm-unpack.log");
|
|
my @specs = $dry_run ? ("$top/SPECS/*.spec") : bsd_glob("$top/SPECS/*.spec");
|
|
die "Expected one spec in source RPM: $builder->{srpm}\n" unless @specs == 1;
|
|
bump_dep_release_suffix($top, $RELEASE_BUMP) if !$dry_run && $RELEASE_BUMP ne '';
|
|
for my $patch (@{$builder->{patches} // []}) {
|
|
my $path = $patch->{staged} // $patch->{absolute_path};
|
|
die "Source patch SHA256 mismatch: $path\n" unless digest_file($path) eq $patch->{sha256};
|
|
run_step(step => "Apply native source patch: $builder->{name}",
|
|
cmd => 'patch --batch --fuzz=0 -p1 -d ' . sh_quote("$top/SPECS")
|
|
. ' -i ' . sh_quote($path), log => "$log/spec-patch.log");
|
|
}
|
|
my $restamped = "$work/restamp-srpm";
|
|
make_path($restamped) unless $dry_run;
|
|
$prefix = "$mock --buildsrpm --spec " . sh_quote($specs[0])
|
|
. ' --sources ' . sh_quote("$top/SOURCES") . ' --resultdir ' . sh_quote($restamped)
|
|
. ' && set -- ' . sh_quote($restamped) . '/*.src.rpm'
|
|
. ' && test "$#" -eq 1 && test -f "$1" && ';
|
|
$srpm = '"$1"';
|
|
}
|
|
return $prefix . "$mock --rebuild $srpm --resultdir " . sh_quote($result);
|
|
}
|
|
|
|
sub native_owner_command {
|
|
my ($plan, $target, $command, $uid) = @_;
|
|
my $overlay = $plan->{overlays}{$uid} // die "Missing native mock overlay\n";
|
|
my $script = 'mount --bind ' . sh_quote($overlay) . ' ' . sh_quote("/etc/mock/$target.cfg")
|
|
. ' && exec sh -c ' . sh_quote($command);
|
|
return 'unshare --mount --propagation private -- sh -c ' . sh_quote($script);
|
|
}
|
|
|
|
sub native_overlay {
|
|
my ($plan, $target, $work, $prereqs) = @_;
|
|
$plan->{overlays} = {1000 => "$work/native-1000.cfg", 0 => "$work/native-genesis-0.cfg",
|
|
procenv => "$work/native-procenv-bootstrap.cfg"};
|
|
return if $dry_run;
|
|
my $base = "$work/native-base.cfg";
|
|
copy("/etc/mock/$target.cfg", $base) or die "Cannot snapshot native mock configuration: $!\n";
|
|
my $url = $prereqs;
|
|
$url =~ s{([^A-Za-z0-9_./~-])}{sprintf('%%%02X', ord($1))}ge;
|
|
my $repo = "\n[xcat-native-inputs]\nname=xCAT native build prerequisites\nbaseurl=file://$url\n"
|
|
. "gpgkey=file://$url/repodata/repomd.xml.key\ngpgcheck=1\nrepo_gpgcheck=1\n"
|
|
. "enabled=1\nskip_if_unavailable=0\n";
|
|
for my $key (1000, 0, 'procenv') {
|
|
my $uid = $key eq 'procenv' ? 1000 : $key;
|
|
open my $fh, '>', $plan->{overlays}{$key} or die "Cannot write native overlay: $!\n";
|
|
print {$fh} 'include(' . JSON::PP->new->encode($base) . ")\n";
|
|
print {$fh} "config_opts['chrootuid'] = $uid\nconfig_opts['chrootgid'] = 1000\n";
|
|
print {$fh} "config_opts['dnf.conf'] += \"\"\"$repo\"\"\"\n";
|
|
print {$fh} "config_opts['plugin_conf']['bind_mount_enable'] = True\n";
|
|
print {$fh} "config_opts['plugin_conf']['procenv_enable'] = " . ($key eq 'procenv' ? 'False' : 'True') . "\n";
|
|
print {$fh} "config_opts['plugin_conf']['bind_mount_opts']['dirs'].append("
|
|
. '(' . JSON::PP->new->encode($prereqs) . ', ' . JSON::PP->new->encode($prereqs) . "))\n";
|
|
close $fh or die "Cannot close native overlay: $!\n";
|
|
}
|
|
open my $ledger, '>', "$work/native-overlays.json" or die "Cannot record native overlays: $!\n";
|
|
print {$ledger} JSON::PP->new->canonical->pretty->encode({base => {path => $base, sha256 => digest_file($base)},
|
|
overlays => [map { {purpose => $_, path => $plan->{overlays}{$_}, sha256 => digest_file($plan->{overlays}{$_})} } (1000, 0, 'procenv')]});
|
|
close $ledger or die "Cannot close native overlay ledger: $!\n";
|
|
}
|
|
|
|
sub build_native_inputs {
|
|
my ($plan, $target, $req, $work, $logs) = @_;
|
|
my $prereqs = "$work/native-prerequisites";
|
|
my $runtime = "$work/native-runtime";
|
|
my @source_roots;
|
|
make_path($prereqs, $runtime) unless $dry_run;
|
|
for my $name (@{$plan->{order}}) {
|
|
my $node = $plan->{nodes}{$name};
|
|
next unless $node->{type} eq 'publisher';
|
|
next if $dry_run;
|
|
verify_input($plan, $node, $node->{staged}, $plan->{trust_db});
|
|
copy($node->{staged}, "$prereqs/" . basename($node->{staged})) or die "Cannot stage publisher RPM: $!\n";
|
|
if ($req->{$name}) {
|
|
copy($node->{staged}, "$runtime/" . basename($node->{staged})) or die "Cannot collect publisher RPM: $!\n";
|
|
}
|
|
}
|
|
sign_and_index_repo($prereqs, $plan) unless $dry_run;
|
|
native_overlay($plan, $target, $work, $prereqs);
|
|
my $sequence = 0;
|
|
for my $name (@{$plan->{order}}) {
|
|
my $node = $plan->{nodes}{$name};
|
|
next unless $node->{type} eq 'srpm';
|
|
my $result = "$work/native-results/$name";
|
|
my $log = "$logs/native/$name";
|
|
my $uniqueext = build_mock_uniqueext("$target-$run_id", ++$sequence, $name);
|
|
my %builder = (%$node, srpm => $node->{staged} // "$work/native-inputs/$name/" . basename($node->{url}));
|
|
my $command = source_rpm_build_command(\%builder, $target, $uniqueext, $result, $log,
|
|
"$work/native-source/$name");
|
|
run_step(step => "Build native prerequisite: $name", log => "$log/run.log",
|
|
cmd => native_owner_command($plan, $target, $command, $name eq 'procenv' ? 'procenv' : 1000));
|
|
next if $dry_run;
|
|
my @rpms = grep { !/\.src\.rpm$/ } bsd_glob("$result/*.rpm");
|
|
validate_outputs($node, \@rpms, 1);
|
|
for my $rpm (@rpms) {
|
|
my $base = basename($rpm);
|
|
die "Conflicting native prerequisite artifact: $base\n" if -e "$prereqs/$base";
|
|
copy($rpm, "$prereqs/$base") or die "Cannot stage native prerequisite: $!\n";
|
|
my $id = rpm_identity($rpm);
|
|
copy($rpm, "$runtime/$base") or die "Cannot collect native output: $!\n" if $req->{$id->{name}};
|
|
}
|
|
sign_and_index_repo($prereqs, $plan);
|
|
my @problems = verify_rpms_checksig($prereqs, $gpg_key_name, $gpg_home, $plan);
|
|
die "Native prerequisite signature failure: @problems\n" if @problems;
|
|
open my $ledger, '>>', "$work/native-results.jsonl" or die "Cannot record native result: $!\n";
|
|
print {$ledger} JSON::PP->new->canonical->encode({name => $name, source_sha256 => $node->{sha256},
|
|
defines => $node->{defines} // [], patches => [map { {path => $_->{path}, sha256 => $_->{sha256}} } @{$node->{patches} // []}],
|
|
outputs => [map { {name => rpm_identity($_)->{name}, unsigned_path => $_,
|
|
unsigned_sha256 => digest_file($_), signed_sha256 => digest_file("$prereqs/" . basename($_))} } @rpms]}) . "\n";
|
|
close $ledger or die "Cannot close native result ledger: $!\n";
|
|
push @source_roots, $result;
|
|
scrub_buildroot($target, $uniqueext, "$log/scrub.log") unless $keep_buildroots;
|
|
}
|
|
return ($runtime, \@source_roots);
|
|
}
|
|
|
|
# Publish each target in its native repository layout with signatures, metadata and local repository helpers.
|
|
sub deploy_target {
|
|
my ($tgt, $info) = @_;
|
|
my $rel = $info->{rel};
|
|
my $src = $info->{repo_dir};
|
|
my $tarch = $info->{profile}{arch};
|
|
my $dest = target_cell($tgt);
|
|
my $subdir = File::Spec->abs2rel($dest, $repo_dep);
|
|
print_step("Deploy $tgt -> $dest");
|
|
return if $dry_run;
|
|
|
|
# Stage the cell in a sibling temp dir, sign+index+verify it THERE, then atomically swap it into
|
|
# place. This makes the deploy self-cleaning and atomic (PR #62 review #3):
|
|
# - self-cleaning: the cell is rebuilt from scratch each run, so stale snap-NVR rpms from an
|
|
# earlier build never accumulate. Previously deploy copied ADDITIVELY into an existing $dest
|
|
# and relied on the pipeline pre-wiping rh<N>/<arch> -- a standalone run accumulated versions.
|
|
# - atomic + verify-before-replace: a failed sign/index/verify leaves the previously-published
|
|
# cell untouched, and no reader ever sees a half-written cell.
|
|
# The staging dir is a sibling of $dest, so the rename is a same-filesystem (atomic) move. The
|
|
# cross-arch genesis (--finalize-xcat-dep) runs later and re-populates the foreign-arch genesis,
|
|
# so rebuilding this single-arch cell from $src is correct.
|
|
make_path(dirname($dest));
|
|
my $stage = "$dest.stage.$$";
|
|
remove_tree($stage) if -d $stage;
|
|
make_path($stage);
|
|
my $ok = eval {
|
|
for my $rpm (bsd_glob("$src/*.rpm")) {
|
|
next if $rpm =~ /\.src\.rpm$/;
|
|
publish_file($rpm, "$stage/" . basename($rpm));
|
|
}
|
|
# --genesis-release: the release itself is published ONCE into <repo-dep>/common, not into
|
|
# each per-EL cell, so nothing from it is kept here -- drop any stale OpenEmbedded Genesis
|
|
# rpm an earlier layout left in the collection. On the STAGE, so the published cell is
|
|
# already correct when it is swapped in.
|
|
remove_genesis_packages($stage, 0) if $genesis_release;
|
|
sign_and_index_repo($stage, $NATIVE_PLANS{$tgt});
|
|
write_dep_repo_metadata($stage, $rel, $tarch, $subdir);
|
|
# Automatic completeness + signature gate on the freshly signed cell -- the single
|
|
# consolidated gate (verify_target_repo, the same one --verify-repo runs). Asserts every
|
|
# manifest-required package is present at its pinned version, the repomd signature verifies,
|
|
# AND every rpm is signed by the key. Runs on the STAGE so a failure never lands in $dest.
|
|
# Suppressible with --no-verify-repo for iteration/debug.
|
|
verify_target_repo($stage, $tgt) unless $no_verify_repo;
|
|
1;
|
|
};
|
|
if (!$ok) {
|
|
my $err = $@;
|
|
remove_tree($stage); # leave the previously-published cell exactly as it was
|
|
die $err;
|
|
}
|
|
|
|
# Atomic replace: rename cannot overwrite a populated dir, so move the old cell aside, swap the
|
|
# staged cell in, then drop the old one. On a failed final rename, restore the old cell.
|
|
my $old = "$dest.old.$$";
|
|
remove_tree($old) if -d $old;
|
|
if (-d $dest) {
|
|
rename($dest, $old) or die "Failed to move old cell $dest aside: $!\n";
|
|
}
|
|
unless (rename($stage, $dest)) {
|
|
my $err = $!;
|
|
rename($old, $dest) if -d $old && !-d $dest; # best-effort restore
|
|
die "Failed to swap staged cell into $dest: $err\n";
|
|
}
|
|
remove_tree($old) if -d $old;
|
|
|
|
my $n = scalar(grep { !/\.src\.rpm$/ } bsd_glob("$dest/*.rpm"));
|
|
print "Deployed $subdir: $n rpms\n";
|
|
}
|
|
|
|
sub publish_genesis_common_repo {
|
|
my $dest = "$repo_dep/common";
|
|
print_step("Publish OpenEmbedded Genesis -> $dest");
|
|
|
|
if ($dry_run) {
|
|
preview_genesis_release_packages('rpm', $dest);
|
|
return;
|
|
}
|
|
|
|
# Every architecture run can publish common, so common has its own lock.
|
|
take_lock("$repo_dep/.common-publish.lock", 'common lock');
|
|
$COMMON_STAGE = tempdir('.common.XXXXXXXX', DIR => $repo_dep, CLEANUP => 0);
|
|
my $published = publish_genesis_release_packages('rpm', $COMMON_STAGE);
|
|
verify_genesis_release_packages('rpm', $COMMON_STAGE);
|
|
sign_and_index_repo($COMMON_STAGE);
|
|
write_common_repo_metadata($COMMON_STAGE);
|
|
# Gate the STAGE, so an incomplete shared repo is never swapped into place. Completeness only:
|
|
# the packages were verified against the release checksums as they were copied, and the deploy
|
|
# asserts every rpm's signature, but until now nothing checked that the repository being
|
|
# published actually carries the whole architecture set the manifest says it must.
|
|
verify_common_repo($COMMON_STAGE) unless $no_verify_repo;
|
|
chmod(0755, $COMMON_STAGE)
|
|
or die "Cannot make $COMMON_STAGE traversable: $!\n";
|
|
replace_common_repository($COMMON_STAGE, $dest);
|
|
print "Published common Genesis repository: $published rpms\n";
|
|
}
|
|
|
|
#--------------------------------------------------------------------------------
|
|
|
|
=head3 verify_common_repo
|
|
|
|
Assert the shared repository carries every package required by [common]. [common] must
|
|
describe every currently supported Genesis architecture.
|
|
|
|
Arguments:
|
|
$dir - the repository to check (the staging directory, before it is swapped into place)
|
|
Returns:
|
|
1, or dies listing every problem
|
|
|
|
=cut
|
|
|
|
#--------------------------------------------------------------------------------
|
|
sub verify_common_repo {
|
|
my ($dir) = @_;
|
|
my %req = %{ common_repository_requirements() };
|
|
my @names = sort keys %req;
|
|
my %present = repo_present_versions($dir, \@names);
|
|
my %present_evr = map { $_ => rpm_evr($dir, $_) } @names;
|
|
my @problems = verify_repo_packages(\%req, \%present, \%present_evr, \&rpm_vercmp_segment);
|
|
if (@problems) {
|
|
print " - $_\n" for @problems;
|
|
die "FATAL: shared Genesis repo INCOMPLETE at $dir (" . scalar(@problems) . " problem(s))\n";
|
|
}
|
|
print "[verify-repo] common complete: " . scalar(@names)
|
|
. " packages present + EVR-satisfied in $dir\n";
|
|
return 1;
|
|
}
|
|
|
|
sub common_repository_requirements {
|
|
my $manifest = "$repo_root/packages-manifest.conf";
|
|
my %MAN = read_manifest($manifest);
|
|
my %common = %{ $MAN{common} // {} };
|
|
die "FATAL: no [common] section in $manifest -- cannot verify the shared Genesis repository\n"
|
|
if !%common;
|
|
|
|
return validate_repository_packages(
|
|
\%common,
|
|
'common',
|
|
rpm_package_prefix(),
|
|
map { rpm_package_name($_) } architectures(),
|
|
);
|
|
}
|
|
|
|
sub replace_common_repository {
|
|
my ($staged, $destination) = @_;
|
|
my $backup = "$repo_dep/.common.previous.$$";
|
|
remove_tree($backup) if -e $backup || -l $backup;
|
|
|
|
$COMMON_DESTINATION = $destination;
|
|
if (-e $destination || -l $destination) {
|
|
$COMMON_BACKUP = $backup;
|
|
rename($destination, $backup)
|
|
or die "Cannot preserve $destination before publication: $!\n";
|
|
}
|
|
unless (rename($staged, $destination)) {
|
|
my $error = $!;
|
|
rename($backup, $destination) if $COMMON_BACKUP && -d $backup;
|
|
die "Cannot publish $destination: $error\n";
|
|
}
|
|
undef($COMMON_STAGE);
|
|
undef($COMMON_DESTINATION);
|
|
remove_tree($backup) if $COMMON_BACKUP && -d $backup;
|
|
undef($COMMON_BACKUP);
|
|
}
|
|
|
|
sub publish_file {
|
|
my ($source, $destination) = @_;
|
|
my ($temporary_fh, $temporary) = tempfile(
|
|
'.xcat-deploy.XXXXXX',
|
|
DIR => dirname($destination),
|
|
UNLINK => 0,
|
|
);
|
|
close($temporary_fh) or die "Cannot close deployment staging file: $!\n";
|
|
|
|
my $mode = (stat($source))[2];
|
|
die "Cannot read mode from $source: $!\n" unless defined($mode);
|
|
my $published = eval {
|
|
copy($source, $temporary)
|
|
or die "Failed to stage $source -> $temporary: $!\n";
|
|
chmod($mode & 0x0fff, $temporary)
|
|
or die "Failed to set mode on $temporary: $!\n";
|
|
rename($temporary, $destination)
|
|
or die "Failed to publish $temporary -> $destination: $!\n";
|
|
1;
|
|
};
|
|
return if $published;
|
|
|
|
my $error = $@ || "Failed to publish $source\n";
|
|
unlink($temporary) if -e $temporary || -l $temporary;
|
|
die $error;
|
|
}
|
|
|
|
# createrepo_c command with deterministic metadata: primary/filelists/other as *.xml.zst, with
|
|
# --set-timestamp-to-revision pinning repomd to SOURCE_DATE_EPOCH.
|
|
#
|
|
# NO --database. It writes *.sqlite.bz2, and SQLite needs POSIX locks. The build tree lives on the
|
|
# shared shared tree, an NFS mount the hypervisor re-exports, and the kernel refuses locks there:
|
|
# every attempt answers errno 524. createrepo_c died with "Cannot open .repodata/primary.sqlite:
|
|
# Can not create db_info table: disk I/O error" on every target once the build hosts moved off
|
|
# virtiofs -- xcat-dep-el-cd #161 and #162 both failed that way and staged nothing.
|
|
#
|
|
# --database is deprecated in createrepo_c 1.1.2 and --no-database is its default. dnf on el8+ and
|
|
# zypper read the XML.
|
|
sub createrepo_c_cmd {
|
|
my ($dir) = @_;
|
|
return MockBuildUtils::createrepo_c_cmd($dir, $SOURCE_DATE_EPOCH);
|
|
}
|
|
|
|
sub sign_and_index_repo {
|
|
my ($dir, $native) = @_;
|
|
return MockBuildUtils::sign_and_index_repo($dir, $native,
|
|
gpg_sign => $gpg_sign, gpg_home => $gpg_home,
|
|
gpg_key_name => $gpg_key_name, gpg_program => $gpg_program,
|
|
source_date_epoch => $SOURCE_DATE_EPOCH, run => \&run_simple);
|
|
}
|
|
|
|
sub write_dep_repo_metadata {
|
|
my ($dir, $rel, $tarch, $subdir) = @_;
|
|
$subdir //= "rh$rel/$tarch";
|
|
my $baseurl = "https://xcat.org/files/xcat/repos/yum/devel/xcat-dep/$subdir";
|
|
my $gpgcheck = $gpg_sign || $subdir =~ /^openeuler/ ? 1 : 0;
|
|
my $gpgkey_line = $gpgcheck ? "gpgkey=$baseurl/repodata/repomd.xml.key" : "# gpgkey=";
|
|
my $label = $subdir =~ /^openeuler/ ? $subdir : "rh$rel $tarch";
|
|
# repo_gpgcheck=1 makes clients verify the DETACHED repomd.xml signature (repomd.xml.asc) against
|
|
# gpgkey before trusting the metadata -- sign_and_index_repo produces both, so enforce it. Mirrors
|
|
# gpgcheck: off when the repo is unsigned.
|
|
open my $r, '>', "$dir/xcat-dep.repo" or die "Cannot write $dir/xcat-dep.repo: $!\n";
|
|
print {$r} <<"EOF";
|
|
[xcat-dep]
|
|
name=xCAT 2 dependencies ($label)
|
|
baseurl=$baseurl
|
|
enabled=1
|
|
gpgcheck=$gpgcheck
|
|
repo_gpgcheck=$gpgcheck
|
|
$gpgkey_line
|
|
EOF
|
|
close $r;
|
|
|
|
write_local_repo_helper($dir);
|
|
write_buildinfo($dir, $subdir);
|
|
}
|
|
|
|
sub write_common_repo_metadata {
|
|
my ($dir) = @_;
|
|
my $baseurl = "https://xcat.org/files/xcat/repos/yum/devel/xcat-dep/common";
|
|
my $gpgcheck = $gpg_sign ? 1 : 0;
|
|
my $gpgkey_line = $gpg_sign ? "gpgkey=$baseurl/repodata/repomd.xml.key" : "# gpgkey=";
|
|
open my $r, '>', "$dir/xcat-dep-common.repo"
|
|
or die "Cannot write $dir/xcat-dep-common.repo: $!\n";
|
|
print {$r} <<"EOF";
|
|
[xcat-dep-common]
|
|
name=xCAT 2 common dependencies
|
|
baseurl=$baseurl
|
|
enabled=1
|
|
gpgcheck=$gpgcheck
|
|
repo_gpgcheck=$gpgcheck
|
|
skip_if_unavailable=1
|
|
$gpgkey_line
|
|
EOF
|
|
close $r;
|
|
|
|
write_local_repo_helper($dir);
|
|
write_buildinfo($dir, 'common');
|
|
}
|
|
|
|
sub write_local_repo_helper {
|
|
my ($dir) = @_;
|
|
|
|
open my $m, '>', "$dir/mklocalrepo.sh" or die "Cannot write $dir/mklocalrepo.sh: $!\n";
|
|
print {$m} <<'EOS';
|
|
#!/bin/sh
|
|
SCRIPT_DIRECTORY=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
|
cd "$SCRIPT_DIRECTORY" || exit 1
|
|
set -- xcat-*.repo
|
|
if [ "$#" -ne 1 ] || [ "$1" = "xcat-*.repo" ]; then
|
|
echo "ERROR: Execute $0 in an xcat-dep repository directory"
|
|
exit 1
|
|
fi
|
|
REPOFILE=$1
|
|
DIRECTORY="/etc/yum.repos.d"
|
|
if [ ! -d "$DIRECTORY" ]; then
|
|
DIRECTORY="/etc/zypp/repos.d"
|
|
fi
|
|
CURRENT_DIRECTORY=$(pwd)
|
|
sed -e 's|baseurl=.*|baseurl=file://'"$CURRENT_DIRECTORY"'|' "$REPOFILE" \
|
|
| sed -e 's|gpgkey=.*|gpgkey=file://'"$CURRENT_DIRECTORY"'/repodata/repomd.xml.key|' \
|
|
> "$DIRECTORY/$REPOFILE"
|
|
EOS
|
|
close $m;
|
|
chmod 0775, "$dir/mklocalrepo.sh";
|
|
}
|
|
|
|
sub write_buildinfo {
|
|
my ($dir, $target) = @_;
|
|
my $build_time = strftime("%a %b %e %H:%M:%S %Z %Y", gmtime($SOURCE_DATE_EPOCH));
|
|
my $build_machine = `hostname`; chomp $build_machine;
|
|
my $commit = `git -C "$repo_root" rev-parse HEAD 2>/dev/null`; chomp $commit;
|
|
if (!$commit && -f "$repo_root/Gitinfo") {
|
|
($commit) = read_lines("$repo_root/Gitinfo");
|
|
$commit =~ s/\s+\z// if defined($commit);
|
|
}
|
|
$commit ||= 'unknown';
|
|
my $commit_short = substr($commit, 0, 7);
|
|
my $release = strftime('snap%Y%m%d%H%M', gmtime($SOURCE_DATE_EPOCH));
|
|
open my $b, '>', "$dir/buildinfo.txt" or die "Cannot write $dir/buildinfo.txt: $!\n";
|
|
print {$b} <<"EOF";
|
|
TARGET=$target
|
|
RELEASE=$release
|
|
BUILD_TIME=$build_time
|
|
BUILD_MACHINE=$build_machine
|
|
COMMIT_ID=$commit_short
|
|
COMMIT_ID_LONG=$commit
|
|
SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH
|
|
EOF
|
|
close $b;
|
|
}
|
|
|
|
|
|
|
|
# Re-run createrepo_c on a repo whose rpm set changed, and (under --gpg-sign) re-sign +
|
|
# re-export repomd. Does NOT re-sign the rpms (cross_copy_genesis already did the copied
|
|
# one; the rest keep their build-time signatures).
|
|
sub reindex_and_sign_repo {
|
|
my ($dir) = @_;
|
|
run_simple(createrepo_c_cmd($dir));
|
|
if ($gpg_sign) {
|
|
local $ENV{GNUPGHOME} = $gpg_home if $gpg_home;
|
|
my $repomd = "$dir/repodata/repomd.xml";
|
|
unlink "$repomd.asc" if -f "$repomd.asc";
|
|
run_simple("gpg -a --detach-sign --default-key " . sh_quote($gpg_key_name) . ' ' . sh_quote($repomd));
|
|
run_simple("gpg -a --export " . sh_quote($gpg_key_name) . " > " . sh_quote("$repomd.key"));
|
|
}
|
|
}
|
|
|
|
sub usage {
|
|
return <<"USAGE";
|
|
Usage: $0 [options]
|
|
|
|
Build xcat-dep RPMs (dep packages, perl packages, and the OS-dependent xCAT-genesis-base),
|
|
consolidate binary/source artifacts, run createrepo, and create tarballs. The full xCAT core
|
|
is built separately by the xcat-core pipeline, not here.
|
|
|
|
Options:
|
|
--repo-root PATH xcat-dep repository root (default: script directory)
|
|
--xcat-source PATH xCAT source root with buildrpms.pl (default: <repo-root>/../xcat-core)
|
|
--output PATH Single base for ALL output; --output-root and --repo-dep derive from
|
|
it. A fail-fast lock is held at <PATH>/.lock, so pass distinct paths
|
|
to run on two hosts in parallel on one NFS (default: <repo-root>/build-output)
|
|
--output-root PATH Override the derived build tree root (default: <output>/mockbuild-all)
|
|
--repo-dep PATH Override the deployable output root; rh8/rh9/rh10/<arch> and common
|
|
are assembled and signed here (default: <output>/xcat-dep)
|
|
--try-unlock-timeout N Wait about N seconds (default 0: one retry of 3s) for a lock
|
|
that a live process holds, then fail with the command that
|
|
removes it. A lock whose owner is proven dead on this host
|
|
is taken over at once. Locks: <output>/.lock, one <repo-dep>/rh<N>/.<arch>.lock
|
|
per target, and <repo-dep>/.common-publish.lock for common
|
|
--finalize-xcat-dep Post-build cross-arch genesis mode (builds nothing). Requires
|
|
--x86_64-repo and --ppc64le-repo. For each matching <os>/x86_64 and
|
|
<os>/ppc64le repo pair, copies the noarch xCAT-genesis-base-ppc64
|
|
(the ppc64le genesis; xCAT names it -ppc64 via tarch, no big-endian
|
|
code) into the x86_64 repo and xCAT-genesis-base-x86_64 into the
|
|
ppc64le repo (dropping any stale foreign-arch genesis), then
|
|
re-indexes + re-signs. Restores the 2.17 cross-arch genesis
|
|
(issue #7610). Honors --gpg-sign/--gpg-key-name/--gpg-home. Use alone.
|
|
--finalize-arch ARCH (finalize) Write, lock and verify only the cells of ARCH (x86_64 or
|
|
ppc64le; repeatable). Run it on the host that builds ARCH, so a
|
|
dead finalize leaves locks that host can reclaim. Default: both
|
|
--x86_64-repo PATH (finalize) x86_64 repo root holding <os>/x86_64 (e.g. rh9/x86_64)
|
|
--ppc64le-repo PATH (finalize) ppc64le repo root holding <os>/ppc64le
|
|
--verify-repo PATH Standalone completeness + signature gate over the per-target repo at PATH
|
|
(builds nothing). Asserts every package packages-manifest.conf requires for
|
|
the target is present at a version satisfying its pin AND that the repomd
|
|
is signed by --gpg-key-name; exits 0 if complete, or lists each MISSING/
|
|
VERSION/UNSIGNED/WRONGKEY problem and fails. The target is derived from the path
|
|
(.../rh<N>/<arch> -> alma+epel-<N>-<arch>, or
|
|
.../openeuler<releaseSP>/<arch> -> openeuler-<releaseSP>-<arch>)
|
|
unless --target is given; the
|
|
manifest and gpg key/home come from the usual options. Use alone.
|
|
--no-verify-repo Suppress the AUTOMATIC post-build completeness+signature gate that runs
|
|
after each target's repo is finalized (default: verification ON)
|
|
--gpg-sign Sign RPMs and repomd.xml in every published repository
|
|
Required for native openEuler publication, including --skip-build.
|
|
Standalone --verify-repo consumes existing signed output.
|
|
--gpg-key-name NAME GPG key name (default: "xCAT Signing Key")
|
|
--gpg-home PATH GNUPGHOME for signing (default: system keyring)
|
|
--target NAME Build only this target (<ID>+epel-<REL>-<ARCH>, or a forcearch
|
|
config from mock-configs/ such as rocky-10-riscv64-xcat, which
|
|
cross-builds that arch on this host); default is the host arch
|
|
across rh8, rh9 and rh10. On openEuler the default retains the exact
|
|
host release and service pack, e.g. openeuler-24.03sp3-x86_64.
|
|
Native targets require the matching host architecture and deploy to
|
|
<repo-dep>/openeuler<releaseSP>/<arch> with signature checks enabled.
|
|
The build host must provide mock and its Perl dependencies; openEuler
|
|
24.03 LTS-SP3 can build older releases in their exact native targets.
|
|
--nproc N Parallel jobs for buildrpms.pl (default: 1)
|
|
--build-timeout SECONDS Wall-clock bound for one build step. Default: none for a native
|
|
target, and 9000 for a forcearch (qemu-user) target, which runs at
|
|
roughly a tenth of native speed. 0 removes the bound. On expiry the
|
|
run prints the step's process tree, each pid's wchan and stack, and
|
|
a 20-second CPU sample -- a deadlocked build uses no ticks -- then
|
|
kills the whole process group.
|
|
--parallel-builds N Max concurrent top-level build steps within one EL target (default: auto)
|
|
--parallel-targets N Concurrent EL targets (rh8/rh9/rh10). 0/auto = all at once, 1 = serial,
|
|
N = cap at N. Each target is fully output-isolated (default: 1 = serial)
|
|
--max-parallel N Global cap on concurrent mock builds across ALL targets, to avoid
|
|
oversubscribing the host. Split evenly across active targets.
|
|
0/auto = host nproc (default: auto)
|
|
--run-id ID Run identifier suffix (default: derived from build timestamp)
|
|
--build-timestamp EPOCH Unix epoch for deterministic builds (default: Gitepoch or git log)
|
|
--skip-build Skip all build steps and only collect/create repo/tarballs
|
|
--skip-xcat-dep Skip xcat-dep mockbuild.pl package steps
|
|
--skip-perl Skip perl package build step
|
|
--install-deps Install this host's build prerequisites (package manager + the perl
|
|
modules the script loads), verify each module now loads, then exit.
|
|
Run once per build host, as root. Use alone.
|
|
--skip-genesis Skip the existing per-EL Genesis image build
|
|
--skip-createrepo Skip createrepo
|
|
--skip-tarball Skip binary/SRPM tarball creation
|
|
--genesis-release PATH Publish a verified OpenEmbedded Genesis RPM release in common
|
|
--scrub-all-chroots Run mock -r <target> --scrub=all before build/collect
|
|
--collect-dir PATH Additional directory to scan recursively for RPMs (repeatable)
|
|
--dry-run Print planned commands without executing
|
|
|
|
Notes:
|
|
- Run this script as root on the build host.
|
|
- ARCH is derived from: uname -m (or from the forcearch target)
|
|
- Top-level parallel queue includes xcat-dep mockbuild.pl steps, the perl builder,
|
|
and the xCAT-genesis-base build (../xcat-core/buildrpms.pl --package xCAT-genesis-base).
|
|
- Child mockbuild scripts are invoked with per-step mock --uniqueext values
|
|
to avoid lock collisions on the same mock config.
|
|
- If --target is omitted, it is deduced from /etc/os-release:
|
|
ID + epel + int(VERSION_ID) + ARCH
|
|
USAGE
|
|
}
|
|
|
|
sub run_simple {
|
|
my ($cmd) = @_;
|
|
my $rc = system($cmd);
|
|
if ($rc != 0) {
|
|
my $exit = $rc == -1 ? 255 : ($rc >> 8);
|
|
die "Command failed (rc=$exit): $cmd\n";
|
|
}
|
|
}
|
|
|
|
sub run_step {
|
|
my (%args) = @_;
|
|
my $step = $args{step} // 'Run command';
|
|
my $cmd = $args{cmd} // die "run_step missing cmd\n";
|
|
my $cwd = $args{cwd};
|
|
my $log = $args{log};
|
|
|
|
print_step($step);
|
|
print "+ $cmd\n";
|
|
if ($cwd) {
|
|
print " (cwd: $cwd)\n";
|
|
}
|
|
if ($log) {
|
|
print " (log: $log)\n";
|
|
}
|
|
|
|
return if $dry_run;
|
|
|
|
my $full_cmd = $cmd;
|
|
if ($cwd) {
|
|
$full_cmd = "cd " . shell_quote($cwd) . " && $cmd";
|
|
}
|
|
if ($log) {
|
|
my $log_dir = dirname($log);
|
|
make_path($log_dir) if !-d $log_dir;
|
|
$full_cmd .= " > " . shell_quote($log) . " 2>&1";
|
|
}
|
|
|
|
# A forcearch step cross-builds through qemu-user, where a deadlocked build consumes no CPU and
|
|
# never exits. Bound the steps that can run emulated, and report why the build stopped.
|
|
my $timeout = $args{timeout} || 0;
|
|
my $r = run_bounded(cmd => $full_cmd, timeout => $timeout, label => $step, out => \*STDOUT);
|
|
die "Step TIMED OUT after $r->{elapsed}s (budget ${timeout}s): $step\nCommand: $cmd\n"
|
|
. " See the stall report above" . ($log ? " and $log" : '') . ".\n"
|
|
if $r->{timed_out};
|
|
if ($r->{ec} != 0) {
|
|
my $exit = $r->{ec} == -1 ? 255 : $r->{ec};
|
|
die "Step failed (rc=$exit): $step\nCommand: $cmd\n";
|
|
}
|
|
}
|
|
|
|
# Scrub a single mock buildroot via mock's own --scrub (never rm). mock takes the buildroot lock for
|
|
# the scrub, so a concurrent build holding it makes mock BLOCK until release rather than corrupt a live
|
|
# chroot (this can never rm a chroot out from under a running build). Failures (already scrubbed or
|
|
# config missing) are tolerated -- a cleanup hiccup must never fail the build. Scrubs both the
|
|
# build chroot and its per-uniqueext bootstrap chroot (each build step gets its own bootstrap, so
|
|
# both must go or /var/lib/mock still leaks). The shared root cache under /var/cache/mock is kept,
|
|
# so rebuilds stay fast. $uniqueext is optional (genesis has none).
|
|
sub scrub_buildroot {
|
|
my ($cfg, $uniqueext, $log) = @_;
|
|
return if !defined $cfg || $cfg eq '';
|
|
my $ext = (defined $uniqueext && $uniqueext ne '')
|
|
? ' --uniqueext ' . sh_quote($uniqueext) : '';
|
|
eval {
|
|
run_step(
|
|
step => "Scrub chroot $cfg$ext",
|
|
cmd => "mock -r " . sh_quote($cfg) . $ext . " --scrub=chroot --scrub=bootstrap",
|
|
log => $log,
|
|
);
|
|
1;
|
|
} or do {
|
|
warn "WARN: chroot scrub failed (tolerated) for $cfg$ext: $@";
|
|
};
|
|
}
|
|
|
|
sub run_build_steps_parallel {
|
|
my (%args) = @_;
|
|
my $steps = $args{steps} // [];
|
|
my $max_processes = $args{max_processes} // 1;
|
|
return if !@{$steps};
|
|
|
|
# Returns the ids of any steps that failed; the caller (build_one_target) enforces
|
|
# zero-tolerance -- ANY failed step fails the whole run, genesis included, with no special-case.
|
|
# We build only packages required for the target (per packages-manifest.conf), so there is no
|
|
# "expected to fail on this arch/el" case left to tolerate. There is likewise no genesis
|
|
# exception: since xcat-core #7696, buildrpms.pl exits 0 iff it produced the genesis rpm, so a
|
|
# non-zero genesis exit is a real failure (the old "tolerate if the rpm is already present"
|
|
# workaround is gone -- a stale artifact must never mask a failed build).
|
|
if ($dry_run || $max_processes <= 1 || @{$steps} == 1) {
|
|
my @failed;
|
|
for my $step (@{$steps}) {
|
|
my $ok = eval { run_step(%{$step}); 1 };
|
|
next if $ok;
|
|
warn "ERROR: build step failed: $step->{step}\n" . ($@ // '');
|
|
push @failed, (defined($step->{id}) && $step->{id} ne '' ? $step->{id} : $step->{step});
|
|
}
|
|
assert_build_progress(scalar(@{$steps}), scalar(@failed));
|
|
return @failed;
|
|
}
|
|
|
|
my $workers = $max_processes;
|
|
$workers = scalar(@{$steps}) if $workers > scalar(@{$steps});
|
|
|
|
print_step('Run build steps in parallel');
|
|
print "max_processes: $workers\n";
|
|
print "queued steps:\n";
|
|
print " - $_->{step}\n" for @{$steps};
|
|
|
|
my %failed;
|
|
my $pm = Parallel::ForkManager->new($workers);
|
|
my %kids;
|
|
$pm->run_on_start(sub { $kids{ $_[0] } = 1 });
|
|
$pm->run_on_finish(
|
|
sub {
|
|
my ($pid, $exit_code, $ident, $signal, $core_dump) = @_;
|
|
delete $kids{$pid};
|
|
return if $exit_code == 0 && $signal == 0 && !$core_dump;
|
|
my $key = defined($ident) ? $ident : "pid:$pid";
|
|
$failed{$key} = {
|
|
exit => $exit_code,
|
|
signal => $signal,
|
|
core_dump => $core_dump ? 1 : 0,
|
|
};
|
|
}
|
|
);
|
|
|
|
# Same reason as the per-target workers: a signal here must reach the builds these workers run.
|
|
my $forward = forward_signals_to_workers(
|
|
pids => \%kids,
|
|
reap => sub { $pm->wait_all_children },
|
|
);
|
|
local $SIG{INT} = $forward;
|
|
local $SIG{TERM} = $forward;
|
|
local $SIG{HUP} = $forward;
|
|
|
|
for my $step (@{$steps}) {
|
|
my %step_copy = %{$step};
|
|
my $ident = delete $step_copy{id};
|
|
$ident = $step_copy{step} if !defined($ident) || $ident eq '';
|
|
|
|
# Same window as the per-target workers, and the same rule about waiting for a slot with
|
|
# the signals unblocked.
|
|
$pm->wait_for_available_procs(1) if $pm->max_procs;
|
|
my $orchestrator = $$;
|
|
my $previous = block_handled_signals();
|
|
my $pid = $pm->start($ident);
|
|
if ($pid) { restore_signal_mask($previous); next; }
|
|
$SIG{$_} = 'DEFAULT' for ($$ == $orchestrator ? () : qw(INT TERM HUP));
|
|
restore_signal_mask($previous);
|
|
|
|
my $ok = eval {
|
|
run_step(%step_copy);
|
|
1;
|
|
};
|
|
if (!$ok) {
|
|
my $err = $@;
|
|
$err = "unknown error\n" if !defined($err) || $err eq '';
|
|
print STDERR "ERROR [$ident] $err";
|
|
$pm->finish(1);
|
|
}
|
|
$pm->finish(0);
|
|
}
|
|
$pm->wait_all_children;
|
|
|
|
if (%failed) {
|
|
my @lines;
|
|
for my $id (sort keys %failed) {
|
|
my $f = $failed{$id};
|
|
push @lines,
|
|
"$id (exit=$f->{exit}, signal=$f->{signal}, core_dump=$f->{core_dump})";
|
|
}
|
|
warn "ERROR: build step(s) failed:\n " . join("\n ", @lines) . "\n";
|
|
}
|
|
|
|
assert_build_progress(scalar(@{$steps}), scalar(keys %failed));
|
|
my @failed_ids = sort keys %failed;
|
|
return @failed_ids;
|
|
}
|
|
|
|
# The caller enforces zero tolerance per required package (verify_target_repo). ALL steps
|
|
# failing is a different thing: the builder itself is unusable (no mock, a broken chroot, no
|
|
# network), this invocation produced nothing, and every package the run would go on to publish
|
|
# would come from somewhere other than this build -- say that, instead of naming the missing
|
|
# packages later.
|
|
sub assert_build_progress {
|
|
my ($attempted, $failures) = @_;
|
|
return unless every_step_failed($attempted, $failures);
|
|
die "FATAL: every build step failed ($failures/$attempted). Check the build logs.\n";
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# repo_present_versions: thin disk layer for the repo gate. Given a built repo dir and the list of
|
|
# required package names, return %present = (name => rpm_version($dir, $name)) for each -- reusing the
|
|
# EXISTING rpm_version so genesis's arch-suffixed naming resolves exactly as the in-line manifest pin
|
|
# check does. rpm_version returns undef for an absent package, which verify_repo_packages then reports
|
|
# as MISSING. Pure disk read; the decision itself lives in verify_repo_packages.
|
|
sub repo_present_versions {
|
|
my ($dir, $names) = @_;
|
|
my %present;
|
|
$present{$_} = rpm_version($dir, $_) for @$names;
|
|
return %present;
|
|
}
|
|
|
|
# gpg_key_fingerprint: resolve a gpg key NAME (e.g. "xCAT Signing Key") to its primary-key
|
|
# fingerprint in the given keyring, so the expected and observed signing identities are compared in
|
|
# the SAME form (a fingerprint). Falls back to the name itself when it cannot be resolved.
|
|
sub gpg_key_fingerprint {
|
|
my ($keyname, $home) = @_;
|
|
my $h = ($home ne '') ? ' --homedir ' . sh_quote($home) : '';
|
|
my $out = `gpg$h --with-colons --fingerprint --list-keys ${\ sh_quote($keyname)} 2>/dev/null` // '';
|
|
# Collect the PRIMARY-key fingerprint of every key matching $keyname (the fpr line right after a
|
|
# 'pub' record; subkey fprs follow 'sub' and are ignored). Return undef -- not a guess -- when the
|
|
# key is absent (unresolved) or when MORE THAN ONE key matches the name (ambiguous): the caller
|
|
# then hard-fails SIGKEY rather than comparing against a possibly-wrong key.
|
|
my (@fprs, $want);
|
|
for my $line (split /\n/, $out) {
|
|
if ($line =~ /^pub:/) { $want = 1; }
|
|
elsif ($line =~ /^sub:/) { $want = 0; }
|
|
elsif ($want && $line =~ /^fpr:+([0-9A-Fa-f]+):/) { push @fprs, $1; $want = 0; }
|
|
}
|
|
my $fingerprint;
|
|
$fingerprint = $fprs[0] if @fprs == 1;
|
|
return $fingerprint;
|
|
}
|
|
|
|
# gpg_key_ids: all acceptable key ids (lowercased) for a signing key NAME -- the primary key id AND
|
|
# every subkey id, in both 16-hex (long) and 8-hex (short) forms. rpm header signatures report the
|
|
# signing SUBKEY id, so the per-rpm gate accepts any id belonging to the key rather than one exact
|
|
# fingerprint. Returns a hashref set (empty if the key can't be listed).
|
|
sub gpg_key_ids {
|
|
my ($keyname, $home) = @_;
|
|
my $h = ($home ne '') ? ' --homedir ' . sh_quote($home) : '';
|
|
my $out = `gpg$h --with-colons --list-keys ${\ sh_quote($keyname)} 2>/dev/null` // '';
|
|
my %ids;
|
|
for my $line (split /\n/, $out) {
|
|
my @f = split /:/, $line;
|
|
next unless ($f[0] // '') =~ /^(?:pub|sub)$/ && defined $f[4] && $f[4] ne '';
|
|
my $id = $f[4];
|
|
$ids{ lc $id } = 1;
|
|
$ids{ lc substr($id, -16) } = 1 if length($id) > 16;
|
|
$ids{ lc substr($id, -8) } = 1 if length($id) > 8;
|
|
}
|
|
return \%ids;
|
|
}
|
|
|
|
# rpm_signer_keyid: the signing key id (lowercased hex) of a built rpm's header signature, or undef
|
|
# when the rpm is not signed. Reads the RSA (or DSA) header pgpsig and pulls the "Key ID <hex>" field.
|
|
sub rpm_signer_keyid {
|
|
my ($rpm) = @_;
|
|
my $keyid;
|
|
for my $tag (qw(RSAHEADER DSAHEADER)) {
|
|
my $out = `rpm -qp --qf '%{$tag:pgpsig}' ${\ sh_quote($rpm)} 2>/dev/null` // '';
|
|
if ($out =~ /Key ID\s+([0-9A-Fa-f]+)/i) { $keyid = lc($1); last; }
|
|
}
|
|
return $keyid;
|
|
}
|
|
|
|
# rpm_evr: the single distinct EPOCH:VERSION-RELEASE of package $name's binary rpm(s) in $dir (epoch
|
|
# defaults to 0 when the header carries none), or undef if none match. Mirrors rpm_version's dedup:
|
|
# more than one distinct EVR means a stale artifact was not cleaned before the build (a version pin
|
|
# could then pass against the wrong rpm). genesis's x86_64 + ppc64 rpms share one EVR, so a normal
|
|
# pair is a single entry.
|
|
sub rpm_evr {
|
|
my ($dir, $name) = @_;
|
|
my $glob = ($name eq 'xCAT-genesis-base')
|
|
? "$dir/xCAT-genesis-base-*.rpm"
|
|
: "$dir/${name}-*.rpm";
|
|
my %evrs;
|
|
for my $f (sort glob($glob)) {
|
|
next if $f =~ /\.src\.rpm$/ || $f =~ /-debug(?:info|source)-/;
|
|
my $n = `rpm -qp --qf '%{name}' ${\ sh_quote($f)} 2>/dev/null`;
|
|
my $match = ($name eq 'xCAT-genesis-base')
|
|
? ($n =~ /^xCAT-genesis-base-/) : ($n eq $name);
|
|
next unless $match;
|
|
my $evr = `rpm -qp --qf '%{epochnum}:%{version}-%{release}' ${\ sh_quote($f)} 2>/dev/null`;
|
|
chomp $evr;
|
|
$evrs{$evr} = 1 if $evr ne '';
|
|
}
|
|
my $evr;
|
|
return $evr unless %evrs;
|
|
die "Multiple EVRs of $name present in $dir: " . join(', ', sort keys %evrs)
|
|
. " (stale artifact not cleaned before the build)\n" if keys(%evrs) > 1;
|
|
($evr) = keys %evrs;
|
|
return $evr;
|
|
}
|
|
|
|
# rpm_vercmp_segment: ONE rpmvercmp segment comparison via rpm's own lua binding, returning -1/0/1.
|
|
# Used as the injected comparator for MockBuildUtils::evr_cmp so the EVR gate uses rpm's canonical
|
|
# version algorithm (epoch/release composition is done in evr_cmp). Long-bracket the args so any
|
|
# version char (. _ ~ ^ +) passes through literally; rpm versions never contain the ]==] sequence.
|
|
sub rpm_vercmp_segment {
|
|
my ($a, $b) = @_;
|
|
$a = '' unless defined $a;
|
|
$b = '' unless defined $b;
|
|
my $out = `rpm --eval '%{lua:print(rpm.vercmp([==[$a]==],[==[$b]==]))}' 2>/dev/null`;
|
|
chomp $out;
|
|
die "FATAL: rpm.vercmp gave no result for '$a' vs '$b'\n" unless $out =~ /^-?\d+$/;
|
|
return $out <=> 0;
|
|
}
|
|
|
|
# verify_rpms_checksig: cryptographically verify EVERY binary rpm in $dir with `rpmkeys --checksig`
|
|
# against an ISOLATED keyring holding only the signing key. This is the RPM-native integrity + origin
|
|
# check: it verifies each rpm's header/payload digests AND that the signature is by this key (NOKEY /
|
|
# NOT OK => a real failure, since the key IS imported). Returns @problems.
|
|
sub verify_rpms_checksig {
|
|
my ($dir, $keyname, $home, $native) = @_;
|
|
my @rpms = grep { !/\.src\.rpm$/ } glob("$dir/*.rpm");
|
|
return () unless @rpms;
|
|
my ($dbopt, $problem) = rpmkeys_keyring($keyname, $home);
|
|
return ($problem) if $problem;
|
|
my $publisher_db;
|
|
if ($native) {
|
|
my $trust = tempdir('native-publisher-XXXXXXXX', TMPDIR => 1, CLEANUP => 1);
|
|
my $ok = eval { $publisher_db = publisher_trust($native, $trust); 1; };
|
|
return ("SIGKEY: $@") unless $ok;
|
|
}
|
|
my @problems;
|
|
for my $rpm (@rpms) {
|
|
if ($native) {
|
|
my $id = rpm_identity($rpm);
|
|
my $owner = $native->{outputs}{$id->{name}};
|
|
if (!$owner) { push @problems, "Undeclared native output: $id->{name}"; next; }
|
|
my $node = $native->{nodes}{$owner};
|
|
if ($node->{type} eq 'publisher') {
|
|
my $ok = eval { verify_input($native, $node, $rpm, $publisher_db); 1; };
|
|
push @problems, $@ unless $ok;
|
|
next;
|
|
}
|
|
}
|
|
push @problems, rpm_checksig_problem($rpm, $dbopt);
|
|
}
|
|
return @problems;
|
|
}
|
|
|
|
# rpmkeys_keyring: an isolated rpm keyring holding only the signing key, as the --dbpath option for
|
|
# rpmkeys. Returns ($dbopt, undef), or (undef, $problem) when the key cannot be exported or imported.
|
|
sub rpmkeys_keyring {
|
|
my ($keyname, $home) = @_;
|
|
require_command('rpmkeys');
|
|
require_command('gpg');
|
|
my $tmpdb = tempdir('rpmkeys-XXXXXXXX', TMPDIR => 1, CLEANUP => 1);
|
|
my $h = ($home ne '') ? ' --homedir ' . sh_quote($home) : '';
|
|
my $keyfile = "$tmpdb/pubkey.asc";
|
|
system("gpg$h --batch --yes -a --export " . sh_quote($keyname) . ' > ' . sh_quote($keyfile) . ' 2>/dev/null');
|
|
return (undef, "SIGKEY: cannot export public key '$keyname' for rpmkeys --checksig") if !-s $keyfile;
|
|
my $dbopt = '--dbpath ' . sh_quote($tmpdb);
|
|
system("rpmkeys $dbopt --import " . sh_quote($keyfile) . ' >/dev/null 2>&1') == 0
|
|
or return (undef, "SIGKEY: rpmkeys --import of '$keyname' into the temp keyring failed");
|
|
return ($dbopt, undef);
|
|
}
|
|
|
|
# rpm_checksig_problem: `rpmkeys --checksig` of one rpm against the keyring, as a problem string or
|
|
# an empty list when its digests and signature verify with the signing key.
|
|
sub rpm_checksig_problem {
|
|
my ($rpm, $dbopt) = @_;
|
|
my $out = `rpmkeys $dbopt --checksig -v ${\ sh_quote($rpm)} 2>&1`;
|
|
return rpmkeys_checksig_problem(basename($rpm), $? >> 8, $out);
|
|
}
|
|
|
|
# repomd_observed_signer: run gpg --verify on the detached repomd signature and extract the identity
|
|
# of the key that actually signed it, as a primary-key fingerprint (the last field of the VALIDSIG
|
|
# status line). Returns '' when the .asc is absent or verification fails (both read as "unsigned").
|
|
sub repomd_observed_signer {
|
|
my ($asc, $file, $home) = @_;
|
|
return '' unless -f $asc && -f $file;
|
|
my $h = ($home ne '') ? ' --homedir ' . sh_quote($home) : '';
|
|
my $out = `gpg$h --status-fd=1 --verify ${\ sh_quote($asc)} ${\ sh_quote($file)} 2>/dev/null` // '';
|
|
# An EXPIRED or REVOKED key, or an expired signature, still emits VALIDSIG -- reject those
|
|
# explicitly so a no-longer-trustworthy signature is a problem, not a pass. A fully-good signature
|
|
# emits GOODSIG; the degraded cases emit EXPKEYSIG/REVKEYSIG/EXPSIG instead.
|
|
return '' if $out =~ /^\[GNUPG:\]\s+(?:EXPKEYSIG|REVKEYSIG|EXPSIG)\b/m;
|
|
for my $line (split /\n/, $out) {
|
|
# VALIDSIG <signing-fpr> <dates...> <primary-key-fpr>; the trailing field is the primary fpr.
|
|
if ($line =~ /^\[GNUPG:\]\s+VALIDSIG\s+(.*\S)\s*$/) {
|
|
my @f = split ' ', $1;
|
|
return $f[-1];
|
|
}
|
|
}
|
|
return '';
|
|
}
|
|
|
|
# verify_target_repo: the completeness + signature gate for ONE built per-target repo -- the single
|
|
# source of truth for "is this repo shippable?", replacing the old assert_required_deps + in-line
|
|
# version-pin loop. It does the IO (manifest parse, rpm_version, gpg --verify) and delegates every
|
|
# DECISION to the two PURE helpers: verify_repo_packages (missing/version) and verify_repo_signature
|
|
# (unsigned/wrongkey). Both problem lists are merged. Prints a one-line OK, or dies listing every
|
|
# problem. Both the automatic post-build gate (deploy_target) and the standalone --verify-repo mode
|
|
# call this, so there is exactly one gate implementation.
|
|
sub verify_target_repo {
|
|
my ($dir, $tgt, $manifest, $sig_required) = @_;
|
|
$manifest //= "$repo_root/packages-manifest.conf";
|
|
my %MAN = read_manifest($manifest);
|
|
my %req = %{ $MAN{$tgt} // {} };
|
|
die "FATAL: no manifest section for target '$tgt' in $manifest\n" if !%req;
|
|
my $native;
|
|
# The WHOLE manifest, deliberately -- the --skip-* flags are NOT applied here. They say what
|
|
# this INVOCATION built; they never say what the verified repository may be missing. Honouring
|
|
# them let a repo with no xCAT-genesis-base pass whenever the verifying run happened to carry
|
|
# --skip-genesis (PR #62 review). A package an earlier run built is still expected to be here.
|
|
my @names = sort keys %req;
|
|
my %present = repo_present_versions($dir, \@names);
|
|
# Full EPOCH:VERSION-RELEASE per package, so a manifest EVR constraint (e.g. genesis-base
|
|
# '>= 2:2.18.0', which %{VERSION}-only matching cannot enforce -- 2.* would accept a pre-2.18
|
|
# genesis) is checked with rpm's own version algorithm (PR #62 review). rpm_vercmp_segment is
|
|
# rpm's rpmvercmp; evr_cmp composes epoch/version/release around it.
|
|
my %present_evr = map { $_ => rpm_evr($dir, $_) } @names;
|
|
my %expected = map { $_ => $req{$_} } @names;
|
|
my @problems = verify_repo_packages(\%expected, \%present, \%present_evr, \&rpm_vercmp_segment);
|
|
if ($tgt eq 'openeuler-24.03-ppc64le') {
|
|
eval { $native = load_inputs($repo_root, \%req); 1 }
|
|
or push @problems, "Native input catalog: $@";
|
|
}
|
|
|
|
# Signature gate: the IO (gpg) lives here; the decision is the pure verify_repo_signature. The
|
|
# pipeline always signs, so a signed repo's repomd MUST be signed by --gpg-key-name. We resolve
|
|
# that key to a fingerprint and extract the fingerprint that actually signed repomd, then compare.
|
|
# Skipped with a printed note only when no gpg key/home is configured (nothing to check against).
|
|
if ($gpg_sign || $gpg_home ne '') {
|
|
require_command('gpg');
|
|
my $repomd = "$dir/repodata/repomd.xml";
|
|
my $asc = "$repomd.asc";
|
|
my $exp_fpr = gpg_key_fingerprint($gpg_key_name, $gpg_home);
|
|
# STRICT: the CLI key MUST resolve to exactly one fingerprint so we can confirm it signed the
|
|
# repo. Undef => absent or ambiguous in the keyring -> we cannot verify -> hard fail, never a
|
|
# presence-only pass.
|
|
if (!defined $exp_fpr) {
|
|
push @problems, "SIGKEY: cannot resolve --gpg-key-name '$gpg_key_name' to a single fingerprint (in the $gpg_home keyring?)";
|
|
} else {
|
|
my %exp_sig = ('repomd' => $exp_fpr);
|
|
my %obs_sig = ('repomd' => repomd_observed_signer($asc, $repomd, $gpg_home));
|
|
push @problems, verify_repo_signature(\%exp_sig, \%obs_sig);
|
|
|
|
# Per-rpm signature gate: a signed repomd over an unsigned or foreign-signed rpm still
|
|
# makes DNF reject that package at install time, so verify EVERY binary rpm -- not just the
|
|
# metadata -- is signed by this key (rpm reports the signing subkey id; accept any id of
|
|
# the key). Closes the "approves a repo DNF later rejects" gap (PR #62 review #4).
|
|
require_command('rpm');
|
|
# (a) RPM-native crypto verification: rpmkeys --checksig against an isolated keyring
|
|
# holding only this key verifies every rpm's digests AND that the signature is by the key.
|
|
push @problems, verify_rpms_checksig($dir, $gpg_key_name, $gpg_home, $native);
|
|
# (b) Explicit signer-id origin check kept alongside: assert each rpm's header signature
|
|
# key id is one of this key's ids (primary/subkey).
|
|
my $accept = gpg_key_ids($gpg_key_name, $gpg_home);
|
|
if (!%$accept) {
|
|
push @problems, "SIGKEY: cannot list key ids for '$gpg_key_name' to verify per-rpm signatures";
|
|
} else {
|
|
my @rpm_sigs = map { [ basename($_), rpm_signer_keyid($_) ] }
|
|
grep { !/\.src\.rpm$/ && (!$native || !native_publisher_rpm($native, $_)) } glob("$dir/*.rpm");
|
|
push @problems, verify_rpm_signatures(\@rpm_sigs, $accept);
|
|
}
|
|
}
|
|
} elsif ($sig_required) {
|
|
# Standalone --verify-repo advertises a signature check; with no keyring we cannot resolve the
|
|
# CLI key or read the signer, so refuse rather than silently pass (which would be a false PASS).
|
|
push @problems, "SIGKEY: --verify-repo requires --gpg-key-name + --gpg-home to check the repomd signature (none configured)";
|
|
} else {
|
|
print "[verify-repo] $tgt: no gpg key/home configured -- skipping repomd signature check\n";
|
|
}
|
|
|
|
if (@problems) {
|
|
print " - $_\n" for @problems;
|
|
die "FATAL: repo INCOMPLETE for $tgt at $dir (" . scalar(@problems) . " problem(s))\n";
|
|
}
|
|
print "[verify-repo] $tgt complete: " . scalar(@names)
|
|
. " required packages present + EVR-satisfied, repomd + every rpm checksig-verified, in $dir\n";
|
|
return 1;
|
|
}
|
|
|
|
sub native_publisher_rpm {
|
|
my ($plan, $rpm) = @_;
|
|
my $id = rpm_identity($rpm);
|
|
my $owner = $plan->{outputs}{$id->{name}} // return 0;
|
|
return $plan->{nodes}{$owner}{type} eq 'publisher';
|
|
}
|
|
|
|
sub reset_staging_repo {
|
|
my ($directory) = @_;
|
|
return unless -d $directory;
|
|
opendir(my $dh, $directory) or die "Cannot read $directory: $!\n";
|
|
my @stale = grep { /\.rpm\z/ } readdir($dh);
|
|
closedir($dh) or die "Cannot close $directory: $!\n";
|
|
return unless @stale;
|
|
print "Removing " . scalar(@stale) . " package(s) left in $directory by an earlier run\n";
|
|
for my $name (@stale) {
|
|
unlink("$directory/$name")
|
|
or die "Cannot remove stale package $directory/$name: $!\n";
|
|
}
|
|
return;
|
|
}
|
|
|
|
sub remove_genesis_packages {
|
|
my ($directory, $source, $keep_names) = @_;
|
|
return unless -d $directory;
|
|
my %keep = map { $_ => 1 } @{ $keep_names // [] };
|
|
opendir(my $dh, $directory) or die "Cannot read $directory: $!\n";
|
|
my @names = grep { /^xCAT-genesis-openembedded-.*\.rpm\z/ } readdir($dh);
|
|
closedir($dh) or die "Cannot close $directory: $!\n";
|
|
for my $name (@names) {
|
|
next if $keep{$name};
|
|
my $path = "$directory/$name";
|
|
next if $source && $path !~ /\.src\.rpm\z/;
|
|
next if !$source && $path =~ /\.src\.rpm\z/;
|
|
unlink($path) or die "Cannot remove stale Genesis package $path: $!\n";
|
|
}
|
|
}
|
|
|
|
sub publish_genesis_release_packages {
|
|
my ($prefix, $destination_root) = @_;
|
|
remove_genesis_packages($destination_root, $prefix eq 'srpm');
|
|
remove_genesis_packages($destination_root, 1) if $prefix eq 'rpm';
|
|
|
|
my $copied = 0;
|
|
for my $relative (genesis_release_files($prefix)) {
|
|
my $source = "$genesis_release/$relative";
|
|
my $destination = "$destination_root/" . basename($relative);
|
|
publish_file($source, $destination);
|
|
verify_release_file($genesis_release_checksums, $relative, $destination);
|
|
$copied++;
|
|
}
|
|
die "Genesis release has no $prefix packages\n" unless $copied;
|
|
return $copied;
|
|
}
|
|
|
|
# Dry runs copy nothing, but they must still report the shared packages a real run publishes.
|
|
sub preview_genesis_release_packages {
|
|
my ($prefix, $destination_root) = @_;
|
|
my @files = genesis_release_files($prefix);
|
|
die "Genesis release has no $prefix packages\n" unless @files;
|
|
for my $relative (@files) {
|
|
print "DRY-RUN publish Genesis release package: $genesis_release/$relative"
|
|
. " -> $destination_root/" . basename($relative) . "\n";
|
|
}
|
|
return scalar(@files);
|
|
}
|
|
|
|
sub genesis_release_files {
|
|
my ($prefix) = @_;
|
|
my @files = sort grep {
|
|
/^\Q$prefix\E\/xCAT-genesis-openembedded-[^\/]+\.rpm\z/
|
|
} keys %{$genesis_release_checksums};
|
|
return @files;
|
|
}
|
|
|
|
sub verify_genesis_release_packages {
|
|
my ($prefix, $destination_root) = @_;
|
|
my @files = genesis_release_files($prefix);
|
|
die "Genesis release has no $prefix packages\n" unless @files;
|
|
for my $relative (@files) {
|
|
my $destination = "$destination_root/" . basename($relative);
|
|
verify_release_file($genesis_release_checksums, $relative, $destination);
|
|
}
|
|
return scalar(@files);
|
|
}
|
|
|
|
sub collect_rpms {
|
|
my (%args) = @_;
|
|
my $roots = $args{roots} // [];
|
|
my $dest = $args{dest_dir} // die "collect_rpms missing dest_dir\n";
|
|
my $cell_arch = $args{arch} // die "collect_rpms missing arch\n";
|
|
my $is_dry = $args{dry_run} ? 1 : 0;
|
|
|
|
my %seen;
|
|
my $copied = 0;
|
|
my $skipped_src = 0;
|
|
my $skipped_foreign = 0;
|
|
my $missing_roots = 0;
|
|
|
|
for my $root (@{$roots}) {
|
|
if (!$root || !-d $root) {
|
|
$missing_roots++;
|
|
print "WARN: missing collection root: $root\n";
|
|
next;
|
|
}
|
|
my @rpms;
|
|
find(
|
|
sub {
|
|
return if !-f $_;
|
|
return if $_ !~ /\.rpm$/;
|
|
push @rpms, $File::Find::name;
|
|
},
|
|
$root,
|
|
);
|
|
@rpms = sort uniq(@rpms);
|
|
for my $rpm (@rpms) {
|
|
next if !-f $rpm;
|
|
if ($rpm =~ /\.src\.rpm$/) {
|
|
$skipped_src++;
|
|
next;
|
|
}
|
|
my $base = basename($rpm);
|
|
next if $genesis_release
|
|
&& $base =~ /^xCAT-genesis-openembedded-/;
|
|
if (!rpm_in_cell($rpm, $cell_arch)) {
|
|
$skipped_foreign++;
|
|
next;
|
|
}
|
|
next if $seen{$base}++;
|
|
if ($is_dry) {
|
|
print "DRY-RUN copy: $rpm -> $dest/$base\n";
|
|
$copied++;
|
|
next;
|
|
}
|
|
copy($rpm, "$dest/$base")
|
|
or die "Failed to copy $rpm to $dest/$base: $!\n";
|
|
$copied++;
|
|
}
|
|
}
|
|
|
|
return ($copied, $skipped_src, $missing_roots, $skipped_foreign);
|
|
}
|
|
|
|
sub collect_srpms {
|
|
my (%args) = @_;
|
|
my $roots = $args{roots} // [];
|
|
my $dest = $args{dest_dir} // die "collect_srpms missing dest_dir\n";
|
|
my $is_dry = $args{dry_run} ? 1 : 0;
|
|
|
|
my %seen;
|
|
my $copied = 0;
|
|
my $skipped_non_src = 0;
|
|
my $missing_roots = 0;
|
|
|
|
for my $root (@{$roots}) {
|
|
if (!$root || !-d $root) {
|
|
$missing_roots++;
|
|
print "WARN: missing source collection root: $root\n";
|
|
next;
|
|
}
|
|
my @rpms;
|
|
find(
|
|
sub {
|
|
return if !-f $_;
|
|
return if $_ !~ /\.rpm$/;
|
|
push @rpms, $File::Find::name;
|
|
},
|
|
$root,
|
|
);
|
|
@rpms = sort uniq(@rpms);
|
|
for my $rpm (@rpms) {
|
|
next if !-f $rpm;
|
|
if ($rpm !~ /\.src\.rpm$/) {
|
|
$skipped_non_src++;
|
|
next;
|
|
}
|
|
my $base = basename($rpm);
|
|
next if $genesis_release
|
|
&& $base =~ /^xCAT-genesis-openembedded-/;
|
|
next if $seen{$base}++;
|
|
if ($is_dry) {
|
|
print "DRY-RUN copy source: $rpm -> $dest/$base\n";
|
|
$copied++;
|
|
next;
|
|
}
|
|
copy($rpm, "$dest/$base")
|
|
or die "Failed to copy $rpm to $dest/$base: $!\n";
|
|
$copied++;
|
|
}
|
|
}
|
|
|
|
return ($copied, $skipped_non_src, $missing_roots);
|
|
}
|
|
|
|
sub resolve_xcat_source {
|
|
my ($requested, $root) = @_;
|
|
# Prefer the sibling ../xcat-core (the real layout: source/xcat-core beside source/xcat-dep)
|
|
# before the legacy xcat-source-code location.
|
|
my @candidates = (
|
|
$requested,
|
|
"$root/../xcat-core",
|
|
"$root/xcat-source-code",
|
|
);
|
|
for my $c (@candidates) {
|
|
next if !defined($c) || $c eq '';
|
|
my $abs = eval { abs_path($c) };
|
|
next if !$abs;
|
|
return $abs if -f "$abs/buildrpms.pl";
|
|
}
|
|
return eval { abs_path($requested) } || $requested;
|
|
}
|
|
|
|
# A lock the run holds until it exits. The owner pid lets the exit handlers tell the run from the
|
|
# build workers it forks, which inherit the lock list.
|
|
sub take_lock {
|
|
my ($path, $label) = @_;
|
|
my $lock = XCAT::NFSLock->acquire($path, timeout => $try_unlock_timeout, label => $label);
|
|
push(@HELD_LOCKS, $lock);
|
|
$LOCK_OWNER_PID //= $$;
|
|
return $lock;
|
|
}
|
|
|
|
# The repository cell a target deploys. The deploy, the carry-over and the cell lock all take the
|
|
# path from here, so the lock covers the directory the deploy writes.
|
|
sub target_cell {
|
|
my ($target) = @_;
|
|
my $native = openeuler_repo_subdir($target);
|
|
return "$repo_dep/$native" if defined $native;
|
|
my $profile = target_profile($target);
|
|
return "$repo_dep/rh$profile->{rel}/$profile->{arch}";
|
|
}
|
|
|
|
# The lock of a repository cell <dir>/<arch> sits beside it, as <dir>/.<arch>.lock: the deploy
|
|
# replaces the cell directory itself.
|
|
sub cell_lock_path {
|
|
my ($cell) = @_;
|
|
return dirname($cell) . '/.' . basename($cell) . '.lock';
|
|
}
|
|
|
|
# Release locks on every exit path, but only from the process that acquired them.
|
|
# Forked build workers inherit the lock list and must leave the parent's locks alone.
|
|
sub _restore_common_repository {
|
|
return unless defined($LOCK_OWNER_PID) && $$ == $LOCK_OWNER_PID;
|
|
remove_tree($COMMON_STAGE)
|
|
if $COMMON_STAGE && -d $COMMON_STAGE && !-l $COMMON_STAGE;
|
|
if ($COMMON_DESTINATION && $COMMON_BACKUP
|
|
&& !-e $COMMON_DESTINATION && -d $COMMON_BACKUP) {
|
|
rename($COMMON_BACKUP, $COMMON_DESTINATION);
|
|
} elsif ($COMMON_DESTINATION && $COMMON_BACKUP
|
|
&& -d $COMMON_DESTINATION && -d $COMMON_BACKUP) {
|
|
remove_tree($COMMON_BACKUP);
|
|
}
|
|
}
|
|
|
|
sub _release_locks_if_owner {
|
|
return unless defined($LOCK_OWNER_PID) && $$ == $LOCK_OWNER_PID;
|
|
$_->release for reverse(@HELD_LOCKS);
|
|
}
|
|
END {
|
|
_restore_common_repository();
|
|
_release_locks_if_owner();
|
|
}
|
|
|
|
sub read_os_release {
|
|
my ($path) = @_;
|
|
my %vals;
|
|
open my $fh, '<', $path or die "Cannot open $path: $!\n";
|
|
while (my $line = <$fh>) {
|
|
chomp $line;
|
|
next if $line =~ /^\s*#/;
|
|
next if $line !~ /=/;
|
|
my ($k, $v) = split /=/, $line, 2;
|
|
$v =~ s/^"(.*)"$/$1/;
|
|
$v =~ s/^'(.*)'$/$1/;
|
|
$vals{$k} = $v;
|
|
}
|
|
close $fh;
|
|
return %vals;
|
|
}
|
|
|
|
sub uniq {
|
|
my %seen;
|
|
return grep { defined($_) && !$seen{$_}++ } @_;
|
|
}
|
|
|
|
sub slurp_chomp {
|
|
my ($path) = @_;
|
|
open my $fh, '<', $path or die "Cannot read $path: $!\n";
|
|
my $line = <$fh>;
|
|
close $fh;
|
|
chomp $line if defined $line;
|
|
return $line // '';
|
|
}
|