2
0
mirror of https://github.com/xcat2/xcat-dep.git synced 2026-11-19 00:42:18 +00:00
Files
xcat-dep/MockBuildUtils.pm
T
Daniel Hilst 133d501354 test(goconserver): the native fixture cannot write the host mock cache
mockbuild.pl recreates the bootstrap cache that mock --scrub=bootstrap removes,
and it built the path from a literal /var/cache/mock. goconserver-openeuler.t
runs the builder under unshare --user --map-root-user, which maps root but leaves
the host cache unwritable, so the builder died in make_path and 12 of 160
assertions failed with "mkdir /var/cache/mock/<cfg>-bootstrap: Permission denied".

The path moved into MockBuildUtils::mock_bootstrap_cache_dirs, which takes the
cache root from XCAT_MOCK_CACHE_ROOT and falls back to /var/cache/mock, so the
production default does not change. run_case points it at a directory of each
case's own, and the builder now writes nothing outside the test's temporary tree.

Each of the six configurations asserts all three directories are present after the
scrub. Removing the recreate loop from mockbuild.pl fails exactly those six and
nothing else, so the assertions measure the fix rather than the fixture.

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
2026-11-08 18:16:10 -03:00

1336 lines
63 KiB
Perl

package MockBuildUtils;
# Build helpers use explicit settings and callbacks instead of script globals.
use strict;
use warnings;
use Exporter 'import';
use File::Basename qw(basename dirname);
use File::Copy qw(copy);
use File::Glob qw(bsd_glob);
use File::Find;
use File::Path qw(make_path remove_tree);
use lib dirname(__FILE__) . '/lib';
use XCAT::NFSLock ();
use Sys::Hostname;
use Digest::MD5 qw(md5_hex);
our @EXPORT_OK = qw(
install_deps_packages install_deps_command missing_perl_modules
sh_quote print_step
version_matches required_pkgs skipped_builder carry_over_rpms rpm_name rpm_arch rpm_source_rpm
source_package rpm_digests_ok mock_bootstrap_cache_dirs
have_rpm read_manifest
derive_target_from_repo_path verify_repo_packages verify_repo_signature verify_rpm_signatures
parse_evr evr_cmp evr_constraint_ok parse_pin rpmkeys_checksig_problem
rpm_version rpm_release rpm_sigmd5 rpm_is_signed restamp_release_line
cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix
createrepo_c_cmd sign_and_index_repo
target_profile FORCEARCH_TARGETS dep_repo_baseurl dep_repo_label
build_mock_uniqueext rpm_in_cell resolve_mock_cfg mock_chroot_uid mock_result_dirs
packaged_path
make_traversable_path epel_target_profile epel_cell_path cell_lock_path
perl_buildreq_compat_needed perl_buildreq_compat_spec perl_mock_cfg_text
native_owner_command NATIVE_BUILD_GID
openeuler_build_target openeuler_repo_subdir
goconserver_needs_pinned_go goconserver_pinned_go_buildrequires
host_build_payload_compressor
recover_common_repository
);
# install_deps_packages($os_id): the host packages mockbuild-all.pl needs to run at all, for the
# given /etc/os-release ID. Kept as data, beside the code that needs them, because the failure mode
# is a build host provisioned by hand: xcat-master-ub had no perl-File-Slurper and xcat-master-ppc no
# perl-IPC-Cmd, and each surfaced as a compile-time abort in the middle of a CD run.
sub install_deps_packages {
my ($os_id) = @_;
$os_id = '' unless defined $os_id;
return (install_deps_packages(''), '/usr/bin/systemd-nspawn') if lc($os_id) eq 'openeuler';
# The perl modules are what actually break a run; the rest is the toolchain the script drives.
return qw(perl perl-File-Slurper perl-IPC-Cmd perl-Parallel-ForkManager perl-Digest-SHA
mock createrepo_c tar findutils rpm rpm-build rpm-sign rpmdevtools gnupg2 wget git)
if $os_id =~ /^(?:opensuse|sles|sled)/;
return qw(perl perl-File-Slurper perl-IPC-Cmd perl-Parallel-ForkManager perl-Digest-SHA
mock createrepo_c tar findutils rpm rpm-build rpm-sign rpmdevtools
dnf-plugins-core gnupg2 wget git);
}
# install_deps_command($os_id): the argv that installs them, non-interactively.
sub install_deps_command {
my ($os_id) = @_;
$os_id = '' unless defined $os_id;
my @pkgs = install_deps_packages($os_id);
return ('zypper', '--non-interactive', 'install', '--no-recommends', @pkgs)
if $os_id =~ /^(?:opensuse|sles|sled)/;
return ('dnf', '--setopt=gpgcheck=1', '--setopt=*.gpgcheck=1', '--setopt=strict=1', '--setopt=install_weak_deps=False', '-y', 'install', @pkgs)
if lc($os_id) eq 'openeuler';
return ('dnf', '-y', 'install', @pkgs);
}
sub openeuler_build_target {
my ($os, $arch) = @_;
return undef unless lc($os->{ID} // '') eq 'openeuler';
my $version = $os->{VERSION} || $os->{VERSION_ID} || '';
if ($version =~ /\A(20|22|24)\.03\s+\(LTS(?:-SP([1-9][0-9]*))?\)\z/) {
$version = "$1.03" . (defined($2) ? "sp$2" : '');
}
my $target = "openeuler-$version-$arch";
openeuler_repo_subdir($target);
return $target;
}
sub openeuler_repo_subdir {
my ($target) = @_;
return undef unless defined($target) && $target =~ /\Aopeneuler-/;
die "Unsupported openEuler build target '$target'\n"
unless $target =~ /\Aopeneuler-((?:20|22|24)\.03(?:sp[1-9][0-9]*)?)-(x86_64|ppc64le)\z/;
return "openeuler$1/$2";
}
# goconserver_needs_pinned_go($target): true when the build must bring its own Go toolchain
# archive instead of taking one from BuildRequires.
#
# goconserver's committed go.mod names a go directive, and GOTOOLCHAIN=local never steps outside
# the Go the chroot already has, so a chroot whose newest Go is older than that directive cannot
# build goconserver at all. Two targets are in that position. openEuler packages no Go under any
# name. Leap 42.3 has no 'golang' package either: it names the compiler 'go', and its newest is
# go1.11 against a directive that asks for 1.25. Leap 15.6 resolves 'golang' to go1.25 and needs
# nothing.
sub goconserver_needs_pinned_go {
my ($target) = @_;
return 0 if !defined($target) || $target eq '';
return 1 if $target =~ /\Aopeneuler-/;
return 1 if $target =~ /\Aopensuse-leap-42\.3-/;
return 0;
}
# goconserver_pinned_go_buildrequires($target): what a pinned-Go build needs IN the chroot,
# beyond the toolchain archive it brings itself. undef when the chroot supplies Go.
#
# go build fetches the pinned modules over HTTPS, so the chroot must hold CA roots. openSUSE
# splits them: 'ca-certificates' is the tooling and carries no root, and the roots are in
# 'ca-certificates-mozilla'. Asking for the tooling alone is worse than asking for nothing.
# Measured in a Leap 42.3 chroot, Go's system pool holds 146 roots before 'ca-certificates' is
# installed and 0 after it, because the package takes the trust store over without filling it;
# go build then fails every module fetch with "x509: certificate signed by unknown authority".
# With 'ca-certificates-mozilla' the pool holds 201. EL and openEuler keep the roots in
# 'ca-certificates' itself and need no second package.
# host_build_payload_compressor($target): the rpm _binary_payload a HOST-built package must use
# for this target, or undef to keep the host default.
#
# A few packages repackage pre-built binaries and so run rpmbuild on the build host instead of in
# the target chroot. The host is EL10 and its rpm writes a zstd payload, but rpm gained zstd in
# 4.14: Leap 42.3 ships 4.11 and reports such a payload as "cpio: Bad magic". xz is readable by
# every rpm these targets ship, and is what the openSUSE core packages already use.
#-------------------------------------------------------------------------------
=head3 mock_bootstrap_cache_dirs
Descriptions: the bootstrap cache directories mock binds into the NEXT package's build.
mock --scrub=bootstrap removes them and does not put them back.
Arguments: $cfg -- the mock configuration name
$root -- the cache root. Defaults to XCAT_MOCK_CACHE_ROOT, then
/var/cache/mock. A test runs the builder in a user namespace that
cannot write the host cache, so it points this at its own directory.
Returns: the three directory paths, in a stable order
=cut
#-------------------------------------------------------------------------------
sub mock_bootstrap_cache_dirs {
my ($cfg, $root) = @_;
$root = $ENV{XCAT_MOCK_CACHE_ROOT} || '/var/cache/mock' unless defined $root and length $root;
return map { "$root/$cfg-bootstrap/$_" } qw(dnf_cache root_cache yum_cache);
}
sub host_build_payload_compressor {
my ($target) = @_;
return undef if !defined($target) || $target eq '';
return 'w9.xzdio' if $target =~ /\Aopensuse-leap-42\.3-/;
return undef;
}
sub goconserver_pinned_go_buildrequires {
my ($target) = @_;
return undef unless goconserver_needs_pinned_go($target);
my @req = qw(coreutils tar gzip ca-certificates);
push @req, 'ca-certificates-mozilla' if $target =~ /\Aopensuse-/;
return join(' ', @req);
}
# missing_perl_modules(@modules): those that cannot be loaded, in order. The point of --install-deps
# is that the run AFTER it cannot die on a missing module, so the modules are proven by loading
# them, not by trusting the package manager's exit code.
sub missing_perl_modules {
my (@modules) = @_;
my @missing;
for my $m (@modules) {
my $file = $m;
$file =~ s{::}{/}g;
$file .= '.pm';
eval { require $file; 1 } or push @missing, $m;
}
return @missing;
}
# sh_quote: single-quote a string for safe use in a shell command.
sub sh_quote {
my ($s) = @_;
$s = '' if !defined $s;
$s =~ s/'/'"'"'/g;
return "'$s'";
}
# The host group the native openEuler builds run as: xcatnative, gid 1000. The native mock overlays
# set chrootgid to it and chrootuid to the catalog's build_uid, so mock creates --resultdir and opens
# state.log, build.log and root.log in it as that uid and group.
use constant NATIVE_BUILD_GID => 1000;
# native_owner_command: wrap a native build step so it runs against $overlay instead of the target's
# own mock configuration, and with the identity mock will write results as.
#
# A dep builder makes its own result directories before mock starts -- ipxe-xcat/mockbuild.pl calls
# make_path on $work_dir/srpm and $work_dir/rpm and passes each one to mock as --resultdir. The
# script runs as root, so without the gid and the umask below those directories are root root 0755
# and mock, reloaded to chrootuid 1000, cannot open its logs in them.
sub native_owner_command {
my ($overlay, $mock_cfg, $command) = @_;
my $script = 'umask 0002 && mount --bind ' . sh_quote($overlay) . ' ' . sh_quote($mock_cfg)
. ' && exec sh -c ' . sh_quote($command);
return 'unshare --mount --propagation private -- setpriv --regid ' . NATIVE_BUILD_GID
. ' --clear-groups -- sh -c ' . sh_quote($script);
}
# print_step: print a step banner.
sub print_step {
my ($msg) = @_;
print "\n== $msg ==\n";
}
# version_matches: does the built version $got satisfy the manifest pin $want? $want may be an
# exact version (2.19.0), a shell-style glob (2.* or 2.19.*), or '*' (any). Globs support * and
# ? and are anchored. Used so xCAT-genesis-base can pin 2.* (its Version walks with xcat-core)
# while the real xcat-dep packages stay exactly pinned.
sub version_matches {
my ($got, $want) = @_;
return 1 if !defined($want) || $want eq '*';
return ($got eq $want) unless $want =~ /[*?]/;
my $re = quotemeta($want);
$re =~ s/\\\*/.*/g;
$re =~ s/\\\?/./g;
return $got =~ /\A$re\z/ ? 1 : 0;
}
# required_pkgs: given a list of manifest package names and the skip flags, return the subset
# that must actually be built and validated. A package whose builder was skipped is NOT required:
# --skip-genesis drops xCAT-genesis-base, --skip-perl drops perl-*, --skip-xcat-dep drops the dep
# builders (everything that is neither genesis nor perl). Pure function (flags passed in) so both
# the version-pin check and assert_required_deps use it and it is unit-testable.
sub required_pkgs {
my ($pkgs, $skip_genesis, $skip_perl, $skip_dep) = @_;
return grep {
!($skip_genesis && $_ eq 'xCAT-genesis-base')
&& !($skip_perl && /^perl-/)
&& !($skip_dep && $_ ne 'xCAT-genesis-base' && $_ !~ /^perl-/)
} @$pkgs;
}
# rpm_name: %{name} from the header of one rpm file, undef when rpm cannot read it.
sub rpm_name {
my ($rpm) = @_;
my $name = `rpm -qp --qf '%{name}' ${\ sh_quote($rpm)} 2>/dev/null`;
return (defined $name && $name ne '') ? $name : undef;
}
# rpm_digests_ok: whether the header and payload digests of one rpm file verify, signatures aside.
sub rpm_digests_ok {
my ($rpm) = @_;
my $out = `rpmkeys --checksig --nosignature -v ${\ sh_quote($rpm)} 2>&1`;
return ($? == 0 && $out !~ /NOT OK/i) ? 1 : 0;
}
# rpm_source_rpm: the %{sourcerpm} header of one rpm file, undef when rpm cannot read it. Every
# subpackage of one build shares it.
sub rpm_source_rpm {
my ($rpm) = @_;
my $srpm = `rpm -qp --qf '%{sourcerpm}' ${\ sh_quote($rpm)} 2>/dev/null`;
return (defined $srpm && $srpm =~ /\.src\.rpm$/) ? $srpm : undef;
}
# source_package: the source package name of a source rpm file name.
sub source_package {
my ($srpm) = @_;
return (defined $srpm && $srpm =~ /^(.+)-[^-]+-[^-]+\.src\.rpm$/) ? $1 : undef;
}
# skipped_builder: whether the builder that produces a source package was skipped by the flags.
# The classes are those of required_pkgs: xCAT-genesis-base, perl-*, and everything else the
# xcat-dep builders make. The OpenEmbedded Genesis is published from a release, not built here.
sub skipped_builder {
my ($source, $skipped) = @_;
return 0 if $source =~ /^xCAT-genesis-openembedded(?:-|$)/;
return $skipped->{genesis} ? 1 : 0 if $source =~ /^xCAT-genesis-base(?:-|$)/;
return $skipped->{perl} ? 1 : 0 if $source =~ /^perl-/;
return $skipped->{dep} ? 1 : 0;
}
# carry_over_rpms: copy the binary rpms a skipped builder published in $from into the run repository
# $to, for the builds whose source package the target manifest still names and of which the run
# carries no member yet. %$skipped holds the
# genesis, perl and dep flags; @$manifest_names is the target's manifest section; $name_of and
# $source_of map an rpm path to its package name and its source rpm (rpm_name and rpm_source_rpm
# in production, injectable for tests), and $trusted says whether an rpm may be re-signed at all
# (signed by the configured key in production), $arch is the cell's architecture and $arch_of
# reads an rpm's. A build is selected when one of its rpms is a manifest package, and is then
# carried whole, subpackages included, so a package the manifest dropped is not republished. The
# carry-over dies instead of publishing a partial or doubtful set: an unreadable rpm in the cell
# outside the OpenEmbedded Genesis family, a
# member of a selected build the key did not sign or of another architecture than the cell's or
# noarch, or a package name published more than once. Returns the copied basenames.
sub carry_over_rpms {
my ($from, $to, $skipped, $manifest_names, $name_of, $source_of, $trusted, $arch, $arch_of) = @_;
my %named = map { $_ => 1 } @$manifest_names;
my %staged = map { my $n = $name_of->($_); defined $n ? ($n => 1) : () }
grep { !/\.src\.rpm$/ } bsd_glob("$to/*.rpm");
my (%members, %selected, @problems);
for my $rpm (sort(bsd_glob("$from/*.rpm"))) {
next if $rpm =~ /\.src\.rpm$/;
# The OpenEmbedded Genesis is published from a release and pruned by name, so a stale file
# of that family is not read at all.
next if basename($rpm) =~ /^xCAT-genesis-openembedded-/;
my $srpm = $source_of->($rpm);
my $name = $name_of->($rpm);
my $source = source_package($srpm);
if (!defined $source || !defined $name) {
push @problems, basename($rpm) . ": header unreadable";
next;
}
next unless skipped_builder($source, $skipped);
push @{ $members{$srpm} }, { rpm => $rpm, name => $name };
$selected{$srpm} = 1
if $named{$name} || ($named{'xCAT-genesis-base'} && $name =~ /^xCAT-genesis-base-/);
}
my %seen;
for my $srpm (sort keys %selected) {
for my $m (@{ $members{$srpm} }) {
push @problems, basename($m->{rpm}) . ": not signed by the configured key" unless $trusted->($m->{rpm});
my $rpm_arch = $arch_of->($m->{rpm});
push @problems, basename($m->{rpm}) . ": architecture " . ($rpm_arch // 'unreadable') . " is not $arch or noarch"
unless defined $rpm_arch && ($rpm_arch eq $arch || $rpm_arch eq 'noarch');
push @problems, "$m->{name}: published more than once" if $seen{ $m->{name} }++ == 1;
}
}
die "FATAL: the published cell $from cannot be carried over:\n " . join("\n ", @problems) . "\n"
if @problems;
my @copied;
for my $srpm (sort keys %selected) {
# A build the run already carries in part is the run's, whole; mixing generations of one
# source package is never right.
next if grep { $staged{ $_->{name} } } @{ $members{$srpm} };
for my $m (@{ $members{$srpm} }) {
my $base = basename($m->{rpm});
copy($m->{rpm}, "$to/$base") or die "Failed to carry $m->{rpm} -> $to: $!\n";
push @copied, $base;
}
}
return sort @copied;
}
# verify_repo_packages: the PURE completeness-decision layer of the repo gate. Given the manifest's
# %expected { pkg => version-pin } and the %present { pkg => version-found-or-undef } actually in a
# built repo, return a list of human-readable problem strings (empty list = every package present at
# its pin):
# "MISSING <pkg> (manifest requires <pin>)" when %present has no (defined) version for <pkg>
# "VERSION <pkg>: repo has <got>, manifest pins <pin>" when present but !version_matches(got, pin)
# Uses version_matches (same semantics as the in-line manifest pin loop), so a '*' or glob pin is
# accepted exactly as there. No file/manifest I/O here -- the disk layer builds %present and passes
# both hashes in, keeping this unit-testable in isolation.
# parse_evr($s): split an EVR string "[epoch:]version[-release]" into ($epoch, $version, $release).
# epoch defaults to '0' when absent or '(none)'; release is undef when the string carries none (so a
# release-less constraint compares version-only). Neither version nor release may contain '-', so the
# single '-' cleanly separates them.
sub parse_evr {
my ($s) = @_;
$s = '' unless defined $s;
my ($epoch, $rest);
if ($s =~ /^\s*(\d+):(.*)$/) { ($epoch, $rest) = ($1, $2); }
else { ($epoch, $rest) = ('0', $s); }
$epoch = '0' if !defined $epoch || $epoch eq '' || lc($epoch) eq '(none)';
my ($ver, $rel) = split /-/, $rest, 2;
return ($epoch, $ver, $rel); # $rel undef when no release given
}
# evr_cmp($got, $want, $vercmp): compare two EVRs with rpm's labelCompare semantics -- epoch first
# (numeric), then version, then release -- returning -1/0/1 (got vs want). $vercmp->($a,$b) is an
# injected rpm-native segment comparator (rpm's rpmvercmp) returning -1/0/1, so this stays pure and
# unit-testable. Release is compared only when the CONSTRAINT specifies one (rpm's EVR semantics: a
# version-only requirement ignores the built release).
sub evr_cmp {
my ($got, $want, $vercmp) = @_;
my ($ge, $gv, $gr) = parse_evr($got);
my ($we, $wv, $wr) = parse_evr($want);
return (($ge <=> $we) <=> 0) if ($ge <=> $we) != 0; # epoch: numeric
my $c = $vercmp->($gv, $wv);
return $c if $c;
return 0 unless defined $wr && $wr ne ''; # constraint release-agnostic
$gr = '' unless defined $gr;
return $vercmp->($gr, $wr);
}
# evr_constraint_ok($got, $op, $want, $vercmp): does the observed EVR satisfy "<op> <want>"?
sub evr_constraint_ok {
my ($got, $op, $want, $vercmp) = @_;
my $c = evr_cmp($got, $want, $vercmp);
return $c >= 0 if $op eq '>=';
return $c > 0 if $op eq '>';
return $c <= 0 if $op eq '<=';
return $c < 0 if $op eq '<';
return $c == 0 if $op eq '=' || $op eq '==';
return undef; # unknown operator
}
# parse_pin($pin): classify a manifest version pin.
# '*' -> ('any')
# '<op> <evr>' (>=,>,<=,<,=) -> ('evr', $op, $evr) full EPOCH:VERSION-RELEASE constraint
# glob or exact version -> ('version') %{VERSION}-only match (version_matches)
sub parse_pin {
my ($pin) = @_;
return ('any') if !defined($pin) || $pin eq '*';
return ('evr', $1, $2) if $pin =~ /^\s*(>=|<=|==|=|>|<)\s*(\S+)\s*$/;
return ('version');
}
# The Leap release that builds each SLE family, and the family it serves. mock-core-configs ships
# no SLE config, because SLE repositories need a subscription, so each cell is built in the
# openSUSE counterpart of its family. The SLE major is not the Leap major: Leap 15.6 carries SLE
# 15 SP6's base, and Leap 42.3 carries SLE 12 SP3's -- rpm 4.11.2, perl 5.18.2, glibc 2.22.
my %LEAP_SLE_MAJOR = (
'42.3' => 12,
'15.6' => 15,
'16.0' => 16,
);
# The reverse, for a deploy directory handed to the gates by path alone. Derived from the table
# above so the two cannot disagree.
my %SUSE_BUILD_TARGET =
map { ("sles$LEAP_SLE_MAJOR{$_}" => "opensuse-leap-$_") } keys %LEAP_SLE_MAJOR;
sub derive_target_from_repo_path {
my ($dir) = @_;
my $tgt;
return $tgt unless defined $dir;
$tgt = "alma+epel-$1-$2" if $dir =~ m{/rh(\d+)/([^/]+)/*$};
if ($dir =~ m{/openeuler((?:20|22|24)\.03(?:sp[1-9][0-9]*)?)/(x86_64|ppc64le)/*$}) {
$tgt = "openeuler-$1-$2";
}
if ($dir =~ m{/(sles\d+)/([^/]+)/*$}) {
my $base = $SUSE_BUILD_TARGET{$1};
$tgt = "$base-$2" if $base;
}
return $tgt;
}
sub verify_repo_packages {
my ($expected, $present_ver, $present_evr, $vercmp) = @_;
$present_evr //= $present_ver;
my @problems;
for my $pkg (sort keys %$expected) {
my $pin = $expected->{$pkg};
my $got_ver = $present_ver->{$pkg};
if (!defined $got_ver) {
push @problems, "MISSING $pkg (manifest requires " . (defined($pin) ? $pin : '*') . ")";
next;
}
my ($kind, $op, $want) = parse_pin($pin);
if ($kind eq 'evr') {
my $got_evr = $present_evr->{$pkg} // $got_ver;
if (!$vercmp) {
push @problems, "EVR $pkg: no EVR comparator available to check '$op $want'";
} elsif (!evr_constraint_ok($got_evr, $op, $want, $vercmp)) {
push @problems, "EVR $pkg: repo has $got_evr, manifest requires $op $want";
}
} elsif ($kind eq 'version') { # VERSION glob/exact (unchanged)
push @problems, "VERSION $pkg: repo has $got_ver, manifest pins $pin"
if !version_matches($got_ver, $pin);
}
# 'any' -> accept
}
return @problems;
}
# rpmkeys_checksig_problem($name, $rc, $out): pure verdict for one `rpmkeys --checksig -v` run against
# an isolated keyring holding only the signing key. A clean rpm exits 0 and every digest/signature
# line reads OK; a tampered digest reads NOT OK; an rpm signed by another key (or unsigned) reads
# NOKEY. Return a problem string (or empty list) so the gate is testable without rpm.
sub rpmkeys_checksig_problem {
my ($name, $rc, $out) = @_;
$out = '' unless defined $out;
return () if ($rc // 0) == 0 && $out !~ /NOT OK|NOKEY|MISSING KEYS/i;
my $why = $out =~ /NOT OK/i ? 'digest/signature NOT OK'
: $out =~ /NOKEY/i ? 'NOKEY (unsigned or signed by an unaccepted key)'
: $out =~ /MISSING KEYS/i ? 'MISSING KEYS'
: "rpmkeys --checksig failed (rc=" . ($rc // '?') . ")";
return "BADSIG rpm $name: $why";
}
# verify_repo_signature: the PURE signature-decision layer of the repo gate. Given %expected
# { unit => expected signing-key identity } and %observed { unit => key that ACTUALLY signed (a
# string the script extracts from gpg), or undef/'' when unsigned / verification failed }, return a
# list of problem strings (empty = every unit signed by the expected key). For EL the single unit is
# 'repomd'. This does a plain string compare only -- it invokes NO gpg: the caller runs gpg --verify,
# extracts the observed key id, and resolves --gpg-key-name to the SAME identity form before calling.
# "UNSIGNED <unit> (expected <key>)" when %observed is absent/empty for <unit>
# "WRONGKEY <unit>: signed by <observed>, expected <key>" when both defined but differ
sub verify_repo_signature {
my ($expected, $observed) = @_;
my @problems;
for my $unit (sort keys %$expected) {
my $exp = $expected->{$unit};
my $obs = $observed->{$unit};
if (!defined($obs) || $obs eq '') {
push @problems, "UNSIGNED $unit (expected " . (defined($exp) ? $exp : '') . ")";
} elsif (defined($exp) && $obs ne $exp) {
push @problems, "WRONGKEY $unit: signed by $obs, expected $exp";
}
}
return @problems;
}
# verify_rpm_signatures: pure decision for the per-rpm signature gate. $rpm_sigs is an arrayref of
# [rpm_basename, observed_keyid|undef] (rpm reports the signing SUBKEY id); $accept is a hashref set
# of acceptable key ids (the signing key's primary + subkey ids, lowercased). Returns one problem per
# rpm that is unsigned or signed by a key not in the set. A signed repomd over unsigned/foreign-signed
# rpms still makes DNF reject the install, so the packages must be checked, not just the metadata.
sub verify_rpm_signatures {
my ($rpm_sigs, $accept) = @_;
my @problems;
for my $rs (@$rpm_sigs) {
my ($name, $kid) = @$rs;
if (!defined($kid) || $kid eq '') {
push @problems, "UNSIGNED rpm $name";
} elsif (!$accept->{ lc $kid }) {
push @problems, "WRONGKEY rpm $name: signed by $kid, expected one of "
. join('/', sort keys %$accept);
}
}
return @problems;
}
# have_rpm: is there a non-src rpm named <name>-... under $dir?
sub have_rpm {
my ($dir, $name) = @_;
my @m = grep { !/\.src\.rpm$/ } glob("$dir/${name}-*.rpm");
return scalar(@m) > 0;
}
# rpm_sigmd5: the SIGMD5 of an rpm -- the digest of its header+payload, independent of the GPG
# signature. Used to compare RPM identity/content: two rpms that share a basename but differ in
# content have different SIGMD5 (a bare filename match is not enough to call them identical).
sub rpm_sigmd5 {
my ($f) = @_;
return '' unless defined $f && -f $f;
my $v = `rpm -qp --qf '%{SIGMD5}' ${\ sh_quote($f)} 2>/dev/null`;
chomp $v;
return $v;
}
# rpm_is_signed: does the rpm carry a PGP/GPG header signature? SIGMD5 (above) is content-only and
# is identical whether or not the rpm is signed, so a cross-copied genesis that was copied but not
# yet signed (a crash between the copy and the rpmsign) still matches by SIGMD5. finalize uses this
# to treat such a rpm as NOT up to date so the copy+sign path re-runs and heals it.
sub rpm_is_signed {
my ($f) = @_;
return 0 unless defined $f && -f $f;
my $v = `rpm -qp --qf '%{SIGPGP}%{SIGGPG}' ${\ sh_quote($f)} 2>/dev/null`;
return 0 if !defined $v;
$v =~ s/\(none\)//g; # unsigned rpms report "(none)" for both tags
$v =~ s/\s+//g;
return $v ne '' ? 1 : 0;
}
# restamp_release_line: given a spec `Release: ...` line and a CD suffix (".snap<YYYYMMDDHHMM>.<n>"),
# return (new_line, changed). Idempotent: a line already ending in exactly $suffix is returned
# unchanged (changed=0). A line carrying a DIFFERENT prior .snap stamp (or several, from an earlier
# corrupted run) has it stripped before the new suffix is appended, so a re-run in a reused tree
# REPLACES the stamp instead of accumulating a second one (…snap...57 -> …snap...58, never
# …snap...57.snap...58). Only the Release token is touched; a non-Release line is returned as-is.
sub restamp_release_line {
my ($line, $suffix) = @_;
return ($line, 0) unless defined $line && $line =~ /^Release:\s*\S/i;
my $qs = quotemeta($suffix);
return ($line, 0) if $line =~ /$qs\s*$/; # already carries THIS suffix
(my $new = $line) =~ s/(?:\.snap\d{12}\.\d+)+(\s*)$/$1/; # drop any prior CD stamp(s)
$new =~ s/(^Release:\s*\S+)/$1$suffix/i;
return ($new, 1);
}
# rpm_version: %{version} of the built binary rpm named <name> under $dir (undef if absent).
# Skips src/debug rpms and confirms the rpm's real %{name} matches (glob can over-match).
# 'xCAT-genesis-base' matches the arch-suffixed rpm name (xCAT-genesis-base-x86_64 / -ppc64).
sub rpm_version {
my ($dir, $name) = @_;
my $glob = ($name eq 'xCAT-genesis-base')
? "$dir/xCAT-genesis-base-*.rpm"
: "$dir/${name}-*.rpm";
my %vers; # distinct %{version}s of the matching binary rpms
for my $f (sort glob($glob)) {
next if $f =~ /\.src\.rpm$/ || $f =~ /-debug(?:info|source)-/;
my $n = `rpm -qp --qf '%{name}' ${\ sh_quote($f)} 2>/dev/null`;
my $match = ($name eq 'xCAT-genesis-base')
? ($n =~ /^xCAT-genesis-base-/) : ($n eq $name);
next unless $match;
my $v = `rpm -qp --qf '%{version}' ${\ sh_quote($f)} 2>/dev/null`;
chomp $v;
$vers{$v} = 1 if $v ne '';
}
return undef unless %vers;
# More than one distinct version present means a stale artifact was not cleaned before the
# build -- a version pin could then pass against the wrong rpm and both could be shipped.
# (For genesis both arches share the same Version, so a normal x86_64+ppc64 pair is one entry.)
die "Multiple versions of $name present in $dir: " . join(', ', sort keys %vers)
. " (stale artifact not cleaned before the build)\n" if keys(%vers) > 1;
my ($v) = keys %vers;
return $v;
}
# rpm_release: %{release} of the built binary rpm named <name> under $dir (undef if absent). Same
# name-matching as rpm_version. Used to confirm a CD --build-number/--release-suffix bump actually
# landed in the built rpm's Release (validating %{VERSION} alone can't catch a silently un-bumped NVR).
sub rpm_release {
my ($dir, $name) = @_;
my $glob = ($name eq 'xCAT-genesis-base')
? "$dir/xCAT-genesis-base-*.rpm"
: "$dir/${name}-*.rpm";
for my $f (sort glob($glob)) {
next if $f =~ /\.src\.rpm$/ || $f =~ /-debug(?:info|source)-/;
my $n = `rpm -qp --qf '%{name}' ${\ sh_quote($f)} 2>/dev/null`;
my $match = ($name eq 'xCAT-genesis-base')
? ($n =~ /^xCAT-genesis-base-/) : ($n eq $name);
next unless $match;
my $r = `rpm -qp --qf '%{release}' ${\ sh_quote($f)} 2>/dev/null`;
chomp $r;
return $r if $r ne '';
}
return undef;
}
# read_manifest: parse packages-manifest.conf into %{ target => { package => version|'*' } }.
# INI format: [target] sections; "package=version|*" entries; blank / "#" / ";" lines ignored.
# Returns an empty hash if the file is absent (callers that build require a section per target).
sub read_manifest {
my ($path) = @_;
my %m;
return %m unless -f $path;
open my $fh, '<', $path or die "Cannot read manifest $path: $!\n";
my $sec;
while (my $line = <$fh>) {
$line =~ s/\r?\n\z//;
$line =~ s/^\s+|\s+$//g;
next if $line eq '' || $line =~ /^[#;]/;
if ($line =~ /^\[(.+?)\]$/) { $sec = $1; $m{$sec} ||= {}; next; }
next unless defined $sec;
my ($k, $v) = split /=/, $line, 2;
$k =~ s/\s+\z//;
$v = defined($v) ? ($v =~ s/^\s+//r) : '';
$m{$sec}{$k} = ($v ne '') ? $v : '*';
}
close $fh;
return %m;
}
sub createrepo_c_cmd {
my ($dir, $epoch) = @_;
return 'createrepo_c --update '
. '--revision ' . sh_quote($epoch) . ' --set-timestamp-to-revision '
. sh_quote($dir);
}
sub sign_and_index_repo {
my ($dir, $native, %options) = @_;
my $run = $options{run} // die "Repository signing requires a command runner\n";
my @rpms = grep { !/\.src\.rpm$/ } bsd_glob("$dir/*.rpm");
if ($native) {
require XCAT::NativeInputs;
require XCAT::BuildUtils;
my @built;
for my $rpm (@rpms) {
my $id = XCAT::NativeInputs::rpm_identity($rpm);
my $owner = $native->{outputs}{$id->{name}} // die "Undeclared native output: $id->{name}\n";
my $node = $native->{nodes}{$owner};
if ($node->{type} eq 'publisher') {
die "Publisher input changed before signing: $rpm\n"
unless XCAT::BuildUtils::digest_file($rpm) eq $node->{sha256};
} else {
push @built, $rpm;
}
}
@rpms = @built;
}
if ($options{gpg_sign} && @rpms) {
local $ENV{GNUPGHOME} = $options{gpg_home} if $options{gpg_home};
$run->('rpmsign --define ' . sh_quote("%_gpg_name $options{gpg_key_name}")
. ' --define ' . sh_quote("%__gpg $options{gpg_program}") . ' --addsign '
. join(' ', map { sh_quote($_) } @rpms));
}
$run->(createrepo_c_cmd($dir, $options{source_date_epoch}));
if ($options{gpg_sign}) {
local $ENV{GNUPGHOME} = $options{gpg_home} if $options{gpg_home};
my $repomd = "$dir/repodata/repomd.xml";
unlink "$repomd.asc" if -f "$repomd.asc";
$run->("gpg -a --detach-sign --default-key " . sh_quote($options{gpg_key_name}) . ' ' . sh_quote($repomd));
$run->("gpg -a --export " . sh_quote($options{gpg_key_name}) . " > " . sh_quote("$repomd.key"));
if ($native) {
$run->('cat ' . sh_quote($native->{publisher_key}) . ' >> ' . sh_quote("$repomd.key"));
}
}
}
# cross_copy_genesis: copy the noarch xCAT-genesis-base-<tarch>-*.rpm from $from into $to, dropping
# any stale foreign-arch genesis already in $to so the repo ends with exactly the fresh set.
# Returns the count of rpms newly copied (0 = already up to date, so the caller can skip
# re-indexing). Idempotent. $sign is an optional coderef ($rpm_path) invoked on each copied rpm
# (e.g. to re-sign it); pass undef to skip signing. Content is compared by SIGMD5, so a stale
# same-name rpm is refreshed rather than mistaken for up to date.
sub cross_copy_genesis {
my ($from, $to, $tarch, $sign) = @_;
my @src = grep { !/\.src\.rpm$/ } glob("$from/xCAT-genesis-base-$tarch-*.rpm");
return 0 if !@src;
my %want = map { basename($_) => $_ } @src;
my @existing = grep { !/\.src\.rpm$/ } glob("$to/xCAT-genesis-base-$tarch-*.rpm");
if (scalar(@existing) == scalar(keys %want)) {
my $up_to_date = 1;
for my $base (keys %want) {
my $dst = "$to/$base";
my $src_sig = rpm_sigmd5($want{$base});
# An empty SIGMD5 (unreadable rpm) means "cannot confirm identical" -> refresh rather
# than risk skipping on a false match (two '' would otherwise compare equal).
if (!-f $dst || $src_sig eq '' || $src_sig ne rpm_sigmd5($dst)) { $up_to_date = 0; last; }
# Content matches, but SIGMD5 cannot see the signature: a crash between the copy and the
# per-rpm sign leaves a same-content-but-UNSIGNED rpm. When a signer is configured, treat
# an unsigned dst as not-up-to-date so the copy+sign path re-runs and signs it.
if ($sign && !rpm_is_signed($dst)) { $up_to_date = 0; last; }
}
return 0 if $up_to_date;
}
for my $old (@existing) {
unlink $old or die "Failed to remove stale genesis $old: $!\n";
print "[finalize] - " . basename($old) . " (stale foreign-arch, removed from $to)\n";
}
my $copied = 0;
for my $base (sort keys %want) {
copy($want{$base}, "$to/$base")
or die "Failed to cross-copy genesis $want{$base} -> $to: $!\n";
print "[finalize] + $base ($from -> $to)\n";
$sign->("$to/$base") if $sign; # e.g. re-sign so the deploy gate never sees an unsigned rpm
$copied++;
}
return $copied;
}
# Cross-populate Genesis RPMs between matching legacy architecture repositories; skip openEuler cells.
# Both peers and their own Genesis RPMs are required. Empty roots are errors.
# The sign and reindex callbacks operate on copied RPMs and selected destination repositories.
# Architectures whose xCAT-genesis-base is cross-provisioned into every peer repo, so a management
# node can netboot nodes of any arch (issue #7610). Each entry maps the repo/subdir arch name to the
# genesis rpm's xCAT "tarch" (xCAT collapses ppc/ppc64le into tarch ppc64; x86_64 stays x86_64). This
# is the SINGLE SOURCE OF TRUTH for the cross-arch matrix -- to add an arch later (e.g. aarch64,
# riscv64) add an entry here AND wire its repo root into finalize_xcat_dep's %repo (the caller passes
# it). Discovery, the per-arch input gate, and the N-way cross-copy all iterate this list.
our @GENESIS_ARCHES = (
{ arch => 'x86_64', tarch => 'x86_64' },
{ arch => 'ppc64le', tarch => 'ppc64' },
);
sub finalize_xcat_dep {
my ($x86_64_repo, $ppc64le_repo, %opt) = @_;
my $sign = $opt{sign};
my $reindex = $opt{reindex};
# The arches whose cells this run writes. The others are read only: each host finalizes the
# cells it deploys, so it never holds a cell lock that only another host could reclaim.
my %known = map { $_->{arch} => 1 } @GENESIS_ARCHES;
my @only = @{ $opt{only} // [ map { $_->{arch} } @GENESIS_ARCHES ] };
for my $a (@only) {
die "FATAL: [finalize] no cross-arch genesis for arch '$a'\n" unless $known{$a};
}
my %write = map { $_ => 1 } @only;
my @dst_arches = grep { $write{ $_->{arch} } } @GENESIS_ARCHES;
print_step('Finalize xcat-dep: cross-arch genesis-base provisioning (issue #7610)');
print "x86_64-repo: $x86_64_repo\n";
print "ppc64le-repo: $ppc64le_repo\n";
# Per-arch repo root, keyed by the @GENESIS_ARCHES arch name. A new arch added to that list must
# also get its root wired here (today both roots are the same CD tree); a missing one fails loudly
# below rather than silently skipping.
my %repo = ( x86_64 => $x86_64_repo, ppc64le => $ppc64le_repo );
# Discover the UNION of OS dirs across ALL arch repos. Anchoring discovery on one arch let an
# <os> built for only the OTHER arch slip through unseen -- finalize then never cross-populated
# that cell and still exited 0 (PR #62 review). Every discovered <os> must carry every arch below.
my %os;
my @no_cross;
for my $a (@GENESIS_ARCHES) {
my $root = $repo{ $a->{arch} }
// die "FATAL: [finalize] no repo root configured for arch '$a->{arch}' (wire it in %repo)\n";
for my $dir (grep { -d "$_/$a->{arch}" } glob("$root/*")) {
my $name = basename($dir);
# openEuler builds one architecture per repository, and openSUSE ships no legacy
# Genesis, so neither has a payload to copy and neither has an architecture peer to
# wait for. cells.tsv gives the sles12 cell x86_64 alone. Both still pass the
# completeness and signature gates: mockbuild-all.pl verifies every finalized cell,
# sles* among them, after this returns.
if ($name =~ /^(?:openeuler|sles)/) {
push @no_cross, "$name/$a->{arch}";
next;
}
$os{$name} = 1;
}
}
print "[finalize] no cross-arch Genesis for: " . join(', ', sort @no_cross) . "\n" if @no_cross;
if (!%os && @no_cross) {
print "[finalize] no repository uses cross-arch Genesis; skipping\n";
return;
}
my $pairs = 0;
for my $osdir (sort keys %os) {
my %adir = map { $_->{arch} => "$repo{$_->{arch}}/$osdir/$_->{arch}" } @GENESIS_ARCHES;
# Pass 1 -- every arch peer repo dir must exist: a one-arch <os> is an incomplete input, not
# something to skip past (skipping would leave a cell without a foreign-arch genesis and still
# exit 0). Checked before the rpm pass so a missing peer is reported as such. Symmetric across
# all arches (catches an x86_64-only AND a ppc64le-only <os>).
for my $a (@GENESIS_ARCHES) {
die "FATAL: [finalize] $osdir: no $a->{arch} peer repo at $adir{$a->{arch}}\n"
. " (every arch must build every EL before finalize)\n" if !-d $adir{ $a->{arch} };
}
# Pass 2 -- every arch must have produced its OWN genesis rpm before finalize cross-populates
# them; otherwise a pair with no genesis rpms would make finalize a silent no-op that still
# exits 0. xCAT collapses ppc/ppc64le into tarch=ppc64, so match on each arch's tarch.
for my $a (@GENESIS_ARCHES) {
die "FATAL: [finalize] $osdir: no $a->{arch} xCAT-genesis-base rpm (tarch $a->{tarch}) in $adir{$a->{arch}}\n"
if !grep { !/\.src\.rpm$/ } glob("$adir{$a->{arch}}/xCAT-genesis-base-$a->{tarch}-*.rpm");
}
# N-way cross-copy: put each arch's genesis into EVERY other arch's repo dir.
my @summary;
for my $src (@GENESIS_ARCHES) {
for my $dst (@dst_arches) {
next if $src->{arch} eq $dst->{arch};
my $n = cross_copy_genesis($adir{$src->{arch}}, $adir{$dst->{arch}}, $src->{tarch}, $sign);
push @summary, "$n $src->{tarch} -> $dst->{arch}";
}
}
# Re-index+sign EVERY arch repo of this <os> each finalize, not only when an rpm was copied
# this run: a crash after a prior run's copy+sign but before its createrepo leaves the genesis
# rpm on disk (so cross_copy_genesis now returns 0) yet ABSENT from repomd.xml -- which no
# signature gate catches. Re-indexing is cheap (tiny repos) and idempotent, and heals that
# partial state; skipped only when no signer/indexer was injected.
if ($reindex) { $reindex->($adir{$_->{arch}}) for @dst_arches; }
print "[finalize] $osdir: " . join(', ', @summary) . "\n";
$pairs++;
}
die "FATAL: --finalize-xcat-dep found no <os> repo dir under any arch root\n"
. " --x86_64-repo '$x86_64_repo'\n --ppc64le-repo '$ppc64le_repo'\n" if $pairs == 0;
print_step('Finalize complete');
}
# bump_dep_release_suffix: append $suffix (e.g. ".snap202607161200.57") to the Release: line of
# every xcat-dep package spec under $repo_root, so the CD build stamps a fresh, monotonic NVR.
# Idempotent: a spec already carrying this exact suffix is left alone (a re-run in the same tree
# does not double-stamp). Preserves any %{?dist}/%{?distver} macro already on the line. Returns the
# count of specs newly stamped. Dies only if NO spec under $repo_root carries a Release: line.
# Pure (takes everything as args) so t/mockbuild-all.t can exercise it directly.
sub bump_dep_release_suffix {
my ($repo_root, $suffix) = @_;
my @specs;
# Only stamp xcat-dep's OWN specs. If someone checked xcat-core out NESTED under $repo_root (the
# legacy `xcat-source-code`/`xcat-core` layout), do NOT descend into it -- rewriting a core spec
# (e.g. xCAT-genesis-base.spec's dynamic Release) would break the lockstep with genesis-scripts.
find(sub {
if (-d $_ && ($_ eq 'xcat-core' || $_ eq 'xcat-source-code')) { $File::Find::prune = 1; return; }
push @specs, $File::Find::name if /\.spec$/ && -f $_;
}, $repo_root);
my ($with_release, $bumped, $already) = (0, 0, 0);
for my $spec (sort @specs) {
open my $in, '<', $spec or die "open $spec: $!\n";
my @lines = <$in>;
close $in;
my ($has_release, $changed) = (0, 0);
for my $line (@lines) {
# case-insensitive: some specs (e.g. Sys-Virt.spec) use a lowercase `release:`
next unless $line =~ /^Release:\s*\S/i;
$has_release = 1;
# restamp_release_line is idempotent (no-op if already carrying $suffix) and strips any
# prior .snap stamp before applying the new one, so a re-run with a different
# --build-number replaces rather than accumulates (unit-tested in t/mockbuild-all.t).
my ($new, $ch) = restamp_release_line($line, $suffix);
if ($ch) { $line = $new; $changed = 1; }
last; # only the first Release: line
}
$with_release++ if $has_release;
$already++ if $has_release && !$changed;
next unless $changed;
# atomic write (temp + rename) so a concurrent per-arch build on the shared NFS tree never
# sees a torn spec; identical suffix -> identical content, so last-writer-wins is safe. The
# temp name carries the hostname AND pid: the two arch build hosts share the NFS tree and can
# reuse the same pid, so pid alone could collide across hosts.
my $tmp = "$spec.bump." . hostname() . ".$$";
open my $out, '>', $tmp or die "open> $tmp: $!\n";
print {$out} @lines;
close $out;
rename $tmp, $spec or die "rename $tmp -> $spec: $!\n";
$bumped++;
}
print "Release bump '$suffix': $bumped newly stamped, $already already stamped, of $with_release spec(s) with a Release line under $repo_root\n";
# Only a genuine "no dep specs at all" is fatal. All-already-stamped is the expected idempotent
# case (re-run in the same tree, or the other arch bumped first) -- NOT an error.
die "FATAL: --build-number given but NO spec carried a Release: line under $repo_root (wrong tree?)\n"
if $with_release == 0;
return $bumped;
}
# build_mock_uniqueext: a mock --uniqueext UNIQUE per (run, build-step) so concurrent mock builds
# never share a chroot root (/var/lib/mock/<cfg>-<uniqueext>). $run is the per-target run id
# (e.g. "alma+epel-8-ppc64le-<n>"), $seq orders the step, $label names the package.
#
# The run id must NOT be blindly tail-truncated. The per-target id leads with the EL/arch token, and
# for the 7-char "ppc64le" arch the EL digit is exactly what falls off the front of a keep-the-last-24
# truncation -- so alma+epel-{8,9,10}-ppc64le all collapse to the same run part. That is catastrophic
# for goconserver, which compiles EVERY EL in the el10 chroot (build_cfg rewritten to -10-): the
# chroot NAME is then identical across the three ELs, and the uniqueext is the ONLY thing keeping
# their roots apart, so three concurrent el8/el9/el10 ppc64le goconserver builds race in one root.
# When the id is too long, keep a readable leading token AND append a short digest of the FULL id, so
# distinct ids always yield distinct uniqueext regardless of where in the string they differ.
# rpm_arch($rpm): the architecture of an rpm. The header decides when the file can be read, so a
# renamed file does not pass for another architecture; a bare file name falls back to its suffix.
sub rpm_arch {
my ($rpm) = @_;
return unless defined $rpm;
if (-f $rpm) {
my $arch = `rpm -qp --qf '%{ARCH}' ${\ sh_quote($rpm)} 2>/dev/null`;
chomp $arch;
return $arch if $arch ne '';
}
my ($arch) = $rpm =~ /\.([A-Za-z0-9_]+)\.rpm$/;
return $arch;
}
# rpm_in_cell($rpm, $target_arch): whether an rpm belongs in the repository cell of $target_arch.
# A noarch builder run in another architecture's chroot (the x86 boot loaders for riscv64) can
# emit that chroot's native rpms beside the noarch one; only noarch and the cell's own
# architecture are kept.
sub rpm_in_cell {
my ($rpm, $target_arch) = @_;
my $arch = rpm_arch($rpm);
return 0 unless defined $arch && defined $target_arch;
return 1 if $arch eq 'noarch';
return $arch eq $target_arch ? 1 : 0;
}
sub build_mock_uniqueext {
my ($run, $seq, $label) = @_;
my $run_part = defined($run) ? $run : 'run';
$run_part =~ s/[^A-Za-z0-9_.-]+/-/g;
$run_part =~ s/^-+|-+$//g;
$run_part = 'run' if $run_part eq '';
if (length($run_part) > 24) {
my $digest = substr(md5_hex($run_part), 0, 8);
(my $head = substr($run_part, 0, 15)) =~ s/-+$//;
$run_part = "$head-$digest";
}
my $label_part = defined($label) ? $label : 'step';
$label_part =~ s/[^A-Za-z0-9_.-]+/-/g;
$label_part =~ s/^-+|-+$//g;
$label_part = 'step' if $label_part eq '';
$label_part = substr($label_part, 0, 20) if length($label_part) > 20;
my $idx = defined($seq) ? int($seq) : 0;
$idx = 0 if $idx < 0;
return sprintf("mba-%02d-%s-%s", $idx, $run_part, $label_part);
}
#--------------------------------------------------------------------------------
=head3 recover_common_repository
Descriptions:
Put back the common tree that an interrupted publication moved aside, and
remove the staging trees it left. Runs only under the common lock, so it
never removes the staging tree of a run that is still publishing.
Arguments:
$base: the --repo-dep directory
Returns:
1 when the recovery ran, 0 when another run holds the common lock.
=cut
#--------------------------------------------------------------------------------
sub recover_common_repository {
my ($base) = @_;
my $path = "$base/.common-publish.lock";
my $lock = eval { XCAT::NFSLock->acquire($path, label => 'common lock') };
unless ($lock) {
# The lock is live or unproven: this is not the place to offer its removal.
print "common recovery skipped: another run holds $path\n";
return 0;
}
my $destination = "$base/common";
my @backups = sort {
((stat($a))[9] // 0) <=> ((stat($b))[9] // 0)
} grep { -d $_ && !-l $_ } bsd_glob("$base/.common.previous.*");
if (!-e $destination && !-l $destination && @backups) {
my $backup = pop(@backups);
unless (rename($backup, $destination)) {
my $error = $!;
$lock->release;
die "Cannot restore interrupted common repository $backup: $error\n";
}
}
remove_tree($_) for grep { -d $_ && !-l $_ } @backups;
for my $staging (bsd_glob("$base/.common.*")) {
next if $staging =~ m{/\.common\.previous\.};
remove_tree($staging) if -d $staging && !-l $staging;
}
$lock->release;
return 1;
}
# epel_target_profile($target, $host_arch): the EL release, the rpm architecture, and whether
# mock cross-builds, for one EPEL target name on one build host. Returns undef for a name that is
# not an EPEL target, so the caller keeps its own error.
#
# The target name carries the architecture of the rpms it produces, and that architecture decides
# the repository cell the build deploys to. A host of another architecture cross-builds the target
# through mock's own --forcearch and qemu-user; the rpms are still the target's architecture.
sub epel_target_profile {
my ($target, $host_arch) = @_;
return undef unless defined $target && defined $host_arch;
my ($rel, $arch) = $target =~ /epel-(\d+)-(.+)\z/;
return undef unless defined $rel && defined $arch;
return { rel => $rel, arch => $arch, forcearch => ($arch eq $host_arch ? 0 : 1) };
}
# epel_cell_path($repo_dep, $profile): the repository cell an EPEL target deploys,
# <repo-dep>/rh<rel>/<arch>, from the profile epel_target_profile returns.
#
# The deploy, the carry-over and the cell lock all take the path from here, so the lock covers the
# directory the deploy writes. Two targets share a lock exactly when they share a cell.
sub epel_cell_path {
my ($repo_dep, $profile) = @_;
return undef unless defined $repo_dep && $profile;
return "$repo_dep/rh$profile->{rel}/$profile->{arch}";
}
# cell_lock_path($cell): the lock of a repository cell <dir>/<arch> sits beside it, as
# <dir>/.<arch>.lock, because the deploy replaces the cell directory itself.
sub cell_lock_path {
my ($cell) = @_;
return dirname($cell) . '/.' . basename($cell) . '.lock';
}
# resolve_mock_cfg($os_id, $rel, $arch[, $cfg_dir]): the mock config for EL release $rel on this
# host, <id>+epel-<rel>-<arch>. /etc/os-release says 'almalinux' where mock-core-configs names the
# file 'alma', so the short form is tried too. $cfg_dir defaults to /etc/mock.
#-------------------------------------------------------------------------------
=head3 packaged_path
Descriptions: find where a package installed one of its documentation or
licence files. The directory is distribution policy: EL puts the
documents of a package under /usr/share/doc, openSUSE under
/usr/share/doc/packages, so a build-time check cannot name it. Read it
back from the package instead.
Arguments:
$paths: array ref of absolute paths, as "rpm -qpd" or "rpm -qpL" prints them
$subdir: the last directory the package owns, normally the package name
$want: the path of the wanted file below $subdir, e.g. "ipxe/COPYING"
Returns:
the one path that ends in "/$subdir/$want". Dies when the number of
matches is not one, and names how many paths were searched.
=cut
#-------------------------------------------------------------------------------
sub packaged_path {
my ($paths, $subdir, $want) = @_;
my @all = @{ $paths || [] };
my @hit = grep { m{^/.*/\Q$subdir\E/\Q$want\E\z} } @all;
die "packaged_path: no packaged path ends in $subdir/$want (searched "
. scalar(@all) . ")\n" if !@hit;
die "packaged_path: $subdir/$want matches " . scalar(@hit)
. " packaged paths: @hit\n" if @hit > 1;
return $hit[0];
}
sub resolve_mock_cfg {
my ($os_id, $rel, $arch, $cfg_dir) = @_;
$cfg_dir //= '/etc/mock';
my %short_forms = (
almalinux => 'alma',
'centos-stream' => 'centos-stream',
rocky => 'rocky',
);
# Resolve by CONFIG-FILE existence, not by running `mock --print-root-path`: the latter can fail
# transiently (bootstrap chroot setup, a concurrent mock holding a lock) and made el10 flakily
# "resolve" to the long form that has no .cfg. Checking <cfg_dir>/<cfg>.cfg is deterministic.
for my $id ($os_id, (exists $short_forms{$os_id} ? ($short_forms{$os_id}) : ())) {
my $candidate = "${id}+epel-${rel}-${arch}";
if (-f "$cfg_dir/${candidate}.cfg") {
print "Mock config resolved: $candidate\n" if $id ne $os_id;
return $candidate;
}
}
my $short = $short_forms{$os_id} // $os_id;
die "Could not find mock config for ${os_id}+epel-${rel}-${arch} "
. "(tried $cfg_dir/${os_id}+epel-${rel}-${arch}.cfg and $cfg_dir/${short}+epel-${rel}-${arch}.cfg)\n";
}
# What a target builds. The mock-core-configs targets (<os>+epel-<rel>-<arch>) build every dep
# natively on the host arch. The forcearch targets shipped in mock-configs/ cross-build another arch
# that has no EPEL: the EPEL-only perl deps of xCAT are built for it
# (mockbuild-perl-packages.pl --epel-gap), and the noarch deps, the x86 boot loaders among them, are
# built in the native, EPEL-free chroot of the same release (the rpms are identical for every arch
# and an emulated build is an order of magnitude slower). See BUILD.md ("riscv64").
my %FORCEARCH_TARGETS = (
'rocky-10-riscv64-xcat' => {
rel => 10,
arch => 'riscv64',
# x86_64 only, as the mock config admits: syslinux-xcat builds on x86 and ppc64le alone.
noarch_cfg => 'rocky-10-x86_64',
dep_builders => [qw(elilo-xcat grub2-xcat ipmitool-xcat syslinux-xcat goconserver conserver-xcat xnba-undi ipxe-xcat uftp)],
required => [qw(ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi ipxe-xcat
perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB)],
},
);
sub FORCEARCH_TARGETS { return %FORCEARCH_TARGETS }
#-------------------------------------------------------------------------------
=head3 target_profile
Descriptions: What one mock target builds, and the repository cell it publishes
into.
Arguments:
$target - a mock target, e.g. alma+epel-10-x86_64
$host_arch - the architecture of the build host, from uname -m
Returns: a hash reference: rel, arch, cell, noarch_cfg, forcearch, epel,
dep_builders, required.
=cut
#-------------------------------------------------------------------------------
sub target_profile {
my ($target, $host_arch) = @_;
if (my $native = openeuler_repo_subdir($target)) {
my ($version, $arch) = $target =~ /\Aopeneuler-(.*)-([^-]+)\z/;
die "Native target '$target' requires a $arch build host, found $host_arch\n"
unless $arch eq $host_arch;
return {
rel => $version,
arch => $arch,
cell => $native,
noarch_cfg => $target,
forcearch => 0,
epel => 0,
dep_builders => [qw(grub2-xcat ipmitool-xcat syslinux-xcat goconserver conserver-xcat xnba-undi ipxe-xcat uftp python3-scp)],
required => [qw(ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi ipxe-xcat
perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB)],
};
}
# Leap has no EPEL and builds natively; the dep set is the EL one without elilo-xcat, which
# packages an EL boot loader. 94daf29 added this mapping and a1ab393 removed it while scoping
# its pull request to EL, and the pipeline has asked for the target ever since.
if (my ($leap, $arch) = $target =~ /\Aopensuse-leap-(\d+\.\d+)-([A-Za-z0-9_]+)\z/) {
die "openSUSE target '$target' requires a $arch build host, found $host_arch\n"
unless $arch eq $host_arch;
# An unmapped Leap release must not publish into sles<leap major>: that would serve one
# family's rpms as another's.
my $rel = $LEAP_SLE_MAJOR{$leap}
or die "openSUSE target '$target' names no SLE family: add Leap $leap to"
. " %LEAP_SLE_MAJOR\n";
return {
rel => $rel,
arch => $arch,
cell => "sles$rel/$arch",
noarch_cfg => $target,
forcearch => 0,
epel => 0,
dep_builders => [qw(grub2-xcat ipmitool-xcat syslinux-xcat goconserver conserver-xcat xnba-undi ipxe-xcat)],
required => [qw(ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi ipxe-xcat
perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB)],
};
}
if (my $fa = $FORCEARCH_TARGETS{$target}) {
return {
%{$fa},
cell => "rh$fa->{rel}/$fa->{arch}",
forcearch => 1,
epel => 0,
};
}
# The target name, not the host, decides the architecture of the rpms and so the repository
# cell they deploy to. A host of another architecture cross-builds the target through mock's
# own --forcearch, which is what gives the emulated build its own step timeout.
my $epel = epel_target_profile($target, $host_arch);
die "Could not parse EL release from target '$target'\n" unless $epel;
return {
rel => $epel->{rel},
arch => $epel->{arch},
cell => "rh$epel->{rel}/$epel->{arch}",
noarch_cfg => $target,
forcearch => $epel->{forcearch},
epel => 1,
dep_builders => [qw(elilo-xcat grub2-xcat ipmitool-xcat syslinux-xcat goconserver conserver-xcat xnba-undi ipxe-xcat uftp)],
# xCAT Requires all of these on every arch, and every one of them builds natively on
# every arch (the noarch deps -- grub2-xcat, xnba-undi, ipxe-xcat -- just repackage committed
# artifacts), so a self-sufficient per-arch build produces the whole set.
required => [qw(ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi ipxe-xcat
perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB)],
};
}
#-------------------------------------------------------------------------------
=head3 dep_repo_baseurl / dep_repo_label
Descriptions: Where a published dep cell says it came from, and what it calls
itself.
xcat.org serves Leap from the sles channel and everything else
from the yum channel. A cell that names the wrong channel points
every client at a tree that does not hold it.
Arguments:
$cell - the repository cell, e.g. rh10/x86_64, sles15/x86_64
Returns: the baseurl, or the human label.
=cut
#-------------------------------------------------------------------------------
sub dep_repo_baseurl {
my ($cell) = @_;
my $channel = $cell =~ m{^sles\d} ? 'sles' : 'yum';
return "https://xcat.org/files/xcat/repos/$channel/devel/xcat-dep/$cell";
}
sub dep_repo_label {
my ($cell) = @_;
my ($dir, $arch) = split m{/}, $cell, 2;
return defined $arch ? "$dir $arch" : $cell;
}
# The Fedora perl source rpms BuildRequire perl-generators and perl-interpreter. openSUSE has
# neither: there perl provides the interpreter and rpm generates the perl dependencies itself, so
# dnf builddep stops before the build. A build-only rpm provides those names and pulls the real
# perl. It is never shipped: only the mock configuration of a SUSE target sees its repository.
sub perl_buildreq_compat_needed {
my ($base_cfg) = @_;
return 0 unless defined $base_cfg;
return $base_cfg =~ /opensuse|sles|suse/i ? 1 : 0;
}
sub perl_buildreq_compat_spec {
return <<'SPEC';
Name: xcat-perl-buildreq-compat
Version: 1
Release: 1
Summary: Build-only compat: provide Fedora perl BuildRequires names on SUSE
License: MIT
BuildArch: noarch
Provides: perl-generators
Provides: perl-interpreter
Provides: perl-devel
Requires: perl
%description
Satisfies the perl-generators, perl-interpreter and perl-devel BuildRequires of
Fedora perl source rpms in an openSUSE chroot. SUSE ships the perl development
headers inside perl itself. Never shipped.
%files
%changelog
SPEC
}
# The deterministic mock configuration one perl build uses. $compat_repo is the directory
# perl_buildreq_compat_needed asked the caller to build; a SUSE target without one is refused,
# because building without the compat is the failure this exists to prevent.
sub perl_mock_cfg_text {
my ($base_cfg, $epoch, $compat_repo) = @_;
my $text = "include('/etc/mock/${base_cfg}.cfg')\n"
. "config_opts['environment']['SOURCE_DATE_EPOCH'] = '$epoch'\n";
return $text unless perl_buildreq_compat_needed($base_cfg);
die "perl_mock_cfg_text: $base_cfg needs the perl BuildRequires"
. " compat repository and none was given\n"
unless defined $compat_repo && length $compat_repo;
# priority=1 so the compat rpm wins over anything of the same name in the target repositories.
$text .= "config_opts['dnf.conf'] += \"\"\"\n[xcat-buildreq-compat]\n"
. "name=xcat perl BuildRequires compat\n"
. "baseurl=file://$compat_repo\nenabled=1\ngpgcheck=0\npriority=1\n\"\"\"\n";
return $text;
}
# mock's own loader reads the configuration, so include() and the assignments around it resolve as
# they do in a build. With no chrootuid, the loader returns the caller, as mock does: as root, the
# user that SUDO_UID, USERHELPER_UID or PKEXEC_UID names.
my $MOCK_CHROOTUID_LOADER = <<'PYTHON';
import os
import sys
try:
from mockbuild.config import MOCKCONFDIR, load_config
from mockbuild.uid import setup_uid_manager
except ImportError:
sys.exit('mock is not installed: cannot import mockbuild.config')
args = (MOCKCONFDIR, sys.argv[1])
if os.geteuid() == 0:
# As mock main(): load the configuration as the user that sudo names.
opts = setup_uid_manager().run_in_subprocess_without_privileges(load_config, *args)
else:
opts = load_config(*args)
print(opts['chrootuid'])
PYTHON
# mock_chroot_uid($cfg): the uid mock builds as with configuration file $cfg.
sub mock_chroot_uid {
my ($cfg) = @_;
open my $pipe, '-|', 'python3', '-c', $MOCK_CHROOTUID_LOADER, $cfg
or die "Cannot run python3 to load $cfg: $!\n";
my $out = do { local $/; <$pipe> } // '';
close $pipe or die "mock cannot load $cfg\n";
$out =~ /\A(\d+)\n\z/ or die "mock gives no chrootuid for $cfg\n";
return $1;
}
# make_traversable_path(@dirs): make_path(@dirs), then give mode 0755 to each directory it created,
# because make_path applies the umask. Directories that already exist keep their mode.
# Returns the directories it created.
sub make_traversable_path {
my @made = make_path(@_);
for my $dir (@made) {
chmod 0755, $dir or die "Cannot make $dir traversable: $!\n";
}
return @made;
}
# mock_result_dirs($dir, $cfg): create the three --resultdir directories one package build gives
# mock, below $dir, owned by the uid mock builds as with $cfg. mock 6.8 reloads its uid manager from
# chrootuid, so it opens state.log, build.log and root.log in --resultdir as that uid. The parents
# that this sub creates get mode 0755, so that uid can reach the result directories.
sub mock_result_dirs {
my ($dir, $cfg) = @_;
my $uid = mock_chroot_uid($cfg);
my @dirs = map { "$dir/$_" } qw(srpm restamp-srpm rpm);
make_traversable_path($dir);
make_path(@dirs);
for my $result (@dirs) {
chown $uid, -1, $result or die "Cannot give $result to uid $uid: $!\n";
}
return @dirs;
}
1;