xCAT/xcat-dep
2
0
Fork 0
mirror of https://github.com/xcat2/xcat-dep.git synced 2026-09-12 12:36:23 +00:00
Code Issues Projects Releases Wiki Activity

fix(mockbuild-all): keep the published rpms of a skipped builder

Browse Source 
Each run collects the rpms it built into its run repository, and deploy
stages the cell from that repository and gates it on the whole manifest,
which a skip run cannot satisfy: with --skip-perl there are no perl
packages, the gate reports them missing, and the deploy fails after the
build succeeded. The gate is right to check the whole manifest, since the
flags describe what this invocation built and not what the cell may lack.

After collection, the run repository now takes the binary rpms a skipped
builder published in the cell: whole builds, so subpackages the manifest
does not name stay too; only builds whose source package the target
manifest still names, so a dropped package is not republished; and only
builds of which the run carries no member yet, so generations of one build
never mix. The bump check, createrepo, the tarball and the deploy gate
then see the same complete set. The carry-over stops the run rather than
publish a partial or doubtful set: an rpm in the cell whose header cannot
be read, apart from the OpenEmbedded Genesis family that is pruned by
name, a member of a selected build the signing key did not sign, by signer
id and by rpmkeys --checksig, or whose digests do not verify when no key
is configured, a member of another architecture than the cell or noarch,
or a package published at more than one version. A package that was never
published is still reported missing, and a full run is unchanged. The
zero-artifact check applies only when a builder ran, so a run that skips
both package builders reaches the deploy.

Signed-off-by: Vinícius Ferrão <2031761+viniciusferrao@users.noreply.github.com>
This commit is contained in:
Vinícius Ferrão
2026-09-09 22:07:28 -03:00
parent 86a2aac183
commit 81205a1c81
2 changed files with 139 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
MockBuildUtils.pm
+112 -1
View File
@@ -8,6 +8,7 @@ use warnings;
use Exporter 'import';
use File::Basename qw(basename);
use File::Copy qw(copy);
use File::Glob qw(bsd_glob);
use File::Find;
use Sys::Hostname;
use Digest::MD5 qw(md5_hex);
@@ -15,7 +16,9 @@ use Digest::MD5 qw(md5_hex);
our @EXPORT_OK = qw(
install_deps_packages install_deps_command missing_perl_modules
sh_quote print_step
version_matches required_pkgs have_rpm read_manifest
version_matches required_pkgs skipped_builder carry_over_rpms rpm_name rpm_arch rpm_source_rpm
source_package rpm_digests_ok
have_rpm read_manifest
verify_repo_packages verify_repo_signature verify_rpm_signatures
parse_evr evr_cmp evr_constraint_ok parse_pin rpmkeys_checksig_problem
rpm_version rpm_release rpm_sigmd5 rpm_is_signed restamp_release_line
@@ -106,6 +109,114 @@ sub required_pkgs {
} @$pkgs;
}
# rpm_name: %{name} from the header of one rpm file, undef when rpm cannot read it.
sub rpm_name {
my ($rpm) = @_;
my $name = `rpm -qp --qf '%{name}' ${\ sh_quote($rpm)} 2>/dev/null`;
return (defined $name && $name ne '') ? $name : undef;
}
# rpm_arch: the %{arch} header of one rpm file, undef when rpm cannot read it.
sub rpm_arch {
my ($rpm) = @_;
my $arch = `rpm -qp --qf '%{arch}' ${\ sh_quote($rpm)} 2>/dev/null`;
return (defined $arch && $arch ne '') ? $arch : undef;
}
# rpm_digests_ok: whether the header and payload digests of one rpm file verify, signatures aside.
sub rpm_digests_ok {
my ($rpm) = @_;
my $out = `rpmkeys --checksig --nosignature -v ${\ sh_quote($rpm)} 2>&1`;
return ($? == 0 && $out !~ /NOT OK/i) ? 1 : 0;
}
# rpm_source_rpm: the %{sourcerpm} header of one rpm file, undef when rpm cannot read it. Every
# subpackage of one build shares it.
sub rpm_source_rpm {
my ($rpm) = @_;
my $srpm = `rpm -qp --qf '%{sourcerpm}' ${\ sh_quote($rpm)} 2>/dev/null`;
return (defined $srpm && $srpm =~ /\.src\.rpm$/) ? $srpm : undef;
}
# source_package: the source package name of a source rpm file name.
sub source_package {
my ($srpm) = @_;
return (defined $srpm && $srpm =~ /^(.+)-[^-]+-[^-]+\.src\.rpm$/) ? $1 : undef;
}
# skipped_builder: whether the builder that produces a source package was skipped by the flags.
# The classes are those of required_pkgs: xCAT-genesis-base, perl-*, and everything else the
# xcat-dep builders make. The OpenEmbedded Genesis is published from a release, not built here.
sub skipped_builder {
my ($source, $skipped) = @_;
return 0 if $source =~ /^xCAT-genesis-openembedded(?:-|$)/;
return $skipped->{genesis} ? 1 : 0 if $source =~ /^xCAT-genesis-base(?:-|$)/;
return $skipped->{perl} ? 1 : 0 if $source =~ /^perl-/;
return $skipped->{dep} ? 1 : 0;
}
# carry_over_rpms: copy the binary rpms a skipped builder published in $from into the run repository
# $to, for the builds whose source package the target manifest still names and of which the run
# carries no member yet. %$skipped holds the
# genesis, perl and dep flags; @$manifest_names is the target's manifest section; $name_of and
# $source_of map an rpm path to its package name and its source rpm (rpm_name and rpm_source_rpm
# in production, injectable for tests), and $trusted says whether an rpm may be re-signed at all
# (signed by the configured key in production), $arch is the cell's architecture and $arch_of
# reads an rpm's. A build is selected when one of its rpms is a manifest package, and is then
# carried whole, subpackages included, so a package the manifest dropped is not republished. The
# carry-over dies instead of publishing a partial or doubtful set: an unreadable rpm in the cell
# outside the OpenEmbedded Genesis family, a
# member of a selected build the key did not sign or of another architecture than the cell's or
# noarch, or a package name published more than once. Returns the copied basenames.
sub carry_over_rpms {
my ($from, $to, $skipped, $manifest_names, $name_of, $source_of, $trusted, $arch, $arch_of) = @_;
my %named = map { $_ => 1 } @$manifest_names;
my %staged = map { my $n = $name_of->($_); defined $n ? ($n => 1) : () }
grep { !/\.src\.rpm$/ } bsd_glob("$to/*.rpm");
my (%members, %selected, @problems);
for my $rpm (sort(bsd_glob("$from/*.rpm"))) {
next if $rpm =~ /\.src\.rpm$/;
# The OpenEmbedded Genesis is published from a release and pruned by name, so a stale file
# of that family is not read at all.
next if basename($rpm) =~ /^xCAT-genesis-openembedded-/;
my $srpm = $source_of->($rpm);
my $name = $name_of->($rpm);
my $source = source_package($srpm);
if (!defined $source || !defined $name) {
push @problems, basename($rpm) . ": header unreadable";
next;
}
next unless skipped_builder($source, $skipped);
push @{ $members{$srpm} }, { rpm => $rpm, name => $name };
$selected{$srpm} = 1
if $named{$name} || ($named{'xCAT-genesis-base'} && $name =~ /^xCAT-genesis-base-/);
}
my %seen;
for my $srpm (sort keys %selected) {
for my $m (@{ $members{$srpm} }) {
push @problems, basename($m->{rpm}) . ": not signed by the configured key" unless $trusted->($m->{rpm});
my $rpm_arch = $arch_of->($m->{rpm});
push @problems, basename($m->{rpm}) . ": architecture " . ($rpm_arch // 'unreadable') . " is not $arch or noarch"
unless defined $rpm_arch && ($rpm_arch eq $arch || $rpm_arch eq 'noarch');
push @problems, "$m->{name}: published more than once" if $seen{ $m->{name} }++ == 1;
}
}
die "FATAL: the published cell $from cannot be carried over:\n " . join("\n ", @problems) . "\n"
if @problems;
my @copied;
for my $srpm (sort keys %selected) {
# A build the run already carries in part is the run's, whole; mixing generations of one
# source package is never right.
next if grep { $staged{ $_->{name} } } @{ $members{$srpm} };
for my $m (@{ $members{$srpm} }) {
my $base = basename($m->{rpm});
copy($m->{rpm}, "$to/$base") or die "Failed to carry $m->{rpm} -> $to: $!\n";
push @copied, $base;
}
}
return sort @copied;
}
# verify_repo_packages: the PURE completeness-decision layer of the repo gate. Given the manifest's
# %expected { pkg => version-pin } and the %present { pkg => version-found-or-undef } actually in a
# built repo, return a list of human-readable problem strings (empty list = every package present at
mockbuild-all.pl
+27 -2
View File
@@ -15,7 +15,7 @@ use Parallel::ForkManager;
use POSIX qw(strftime);
use FindBin qw($RealBin);
use lib $RealBin, "$RealBin/lib";
use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs
use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs carry_over_rpms rpm_name rpm_arch rpm_source_rpm rpm_digests_ok
install_deps_packages install_deps_command missing_perl_modules
read_manifest verify_repo_packages verify_repo_signature verify_rpm_signatures
rpm_version rpm_release rpm_sigmd5 restamp_release_line
@@ -856,7 +856,7 @@ my ($copied, $skipped_src, $missing_roots) = collect_rpms(
# let a run whose builders all failed reach createrepo and the deployable tree, and fail
# much later in the repo gate (verify_target_repo), naming missing packages instead of the
# failed builds.
if (!$dry_run && $copied == 0) {
if (!$dry_run && $copied == 0 && @collect_roots) {
die "No binary RPMs were collected. Check build logs and collection roots.\n";
}
@@ -872,6 +872,31 @@ if (!$skip_genesis && !$dry_run) {
}
}
# A skipped builder built nothing this run, so everything it published in the cell joins the run
# repository here, ahead of the bump check, createrepo, the tarballs and the deploy gate.
if (!$dry_run && ($skip_genesis || $skip_perl || $skip_xcat_dep)) {
my $published = "$repo_dep/rh$rel/$arch";
if (-d $published) {
my %skipped = (genesis => $skip_genesis, perl => $skip_perl, dep => $skip_xcat_dep);
# Only an rpm the configured key signed, by signer id and by rpmkeys --checksig, may be
# re-signed and republished; an unsigned run still requires the digests to verify.
my $trusted = \&rpm_digests_ok;
if ($gpg_sign || $gpg_home ne '') {
my ($dbopt, $problem) = rpmkeys_keyring($gpg_key_name, $gpg_home);
die "FATAL: $problem\n" if $problem;
my $accept = gpg_key_ids($gpg_key_name, $gpg_home);
$trusted = sub {
my $id = rpm_signer_keyid($_[0]);
return (defined $id && $accept->{$id} && !rpm_checksig_problem($_[0], $dbopt)) ? 1 : 0;
};
}
for my $base (carry_over_rpms($published, $repo_dir, \%skipped, [sort keys %req],
\&rpm_name, \&rpm_source_rpm, $trusted, $arch, \&rpm_arch)) {
print "[collect] $base kept from the published cell $published\n";
}
}
}
# Repo completeness -- every required package present at its pinned version (a '*' pin accepts any),
# missing packages included -- is now gated ONCE, centrally, in deploy_target via verify_target_repo
# (the single consolidated gate; it also runs under --skip-build and validates the deployed repo).