diff --git a/BUILD.md b/BUILD.md index 2c21ad4..1598239 100644 --- a/BUILD.md +++ b/BUILD.md @@ -142,7 +142,17 @@ Use these flags to skip specific operations: - `mockbuild-all.pl` and package sources present under ``. - xCAT sources present under ``. -Install baseline tooling: +Install baseline tooling — let the script do it, so the list cannot drift from what it loads: + +```bash +./mockbuild-all.pl --install-deps # as root, once per build host +``` + +It installs the toolchain and the Perl modules for this host's package manager (dnf on EL, zypper +on SUSE), then **loads** each module and fails if one is still missing. That last step is the point: +a missing module surfaces otherwise as a compile-time abort inside `XCAT::BuildUtils`, in the middle +of a CD run, which is how `perl-File-Slurper` and `perl-IPC-Cmd` each took a pipeline down. The +equivalent by hand: ```bash dnf -y install perl perl-File-Slurper perl-IPC-Cmd \ diff --git a/MockBuildUtils.pm b/MockBuildUtils.pm index 07d0876..aa9065b 100644 --- a/MockBuildUtils.pm +++ b/MockBuildUtils.pm @@ -13,6 +13,7 @@ use Sys::Hostname; use Digest::MD5 qw(md5_hex); our @EXPORT_OK = qw( + install_deps_packages install_deps_command missing_perl_modules sh_quote print_step version_matches required_pkgs have_rpm read_manifest verify_repo_packages verify_repo_signature verify_rpm_signatures @@ -22,6 +23,47 @@ our @EXPORT_OK = qw( build_mock_uniqueext ); +# install_deps_packages($os_id): the host packages mockbuild-all.pl needs to run at all, for the +# given /etc/os-release ID. Kept as data, beside the code that needs them, because the failure mode +# is a build host provisioned by hand: xcat-master-ub had no perl-File-Slurper and xcat-master-ppc no +# perl-IPC-Cmd, and each surfaced as a compile-time abort in the middle of a CD run. +sub install_deps_packages { + my ($os_id) = @_; + $os_id = '' unless defined $os_id; + # The perl modules are what actually break a run; the rest is the toolchain the script drives. + return qw(perl perl-File-Slurper perl-IPC-Cmd perl-Parallel-ForkManager perl-Digest-SHA + mock createrepo_c tar findutils rpm rpm-build rpm-sign rpmdevtools gnupg2 wget git) + if $os_id =~ /^(?:opensuse|sles|sled)/; + return qw(perl perl-File-Slurper perl-IPC-Cmd perl-Parallel-ForkManager perl-Digest-SHA + mock createrepo_c tar findutils rpm rpm-build rpm-sign rpmdevtools + dnf-plugins-core gnupg2 wget git); +} + +# install_deps_command($os_id): the argv that installs them, non-interactively. +sub install_deps_command { + my ($os_id) = @_; + $os_id = '' unless defined $os_id; + my @pkgs = install_deps_packages($os_id); + return ('zypper', '--non-interactive', 'install', '--no-recommends', @pkgs) + if $os_id =~ /^(?:opensuse|sles|sled)/; + return ('dnf', '-y', 'install', @pkgs); +} + +# missing_perl_modules(@modules): those that cannot be loaded, in order. The point of --install-deps +# is that the run AFTER it cannot die on a missing module, so the modules are proven by loading +# them, not by trusting the package manager's exit code. +sub missing_perl_modules { + my (@modules) = @_; + my @missing; + for my $m (@modules) { + my $file = $m; + $file =~ s{::}{/}g; + $file .= '.pm'; + eval { require $file; 1 } or push @missing, $m; + } + return @missing; +} + # sh_quote: single-quote a string for safe use in a shell command. sub sh_quote { my ($s) = @_; diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 85a481c..306175d 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -16,6 +16,7 @@ use POSIX qw(strftime); use FindBin qw($RealBin); use lib $RealBin, "$RealBin/lib"; use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs + install_deps_packages install_deps_command missing_perl_modules read_manifest verify_repo_packages verify_repo_signature verify_rpm_signatures rpm_version rpm_release rpm_sigmd5 restamp_release_line cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix @@ -51,7 +52,7 @@ use XCAT::GenesisRelease qw( # otherwise warn loudly and continue unisolated. MOCKBUILD_ALL_MOUNTNS guards against a re-exec loop. # Build-free modes (--verify-repo, --finalize-xcat-dep) run no mock and are documented no-root, so they # skip the re-exec entirely -- no cgroup exposure, and no spurious non-root warning. -my $mountns_build_free = grep { /^--(?:verify-repo|finalize-xcat-dep)(?:=|$)/ } @ARGV; +my $mountns_build_free = grep { /^--(?:verify-repo|finalize-xcat-dep|install-deps)(?:=|$)/ } @ARGV; unless ($ENV{MOCKBUILD_ALL_MOUNTNS} || $mountns_build_free) { if ($> != 0) { warn "WARN: not root -- skipping mount-namespace isolation (host-cgroup propagation guard); " @@ -101,6 +102,7 @@ my $GOCONSERVER_REF = '6166fe5ec1c5b3c20475e322a9f0e8e93c87e45f'; my $skip_build = 0; my $skip_xcat_dep = 0; my $skip_perl = 0; +my $install_deps = 0; my $skip_genesis = 0; my $skip_createrepo = 0; my $skip_tarball = 0; @@ -161,6 +163,7 @@ GetOptions( 'skip-build!' => \$skip_build, 'skip-xcat-dep!' => \$skip_xcat_dep, 'skip-perl!' => \$skip_perl, + 'install-deps!' => \$install_deps, 'skip-genesis!' => \$skip_genesis, 'skip-createrepo!' => \$skip_createrepo, 'skip-tarball!' => \$skip_tarball, @@ -321,6 +324,28 @@ die "Could not resolve ID from /etc/os-release\n" if $os_id eq ''; die "Could not resolve major release from VERSION_ID='$version_id' in /etc/os-release\n" if !defined($rel) || $rel eq ''; +# --install-deps: make THIS host able to run the script, then exit. It has to come before the +# require_command checks below -- those are the very things it installs, and a host that lacks them +# would die here with no way to fix itself. Run once per build host, as root. +# +# The perl modules are re-checked by LOADING them afterwards rather than trusting the package +# manager: a module that is still missing is exactly the failure this mode exists to prevent, and it +# aborted CD runs mid-build twice (perl-File-Slurper on xcat-master-ub, perl-IPC-Cmd on +# xcat-master-ppc), each time as a compile-time error inside XCAT::BuildUtils. +if ($install_deps) { + die "--install-deps must run as root (uid=$>)\n" if $> != 0; + my @cmd = install_deps_command($os_id); + print_step("Install build prerequisites ($os_id)"); + print " " . join(' ', @cmd) . "\n"; + run_command(@cmd); + my @modules = qw(File::Slurper IPC::Cmd Parallel::ForkManager Digest::SHA); + my @missing = missing_perl_modules(@modules); + die "FATAL: still missing after install: " . join(', ', @missing) . "\n" if @missing; + print " perl modules present: " . join(', ', @modules) . "\n"; + print " host is ready\n"; + exit 0; +} + for my $bin (qw(perl uname createrepo_c tar find rpm)) { require_command($bin); } @@ -1244,6 +1269,9 @@ Options: --skip-build Skip all build steps and only collect/create repo/tarballs --skip-xcat-dep Skip xcat-dep mockbuild.pl package steps --skip-perl Skip perl package build step + --install-deps Install this host's build prerequisites (package manager + the perl + modules the script loads), verify each module now loads, then exit. + Run once per build host, as root. Use alone. --skip-genesis Skip the existing per-EL Genesis image build --skip-createrepo Skip createrepo --skip-tarball Skip binary/SRPM tarball creation diff --git a/t/mockbuild-all.t b/t/mockbuild-all.t index e7579aa..7511ca9 100644 --- a/t/mockbuild-all.t +++ b/t/mockbuild-all.t @@ -10,7 +10,8 @@ use lib "$RealBin/.."; use File::Temp qw(tempdir); use File::Path qw(make_path); use File::Basename qw(basename); -use MockBuildUtils qw(required_pkgs version_matches rpm_sigmd5 rpm_version rpm_release rpm_is_signed +use MockBuildUtils qw(install_deps_packages install_deps_command missing_perl_modules + required_pkgs version_matches rpm_sigmd5 rpm_version rpm_release rpm_is_signed restamp_release_line cross_copy_genesis finalize_xcat_dep read_manifest verify_repo_packages verify_repo_signature verify_rpm_signatures parse_evr evr_constraint_ok parse_pin rpmkeys_checksig_problem @@ -481,4 +482,33 @@ my $vercmp = sub { } } +# ---- --install-deps: the host prerequisites (the modules are what actually break a run) ---------- +# Two CD runs died at compile time inside XCAT::BuildUtils because a builder lacked a module +# (perl-File-Slurper on one host, perl-IPC-Cmd on another), so the list must carry every module the +# script loads, and the mode must PROVE them by loading rather than trusting the package manager. +{ + my @el = install_deps_packages('almalinux'); + for my $need (qw(perl-File-Slurper perl-IPC-Cmd perl-Parallel-ForkManager mock createrepo_c)) { + ok(scalar(grep { $_ eq $need } @el), "EL prerequisites include $need"); + } + my @cmd = install_deps_command('almalinux'); + is($cmd[0], 'dnf', 'EL installs with dnf'); + ok(scalar(grep { $_ eq '-y' } @cmd), '... non-interactively'); + + my @suse = install_deps_command('opensuse-leap'); + is($suse[0], 'zypper', 'SUSE installs with zypper'); + ok(scalar(grep { $_ eq '--non-interactive' } @suse), '... non-interactively'); + is_deeply([ grep { /^perl-/ } install_deps_packages('opensuse-leap') ], + [ grep { /^perl-/ } @el ], + 'both families install the same perl modules'); + + # the probe reports what cannot be loaded, and nothing else + is_deeply([ missing_perl_modules('Digest::SHA') ], [], + 'missing_perl_modules: a loadable module is not reported'); + is_deeply([ missing_perl_modules('No::Such::Module::Here') ], ['No::Such::Module::Here'], + 'missing_perl_modules: an absent module is reported'); + is_deeply([ missing_perl_modules('Digest::SHA', 'No::Such::Module::Here') ], + ['No::Such::Module::Here'], '... and only the absent one, from a mixed list'); +} + done_testing;