diff --git a/BUILD.md b/BUILD.md index 1c87ae4..bb575ed 100644 --- a/BUILD.md +++ b/BUILD.md @@ -402,6 +402,21 @@ Codename ↔ version (the single supported set — `BuildUtils` is the source of - **`t/verify-repo.t`** — end-to-end tests of the repo gate against fixture apt trees (missing secondary arch, arch:all-only index, unsigned repo, missing manifest section). +## Prerequisites (once per build host, as root) + +```bash +./sbuild-all.pl --install-deps +``` + +Installs the sbuild/schroot toolchain and the Perl modules the script loads, then **loads** each one +and fails if any is still missing. That last step is the point: a missing module surfaces otherwise +as a compile-time abort inside `XCAT::BuildUtils`, mid-run — which is how `File::Slurper` being +absent on `xcat-master-ub` took a CD run down. `IPC::Cmd` is deliberately not in the package list: +it is core on Debian/Ubuntu, no such package exists, and naming one fails the whole install; the +probe is what asserts it is usable. + +The per-codename sbuild chroots are separate host state — see `ci/mk-dep-chroots.sh`. + ## Usage (per arch, as root on the matching build host) Run `sbuild-all.pl` on the build host for the arch you are building (amd64 on the x86 Ubuntu host, diff --git a/BuildUtils.pm b/BuildUtils.pm index bc2c8b4..c557b8b 100644 --- a/BuildUtils.pm +++ b/BuildUtils.pm @@ -22,6 +22,7 @@ use Digest::MD5; use MIME::Base64 qw(encode_base64); our @EXPORT_OK = qw( + install_deps_packages install_deps_command missing_perl_modules sh_quote print_step version_matches required_pkgs read_manifest standard_options verify_repo_packages verify_repo_signature verify_repo_arches @@ -54,6 +55,38 @@ sub version_to_codename { my ($v) = @_; return $VERSION_TO_CODENAME{$v // ''}; } # Shared / format-agnostic helpers (identical semantics to MockBuildUtils.pm; see migration note). # --------------------------------------------------------------------------------------------------- +# install_deps_packages(): the host packages sbuild-all.pl needs to run at all. Kept as data beside +# the code that needs them: the failure mode is a build host provisioned by hand, and a missing perl +# module surfaces as a compile-time abort in the middle of a CD run rather than as a clear message +# (xcat-master-ub had no File::Slurper, which is how this was found). +sub install_deps_packages { + # NOTE: no libipc-cmd-perl -- IPC::Cmd is CORE on Debian/Ubuntu (it ships in perl-modules) and + # no such package exists, so naming it fails the whole install. That it is present is asserted + # by the module probe, not by installing a package. + return qw(perl libfile-slurper-perl libparallel-forkmanager-perl + sbuild schroot debootstrap apt-utils dpkg-dev devscripts equivs quilt fakeroot + build-essential reprepro gnupg rsync wget git); +} + +# install_deps_command(): the argv that installs them, non-interactively. +sub install_deps_command { + return ('apt-get', 'install', '-y', '--no-install-recommends', install_deps_packages()); +} + +# missing_perl_modules(@modules): those that cannot be loaded, in order. --install-deps proves the +# modules by LOADING them, rather than trusting the package manager's exit code. +sub missing_perl_modules { + my (@modules) = @_; + my @missing; + for my $m (@modules) { + my $file = $m; + $file =~ s{::}{/}g; + $file .= '.pm'; + eval { require $file; 1 } or push @missing, $m; + } + return @missing; +} + # sh_quote: single-quote a string for safe use in a shell command. sub sh_quote { my ($s) = @_; diff --git a/sbuild-all.pl b/sbuild-all.pl index 0f566c8..c22aef1 100755 --- a/sbuild-all.pl +++ b/sbuild-all.pl @@ -46,6 +46,7 @@ use lib $RealBin, "$RealBin/lib"; # it, including the builds that never pass --genesis-release. Its subs are therefore called # fully-qualified. use BuildUtils qw(sh_quote print_step version_matches required_pkgs read_manifest standard_options + install_deps_packages install_deps_command missing_perl_modules verify_repo_packages verify_repo_signature verify_repo_arches parse_packages_index parse_release_architectures resolve_present_names index_has_native_arch control_binary_arch skip_arch_all_on @@ -73,6 +74,7 @@ my $build_number; # (4 per host). N caps it to N; 1 forces serial. my $parallel_targets = 0; my ($skip_build, $skip_install, $skip_genesis, $skip_xcat_dep) = (0,0,0,0); +my $install_deps = 0; my ($skip_createrepo, $skip_tarball) = (0,0); my $dry_run = 0; # --publish: run the FINALIZATION phase (assemble + sign + gate + tarball). See the "Publishing" @@ -176,6 +178,7 @@ $spec{'genesis-deb=s'} = \@genesis_debs; $spec{'genesis-rpm=s'} = \$genesis_rpm; $spec{'genesis-rpm-ppc=s'} = \$genesis_rpm_ppc; $spec{'require-ppc-genesis!'} = \$require_ppc_genesis; +$spec{'install-deps!'} = \$install_deps; # make this host able to run at all, then exit $spec{'publish!'} = \$publish; # run the finalization (assemble+sign+gate+tarball) $spec{'publish-lock-wait=i'} = \$PUBLISH_LOCK_WAIT; # seconds to queue behind another publisher $spec{'expect-arch=s'} = \@expect_arch; # repeatable; each value may be a space/comma list @@ -188,6 +191,27 @@ $spec{'man'} = sub { pod2usage(-verbose => 2, -exitval => 0); GetOptions(%spec) or pod2usage(-verbose => 1, -exitval => 2); +# --install-deps: make THIS host able to run the script, then exit. It comes first because +# everything below assumes the toolchain is present, and a host that lacks it would fail with a +# compile-time error inside a module rather than a message it can act on -- which is how +# File::Slurper being absent on xcat-master-ub aborted a CD run mid-build. The modules are proven by +# LOADING them, not by trusting apt's exit code. Run once per build host, as root. +if ($install_deps) { + die "FATAL: --install-deps must run as root (uid=$>)\n" if $> != 0; + my @cmd = install_deps_command(); + print_step('Install build prerequisites'); + print " " . join(' ', @cmd) . "\n"; + local $ENV{DEBIAN_FRONTEND} = 'noninteractive'; + system('apt-get', 'update', '-q') == 0 or die "FATAL: apt-get update failed\n"; + system(@cmd) == 0 or die "FATAL: " . join(' ', @cmd) . " failed\n"; + my @modules = qw(File::Slurper IPC::Cmd Parallel::ForkManager Digest::MD5); + my @missing = missing_perl_modules(@modules); + die "FATAL: still missing after install: " . join(', ', @missing) . "\n" if @missing; + print " perl modules present: " . join(', ', @modules) . "\n"; + print " host is ready\n"; + exit 0; +} + # --------------------------------------------------------------------------------------------------- # Configuration # --------------------------------------------------------------------------------------------------- @@ -1485,6 +1509,12 @@ the default gives 8 concurrent build streams for a 4-codename matrix (4 per host Skip the corresponding phase(s). C<--skip-build --skip-genesis> is the finalization run (it publishes by default; see C<--publish>). C<--skip-createrepo> forces "do not publish" and always wins. +=item B<--install-deps> + +Install this host's build prerequisites (the sbuild/schroot toolchain and the Perl modules the +script loads), verify each module now loads, then exit. Run once per build host, as root. Use +alone. + =item B<--publish-lock-wait> C How long to queue behind another publisher before failing (default 1800). Publishing takes one diff --git a/t/sbuild-all.t b/t/sbuild-all.t index 8f2075e..dfd09fa 100644 --- a/t/sbuild-all.t +++ b/t/sbuild-all.t @@ -12,7 +12,8 @@ use lib "$RealBin/.."; use File::Temp qw(tempdir); use File::Path qw(make_path); use File::Basename qw(basename); -use BuildUtils qw(required_pkgs version_matches read_manifest standard_options +use BuildUtils qw(install_deps_packages install_deps_command missing_perl_modules + required_pkgs version_matches read_manifest standard_options verify_repo_packages verify_repo_signature verify_repo_arches parse_packages_index parse_release_architectures resolve_present_names index_has_native_arch control_binary_arch skip_arch_all_on @@ -592,4 +593,29 @@ STUB } } +# ---- --install-deps: the host prerequisites (the modules are what actually break a run) ---------- +# A CD run died at compile time inside XCAT::BuildUtils because this builder lacked File::Slurper, +# so the list must carry every module the script loads, and the mode must PROVE them by loading +# rather than trusting apt's exit code. +{ + my @pkgs = install_deps_packages(); + for my $need (qw(libfile-slurper-perl libparallel-forkmanager-perl sbuild schroot apt-utils dpkg-dev)) { + ok(scalar(grep { $_ eq $need } @pkgs), "prerequisites include $need"); + } + my @cmd = install_deps_command(); + is($cmd[0], 'apt-get', 'installs with apt-get'); + ok(scalar(grep { $_ eq '-y' } @cmd), '... non-interactively'); + ok(scalar(grep { $_ eq '--no-install-recommends' } @cmd), '... without recommends'); + + # IPC::Cmd is CORE on Debian/Ubuntu -- there is no package to name, and naming one fails the + # whole install. The probe is what asserts it is usable. + ok(!scalar(grep { /ipc-cmd/ } @pkgs), 'no libipc-cmd-perl: IPC::Cmd is core on Debian/Ubuntu'); + is_deeply([ missing_perl_modules('IPC::Cmd') ], [], '... and it loads'); + + is_deeply([ missing_perl_modules('Digest::MD5') ], [], + 'missing_perl_modules: a loadable module is not reported'); + is_deeply([ missing_perl_modules('No::Such::Module::Here') ], ['No::Such::Module::Here'], + 'missing_perl_modules: an absent module is reported'); +} + done_testing;