diff --git a/BUILD.md b/BUILD.md index d386be7..062c308 100644 --- a/BUILD.md +++ b/BUILD.md @@ -59,12 +59,13 @@ Each build path uses `mock` for chroot isolation. Top-level steps are paralleliz 1. Optional chroot cleanup (`--scrub-all-chroots`) 2. Parallel build execution -3. Optional install/smoke checks inside child builders (disabled with `--skip-install`) -4. Binary RPM collection into `repo//` -5. Source RPM collection into `repo-src/` -6. `createrepo --update` on both repo trees -7. Tarball creation for both repo trees -8. Summary generation (`summary.txt`) +3. Post-build chroot scrub — reclaims each build step's mock chroot (unless `--keep-buildroots`) +4. Optional install/smoke checks inside child builders (disabled with `--skip-install`) +5. Binary RPM collection into `repo//` +6. Source RPM collection into `repo-src/` +7. `createrepo --update` on both repo trees +8. Tarball creation for both repo trees +9. Summary generation (`summary.txt`) # Skip and Control Flags @@ -86,6 +87,16 @@ Use these flags to skip specific operations: - Skips tarball creation for both binary and SRPM repos. - `--scrub-all-chroots` - Runs `mock -r --scrub=all` before build and collection. +- `--keep-buildroots` + - Keeps each build step's mock chroot after the build instead of scrubbing it. By default, + after the parallel build phase every step's buildroot (dep packages, the per-package perl + chroots, and `xCAT-genesis-base`) is reclaimed with + `mock -r --uniqueext --scrub=chroot --scrub=bootstrap` — a lock-safe scrub (a + chroot still held by a concurrent build is refused and skipped). Both the build chroot and its + per-uniqueext bootstrap chroot are removed (each build step gets its own bootstrap, so both + must go); the shared root cache under `/var/cache/mock` is kept so rebuilds stay fast. This + stops `/var/lib/mock` from growing unbounded across runs. Pass `--keep-buildroots` to preserve + a buildroot for debugging a failed build. - `--collect-dir ` - Adds extra artifact roots to the collection phase (repeatable). - `--dry-run` diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 44c817f..4754dbc 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -41,6 +41,7 @@ my $skip_genesis = 0; my $skip_createrepo = 0; my $skip_tarball = 0; my $scrub_all_chroots = 0; +my $keep_buildroots = 0; # keep per-step mock chroots after build (default: --scrub=chroot each) my $dry_run = 0; my @extra_collect_dirs; my $repo_dep = ''; @@ -85,6 +86,7 @@ GetOptions( 'skip-createrepo!' => \$skip_createrepo, 'skip-tarball!' => \$skip_tarball, 'scrub-all-chroots!' => \$scrub_all_chroots, + 'keep-buildroots!' => \$keep_buildroots, 'collect-dir=s@' => \@extra_collect_dirs, 'dry-run!' => \$dry_run, ) or die usage(); @@ -356,6 +358,7 @@ print "skip_install: $skip_install\n"; print "skip_createrepo: $skip_createrepo\n"; print "skip_tarball: $skip_tarball\n"; print "scrub_all_chroots:$scrub_all_chroots\n"; +print "keep_buildroots: $keep_buildroots\n"; print "dry_run: $dry_run\n"; print "perl_builder: $perl_builder\n"; print "tarball: $tarball\n"; @@ -399,6 +402,8 @@ if (!$skip_build) { step => "Build xcat-dep: $name", cmd => $cmd, log => "$log_root/$name/run.log", + scrub_cfg => $target, + scrub_uniqueext => $step_uniqueext, }; push @collect_roots, $step_result; } @@ -425,6 +430,7 @@ if (!$skip_build) { # committed .src.rpm, so hand the suffix down for the builder to re-stamp them. ($RELEASE_BUMP ne '' ? ('--release-suffix', sh_quote($RELEASE_BUMP)) : ()), ($skip_install ? '--skip-install' : ()), + ($keep_buildroots ? '--keep-buildroots' : ()), ); push @build_steps, { id => 'perl', @@ -468,6 +474,7 @@ if (!$skip_build) { cmd => $cmd, cwd => $xcat_src, log => "$log_root/genesis-build.log", + scrub_cfg => "xCAT-genesis-base-$target", }; } @@ -482,6 +489,23 @@ if (!$skip_build) { steps => \@build_steps, max_processes => $effective_parallel_builds, ); + + # Reclaim each build step's mock chroot now that the step copied its RPMs/logs out to + # its --result-dir (collect_rpms reads those, never /var/lib/mock). mock's own cleanup + # leaves these chroots behind -- and keeps them entirely on failure -- so /var/lib/mock + # grows ~15-17G per run until the host fills and every dnf transaction fails for lack of + # space. Scrub each via `mock --scrub=chroot --scrub=bootstrap` (never rm): it takes the + # chroot lock, so a chroot still used by a concurrent build is refused and safely skipped. + # Both the build chroot and its per-uniqueext bootstrap are removed; the root cache stays + # for fast rebuilds. Perl packages are scrubbed inside mockbuild-perl-packages.pl (it + # derives its own per-package uniqueexts). + unless ($keep_buildroots) { + for my $s (@build_steps) { + next unless defined $s->{scrub_cfg}; + (my $slug = $s->{id}) =~ s/[^\w.-]+/-/g; + scrub_buildroot($s->{scrub_cfg}, $s->{scrub_uniqueext}, "$log_root/scrub-$slug.log"); + } + } } } @@ -947,6 +971,29 @@ sub run_step { } } +# Scrub a single mock buildroot via mock's own lock-safe --scrub. Never rm: if a concurrent build +# still holds the chroot lock, mock refuses and we skip it. Failures (already scrubbed, locked, or +# config missing) are tolerated -- a cleanup hiccup must never fail the build. Scrubs both the +# build chroot and its per-uniqueext bootstrap chroot (each build step gets its own bootstrap, so +# both must go or /var/lib/mock still leaks). The shared root cache under /var/cache/mock is kept, +# so rebuilds stay fast. $uniqueext is optional (genesis has none). +sub scrub_buildroot { + my ($cfg, $uniqueext, $log) = @_; + return if !defined $cfg || $cfg eq ''; + my $ext = (defined $uniqueext && $uniqueext ne '') + ? ' --uniqueext ' . sh_quote($uniqueext) : ''; + eval { + run_step( + step => "Scrub chroot $cfg$ext", + cmd => "mock -r " . sh_quote($cfg) . $ext . " --scrub=chroot --scrub=bootstrap", + log => $log, + ); + 1; + } or do { + warn "WARN: chroot scrub failed (tolerated) for $cfg$ext: $@"; + }; +} + sub run_build_steps_parallel { my (%args) = @_; my $steps = $args{steps} // []; diff --git a/mockbuild-perl-packages.pl b/mockbuild-perl-packages.pl index fba8c62..c31aac9 100755 --- a/mockbuild-perl-packages.pl +++ b/mockbuild-perl-packages.pl @@ -13,6 +13,7 @@ my $repo_root = abs_path(dirname(__FILE__)); my $work_dir = '/tmp/perl-list6-mockbuild'; my $mock_cfg = ''; my $mock_uniqueext = ''; +my $keep_buildroots = 0; my $result_dir = ''; my $log_dir = ''; my $packages_csv = ''; @@ -29,6 +30,7 @@ GetOptions( 'work-dir=s' => \$work_dir, 'mock-cfg=s' => \$mock_cfg, 'mock-uniqueext=s' => \$mock_uniqueext, + 'keep-buildroots!' => \$keep_buildroots, 'result-dir=s' => \$result_dir, 'log-dir=s' => \$log_dir, 'packages=s' => \$packages_csv, @@ -230,6 +232,17 @@ for my $idx (0 .. $#packages) { allow_erasing => $allow_erasing, release_suffix => $release_suffix, ); + unless ($keep_buildroots) { + # Reclaim this perl package's chroot AND its per-uniqueext bootstrap via mock's lock-safe + # --scrub (never rm). Each package has its own uniqueext (hence its own chroot+bootstrap), + # so scrubbing one never touches a sibling's concurrent build. Runs regardless of build + # result so failed chroots are reclaimed too; the root cache is kept for fast rebuilds. + # The orchestrator can't name these chroots (package_uniqueext derives them), so we scrub + # here. + (my $ps = $pkg) =~ s/[^\w.-]+/-/g; + system("mock -r " . sh_quote($mock_cfg) . " --uniqueext " . sh_quote($pkg_uniqueext) + . " --scrub=chroot --scrub=bootstrap > " . sh_quote("$log_dir/scrub-$ps.log") . " 2>&1"); + } $pm->finish($ok ? 0 : 1); } $pm->wait_all_children;