xCAT/xcat-dep
2
0
Fork 0
mirror of https://github.com/xcat2/xcat-dep.git synced 2026-09-12 04:26:25 +00:00
Code Issues Projects Releases Wiki Activity

fix(xcat-dep): scrub mock chroots after each build to stop /var/lib/mock leak

Browse Source 
The EL build orchestrator never reclaimed the per-step mock buildroots it
created. Each build step makes a build chroot AND a per-uniqueext bootstrap
chroot under /var/lib/mock (dep packages, per-package perl chroots, and
xCAT-genesis-base); the child builders copy their RPMs/logs to their
--result-dir and exit without scrubbing, and mock's own cleanup leaves them
behind (and keeps them entirely on failure). So /var/lib/mock grew ~15-17G per
run, unbounded, until the build host filled to 99% and every mock dnf
transaction failed for lack of space ("Error: needs N MB more space on the /
filesystem", rc=30), cascading to rc=2/rc=255 across packages and killing
otherwise-healthy builds.

After the parallel build phase, scrub each step's buildroot with
"mock -r <chroot> --uniqueext <ext> --scrub=chroot --scrub=bootstrap" -- mock's
own lock-safe scrub: a chroot still held by a concurrent build is refused and
skipped (never rm, which would race a live build). Both the build chroot and its
per-uniqueext bootstrap are removed (each is per-uniqueext, so both leak); the
shared root cache under /var/cache/mock is kept so rebuilds stay fast. The
orchestrator scrubs the dep-package and genesis chroots (whose uniqueext it
assigns); mockbuild-perl-packages.pl scrubs each of its per-package chroots (it
derives its own per-package uniqueext). Scrubs are non-fatal -- a cleanup hiccup
never fails the build. --keep-buildroots preserves the buildroots for debugging.

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
This commit is contained in:
Daniel Hilst
2026-07-28 07:22:59 -03:00
parent c70e09ceeb
commit 81c5a27eb8
3 changed files with 77 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
BUILD.md
+17 -6
View File
@@ -59,12 +59,13 @@ Each build path uses `mock` for chroot isolation. Top-level steps are paralleliz
1. Optional chroot cleanup (`--scrub-all-chroots`)
2. Parallel build execution
3. Optional install/smoke checks inside child builders (disabled with `--skip-install`)
4. Binary RPM collection into `repo/<ARCH>/`
5. Source RPM collection into `repo-src/`
6. `createrepo --update` on both repo trees
7. Tarball creation for both repo trees
8. Summary generation (`summary.txt`)
3. Post-build chroot scrub — reclaims each build step's mock chroot (unless `--keep-buildroots`)
4. Optional install/smoke checks inside child builders (disabled with `--skip-install`)
5. Binary RPM collection into `repo/<ARCH>/`
6. Source RPM collection into `repo-src/`
7. `createrepo --update` on both repo trees
8. Tarball creation for both repo trees
9. Summary generation (`summary.txt`)
# Skip and Control Flags
@@ -86,6 +87,16 @@ Use these flags to skip specific operations:
- Skips tarball creation for both binary and SRPM repos.
- `--scrub-all-chroots`
- Runs `mock -r <TARGET> --scrub=all` before build and collection.
- `--keep-buildroots`
- Keeps each build step's mock chroot after the build instead of scrubbing it. By default,
after the parallel build phase every step's buildroot (dep packages, the per-package perl
chroots, and `xCAT-genesis-base`) is reclaimed with
`mock -r <CHROOT> --uniqueext <EXT> --scrub=chroot --scrub=bootstrap` — a lock-safe scrub (a
chroot still held by a concurrent build is refused and skipped). Both the build chroot and its
per-uniqueext bootstrap chroot are removed (each build step gets its own bootstrap, so both
must go); the shared root cache under `/var/cache/mock` is kept so rebuilds stay fast. This
stops `/var/lib/mock` from growing unbounded across runs. Pass `--keep-buildroots` to preserve
a buildroot for debugging a failed build.
- `--collect-dir <PATH>`
- Adds extra artifact roots to the collection phase (repeatable).
- `--dry-run`
mockbuild-all.pl
+47
View File
@@ -41,6 +41,7 @@ my $skip_genesis = 0;
my $skip_createrepo = 0;
my $skip_tarball = 0;
my $scrub_all_chroots = 0;
my $keep_buildroots = 0; # keep per-step mock chroots after build (default: --scrub=chroot each)
my $dry_run = 0;
my @extra_collect_dirs;
my $repo_dep = '';
@@ -85,6 +86,7 @@ GetOptions(
'skip-createrepo!' => \$skip_createrepo,
'skip-tarball!' => \$skip_tarball,
'scrub-all-chroots!' => \$scrub_all_chroots,
'keep-buildroots!' => \$keep_buildroots,
'collect-dir=s@' => \@extra_collect_dirs,
'dry-run!' => \$dry_run,
) or die usage();
@@ -356,6 +358,7 @@ print "skip_install: $skip_install\n";
print "skip_createrepo: $skip_createrepo\n";
print "skip_tarball: $skip_tarball\n";
print "scrub_all_chroots:$scrub_all_chroots\n";
print "keep_buildroots: $keep_buildroots\n";
print "dry_run: $dry_run\n";
print "perl_builder: $perl_builder\n";
print "tarball: $tarball\n";
@@ -399,6 +402,8 @@ if (!$skip_build) {
step => "Build xcat-dep: $name",
cmd => $cmd,
log => "$log_root/$name/run.log",
scrub_cfg => $target,
scrub_uniqueext => $step_uniqueext,
};
push @collect_roots, $step_result;
}
@@ -425,6 +430,7 @@ if (!$skip_build) {
# committed .src.rpm, so hand the suffix down for the builder to re-stamp them.
($RELEASE_BUMP ne '' ? ('--release-suffix', sh_quote($RELEASE_BUMP)) : ()),
($skip_install ? '--skip-install' : ()),
($keep_buildroots ? '--keep-buildroots' : ()),
);
push @build_steps, {
id => 'perl',
@@ -468,6 +474,7 @@ if (!$skip_build) {
cmd => $cmd,
cwd => $xcat_src,
log => "$log_root/genesis-build.log",
scrub_cfg => "xCAT-genesis-base-$target",
};
}
@@ -482,6 +489,23 @@ if (!$skip_build) {
steps => \@build_steps,
max_processes => $effective_parallel_builds,
);
# Reclaim each build step's mock chroot now that the step copied its RPMs/logs out to
# its --result-dir (collect_rpms reads those, never /var/lib/mock). mock's own cleanup
# leaves these chroots behind -- and keeps them entirely on failure -- so /var/lib/mock
# grows ~15-17G per run until the host fills and every dnf transaction fails for lack of
# space. Scrub each via `mock --scrub=chroot --scrub=bootstrap` (never rm): it takes the
# chroot lock, so a chroot still used by a concurrent build is refused and safely skipped.
# Both the build chroot and its per-uniqueext bootstrap are removed; the root cache stays
# for fast rebuilds. Perl packages are scrubbed inside mockbuild-perl-packages.pl (it
# derives its own per-package uniqueexts).
unless ($keep_buildroots) {
for my $s (@build_steps) {
next unless defined $s->{scrub_cfg};
(my $slug = $s->{id}) =~ s/[^\w.-]+/-/g;
scrub_buildroot($s->{scrub_cfg}, $s->{scrub_uniqueext}, "$log_root/scrub-$slug.log");
}
}
}
}
@@ -947,6 +971,29 @@ sub run_step {
}
}
# Scrub a single mock buildroot via mock's own lock-safe --scrub. Never rm: if a concurrent build
# still holds the chroot lock, mock refuses and we skip it. Failures (already scrubbed, locked, or
# config missing) are tolerated -- a cleanup hiccup must never fail the build. Scrubs both the
# build chroot and its per-uniqueext bootstrap chroot (each build step gets its own bootstrap, so
# both must go or /var/lib/mock still leaks). The shared root cache under /var/cache/mock is kept,
# so rebuilds stay fast. $uniqueext is optional (genesis has none).
sub scrub_buildroot {
my ($cfg, $uniqueext, $log) = @_;
return if !defined $cfg || $cfg eq '';
my $ext = (defined $uniqueext && $uniqueext ne '')
? ' --uniqueext ' . sh_quote($uniqueext) : '';
eval {
run_step(
step => "Scrub chroot $cfg$ext",
cmd => "mock -r " . sh_quote($cfg) . $ext . " --scrub=chroot --scrub=bootstrap",
log => $log,
);
1;
} or do {
warn "WARN: chroot scrub failed (tolerated) for $cfg$ext: $@";
};
}
sub run_build_steps_parallel {
my (%args) = @_;
my $steps = $args{steps} // [];
mockbuild-perl-packages.pl
+13
View File
@@ -13,6 +13,7 @@ my $repo_root = abs_path(dirname(__FILE__));
my $work_dir = '/tmp/perl-list6-mockbuild';
my $mock_cfg = '';
my $mock_uniqueext = '';
my $keep_buildroots = 0;
my $result_dir = '';
my $log_dir = '';
my $packages_csv = '';
@@ -29,6 +30,7 @@ GetOptions(
'work-dir=s' => \$work_dir,
'mock-cfg=s' => \$mock_cfg,
'mock-uniqueext=s' => \$mock_uniqueext,
'keep-buildroots!' => \$keep_buildroots,
'result-dir=s' => \$result_dir,
'log-dir=s' => \$log_dir,
'packages=s' => \$packages_csv,
@@ -230,6 +232,17 @@ for my $idx (0 .. $#packages) {
allow_erasing => $allow_erasing,
release_suffix => $release_suffix,
);
unless ($keep_buildroots) {
# Reclaim this perl package's chroot AND its per-uniqueext bootstrap via mock's lock-safe
# --scrub (never rm). Each package has its own uniqueext (hence its own chroot+bootstrap),
# so scrubbing one never touches a sibling's concurrent build. Runs regardless of build
# result so failed chroots are reclaimed too; the root cache is kept for fast rebuilds.
# The orchestrator can't name these chroots (package_uniqueext derives them), so we scrub
# here.
(my $ps = $pkg) =~ s/[^\w.-]+/-/g;
system("mock -r " . sh_quote($mock_cfg) . " --uniqueext " . sh_quote($pkg_uniqueext)
. " --scrub=chroot --scrub=bootstrap > " . sh_quote("$log_dir/scrub-$ps.log") . " 2>&1");
}
$pm->finish($ok ? 0 : 1);
}
$pm->wait_all_children;