From ca1f2f3a366af9cbe443b7e2bb20aaa0d5cfb5e4 Mon Sep 17 00:00:00 2001
From: Daniel Hilst <392820+dhilst@users.noreply.github.com>
Date: Thu, 30 Jul 2026 14:48:50 -0300
Subject: [PATCH] fix(xcat-dep): stop mid-batch bootstrap scrub racing
 concurrent perl builds

The parallel perl-package builder scrubbed each package's chroot AND bootstrap
(`mock --uniqueext <pkg> --scrub=chroot --scrub=bootstrap`) as soon as that
package finished, mid-batch. The comment assumed the --uniqueext made this
sibling-safe, but mock's bootstrap scrub ignores --uniqueext and removes the
CONFIG-LEVEL shared bootstrap cache (/var/cache/mock/<cfg>-bootstrap/, keyed by
config name, not uniqueext). Under concurrency a faster sibling's post-build
scrub deleted that shared cache while a slower sibling was still setting up its
buildsrpm chroot and about to bind-mount it, so the bind failed with mount
rc=32 and the whole target failed a build that was otherwise fine.

Observed: on alma+epel-10-ppc64le, perl-Sys-Virt (the slowest, a libvirt C
binding) died at --buildsrpm binding
/var/cache/mock/alma+epel-10-ppc64le-bootstrap/yum_cache ~1s after two faster
siblings had just scrubbed that shared bootstrap; the other five perl packages
passed. Non-deterministic and load-triggered, so it surfaced under the 3-way
concurrent CD load.

Reclaim only the (uniqueext-local) build chroot per package during the batch,
and defer the shared bootstrap reclamation to a single serialized pass after
all workers join, when nothing can be binding it. Disk reclamation is preserved
(the ~GB build chroots are still freed immediately; the bootstrap roots + shared
cache are freed at batch end).

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
---
 mockbuild-perl-packages.pl | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/mockbuild-perl-packages.pl b/mockbuild-perl-packages.pl
index c31aac9..ecd0260 100755
--- a/mockbuild-perl-packages.pl
+++ b/mockbuild-perl-packages.pl
@@ -233,20 +233,35 @@ for my $idx (0 .. $#packages) {
         release_suffix => $release_suffix,
     );
     unless ($keep_buildroots) {
-        # Reclaim this perl package's chroot AND its per-uniqueext bootstrap via mock's lock-safe
-        # --scrub (never rm). Each package has its own uniqueext (hence its own chroot+bootstrap),
-        # so scrubbing one never touches a sibling's concurrent build. Runs regardless of build
-        # result so failed chroots are reclaimed too; the root cache is kept for fast rebuilds.
-        # The orchestrator can't name these chroots (package_uniqueext derives them), so we scrub
-        # here.
+        # Reclaim ONLY this package's build chroot here (it's the ~GB disk hog). --scrub=chroot is
+        # uniqueext-local, so it never touches a concurrent sibling. Do NOT --scrub=bootstrap here:
+        # despite the --uniqueext, mock's bootstrap scrub removes the CONFIG-LEVEL shared bootstrap
+        # cache (/var/cache/mock/<cfg>-bootstrap/, keyed by config name, NOT uniqueext). Doing that
+        # mid-batch deletes the cache a still-starting sibling is about to bind-mount into its own
+        # bootstrap root -> `mount rc=32` and a spurious build failure (observed: perl-Sys-Virt's
+        # buildsrpm raced a faster sibling's post-build bootstrap scrub). The shared bootstrap is
+        # reclaimed once below, after ALL workers finish, when nothing can be binding it.
         (my $ps = $pkg) =~ s/[^\w.-]+/-/g;
         system("mock -r " . sh_quote($mock_cfg) . " --uniqueext " . sh_quote($pkg_uniqueext)
-             . " --scrub=chroot --scrub=bootstrap > " . sh_quote("$log_dir/scrub-$ps.log") . " 2>&1");
+             . " --scrub=chroot > " . sh_quote("$log_dir/scrub-$ps.log") . " 2>&1");
     }
     $pm->finish($ok ? 0 : 1);
 }
 $pm->wait_all_children;
 
+# Now that every worker has exited, reclaim the per-uniqueext bootstrap roots + the shared
+# config-level bootstrap cache. Serialized and post-join, so no scrub can race a concurrent
+# bind (that race is exactly what the per-package note above avoids). Best-effort: the first
+# scrub drops /var/cache/mock/<cfg>-bootstrap; each also removes its uniqueext bootstrap root.
+unless ($keep_buildroots) {
+    for my $idx (0 .. $#packages) {
+        my $pkg_uniqueext = package_uniqueext($mock_uniqueext, $idx + 1, $packages[$idx]);
+        (my $ps = $packages[$idx]) =~ s/[^\w.-]+/-/g;
+        system("mock -r " . sh_quote($mock_cfg) . " --uniqueext " . sh_quote($pkg_uniqueext)
+             . " --scrub=bootstrap >> " . sh_quote("$log_dir/scrub-$ps.log") . " 2>&1");
+    }
+}
+
 for my $pkg (@packages) {
     my $status_file = "$log_dir/$pkg/status.txt";
     if (!-f $status_file) {