build(mockbuild): retry a mock run once when its package manager fails (rc=30)

A riscv64 run lost perl-Crypt-SSLeay to a transient mirror problem: the
bootstrap chroot's dnf got HTTP 404 for BaseOS primary.xml.gz on every Rocky
mirror (metadata mid-sync) and mock exited 30 (YumError) from --buildsrpm,
which the builder treated as a hard failure. Give the mock-driven builders --
mockbuild-perl-packages.pl, ipmitool, grub2-xcat and conserver -- a small
run_mock wrapper that reruns the same mock command once when it exits 30
(the package-manager failure code; build failures exit 10 and are not
retried), and use it for their --buildsrpm/--rebuild invocations.

conserver/mockbuild.pl

@@ -102,7 +102,7 @@ print "SRPM: $srpm\n";
 print "== mock --rebuild ($mock_cfg) ==\n";
 my $mock_result = "$work_dir/mock-result";
 make_path($mock_result);
-run("mock -r " . sh_quote($mock_cfg) . $uniq
+run_mock("mock -r " . sh_quote($mock_cfg) . $uniq
     . " --rebuild " . sh_quote($srpm)
     . " --resultdir " . sh_quote($mock_result)
     . " > " . sh_quote("$log_dir/mock-rebuild.log") . " 2>&1");
@@ -146,6 +146,18 @@ sub capture {
     chomp $out if defined $out;
     return $out // '';
 }
+# mock exits 30 when its package manager failed (chroot init, build deps), which against public
+# mirrors is most often a transient download error (stale mirror metadata): retry such a run once.
+sub run_mock {
+    my ($cmd) = @_;
+    my $rc = system('bash', '-c', $cmd);
+    if ($rc != -1 && ($rc >> 8) == 30) {
+        print "mock failed with rc=30 (package manager); retrying once\n";
+        $rc = system('bash', '-c', $cmd);
+    }
+    die "FATAL: command failed (rc=" . ($rc >> 8) . "): $cmd\n" if $rc != 0;
+    return 1;
+}
 sub sh_quote { my ($s) = @_; $s =~ s/'/'\\''/g; return "'$s'"; }
 sub usage {
     return "usage: mockbuild.pl --mock-cfg <cfg> [--target-arch ARCH] [--result-dir DIR] [--work-dir DIR]\n"

grub2-xcat/mockbuild.pl

@@ -152,7 +152,7 @@ print "Prep dry run passed. Applied patches: $patch_count\n";
 print_step("Build SRPM with mock");
 my $srpm_out = "$work_dir/srpm";
 make_path($srpm_out);
-run(
+run_mock(
     "mock -r " . sh_quote($det_mock_cfg) . $mock_uniqueext_opt .
     " --buildsrpm --spec " . sh_quote($spec_file) .
     " --sources " . sh_quote($pkg_dir) .
@@ -170,7 +170,7 @@ print "SRPM: $srpm\n";
 print_step("Rebuild RPM with mock");
 my $rpm_out = "$work_dir/rpm";
 make_path($rpm_out);
-run(
+run_mock(
     "mock -r " . sh_quote($det_mock_cfg) . $mock_uniqueext_opt .
     " --rebuild " . sh_quote($srpm) .
     " --resultdir " . sh_quote($rpm_out) .
@@ -384,6 +384,22 @@ sub capture {
     return $out;
 }
 
+# mock exits 30 when its package manager failed (chroot init, build deps), which against public
+# mirrors is most often a transient download error (stale mirror metadata): retry such a run once.
+sub run_mock {
+    my ($cmd) = @_;
+    print "+ $cmd\n";
+    my $rc = system($cmd);
+    if ($rc != -1 && ($rc >> 8) == 30) {
+        print "mock failed with rc=30 (package manager); retrying once\n";
+        $rc = system($cmd);
+    }
+    if ($rc != 0) {
+        my $exit = $rc == -1 ? 255 : ($rc >> 8);
+        die "Command failed (rc=$exit): $cmd\n";
+    }
+}
+
 sub run_capture_rc {
     my ($cmd, $log_file) = @_;
     my $full = "$cmd > " . sh_quote($log_file) . " 2>&1";

ipmitool/mockbuild.pl

@@ -138,7 +138,7 @@ print "Patch application check passed. Applied patches: $patch_count\n";
 print_step("Build SRPM with mock");
 my $srpm_out = "$work_dir/srpm";
 make_path($srpm_out);
-run(
+run_mock(
     "mock -r " . sh_quote($det_mock_cfg) . $mock_uniqueext_opt .
     " --buildsrpm --spec " . sh_quote($spec_file) .
     " --sources " . sh_quote($pkg_dir) .
@@ -156,7 +156,7 @@ print "SRPM: $srpm\n";
 print_step("Rebuild RPM with mock");
 my $rpm_out = "$work_dir/rpm";
 make_path($rpm_out);
-run(
+run_mock(
     "mock -r " . sh_quote($det_mock_cfg) . $mock_uniqueext_opt .
     " --rebuild " . sh_quote($srpm) .
     " --define " . sh_quote("use_source_date_epoch_as_buildtime 1") .
@@ -205,7 +205,7 @@ if (!$skip_install && $target_arch ne $arch) {
     # A cross-built rpm cannot be installed on this host: install it into the (emulated) build
     # chroot instead and run the binary there.
     print_step("Install RPM into the chroot and run smoke tests");
-    run("mock -r " . sh_quote($det_mock_cfg) . $mock_uniqueext_opt . " --install " . sh_quote($main_rpm)
+    run_mock("mock -r " . sh_quote($det_mock_cfg) . $mock_uniqueext_opt . " --install " . sh_quote($main_rpm)
         . " > " . sh_quote("$log_dir/smoke-chroot-install.log") . " 2>&1");
     my $bin = '/opt/xcat/bin/ipmitool-xcat';
     my $version_log = "$log_dir/smoke-version.log";
@@ -403,6 +403,22 @@ sub capture {
     return $out;
 }
 
+# mock exits 30 when its package manager failed (chroot init, build deps), which against public
+# mirrors is most often a transient download error (stale mirror metadata): retry such a run once.
+sub run_mock {
+    my ($cmd) = @_;
+    print "+ $cmd\n";
+    my $rc = system($cmd);
+    if ($rc != -1 && ($rc >> 8) == 30) {
+        print "mock failed with rc=30 (package manager); retrying once\n";
+        $rc = system($cmd);
+    }
+    if ($rc != 0) {
+        my $exit = $rc == -1 ? 255 : ($rc >> 8);
+        die "Command failed (rc=$exit): $cmd\n";
+    }
+}
+
 sub run_capture_rc {
     my ($cmd, $log_file) = @_;
     my $full = "$cmd > " . sh_quote($log_file) . " 2>&1";

mockbuild-perl-packages.pl

@@ -514,7 +514,7 @@ sub build_package {
 
             my $srpm_result = "$pkg_run_dir/srpm";
             make_path($srpm_result);
-            run(
+            run_mock(
                 "mock -r " . sh_quote($det_mock_cfg) . $mock_uniqueext_opt .
                 " --buildsrpm --spec " . sh_quote($spec) .
                 " --sources " . sh_quote($source_dir) .
@@ -539,7 +539,7 @@ sub build_package {
             $additional_opt .= " --additional-package " . sh_quote($_) for @need_rpms;
         }
 
-        run(
+        run_mock(
             "mock -r " . sh_quote($det_mock_cfg) . $mock_uniqueext_opt .
             " --rebuild " . sh_quote($srpm_path) . $additional_opt .
             " --define " . sh_quote("use_source_date_epoch_as_buildtime 1") .
@@ -784,6 +784,22 @@ sub capture {
     return $out;
 }
 
+# mock exits 30 when its package manager failed (chroot init, build deps), which against public
+# mirrors is most often a transient download error (stale mirror metadata): retry such a run once.
+sub run_mock {
+    my ($cmd) = @_;
+    print "+ $cmd\n";
+    my $rc = system($cmd);
+    if ($rc != -1 && ($rc >> 8) == 30) {
+        print "mock failed with rc=30 (package manager); retrying once\n";
+        $rc = system($cmd);
+    }
+    if ($rc != 0) {
+        my $exit = $rc == -1 ? 255 : ($rc >> 8);
+        die "Command failed (rc=$exit): $cmd\n";
+    }
+}
+
 sub run_capture_rc {
     my ($cmd, $log_file) = @_;
     my $full = "$cmd > " . sh_quote($log_file) . " 2>&1";