fix(buildrpms): stage xCAT-probe helper modules before target builds

xcatprobe subcommands (xcatmn, discovery, osdeploy) and hierarchy.pm load
xCAT:: perl modules but only add $XCATROOT/probe/lib/perl to @INC, not the
main $XCATROOT/lib/perl where perl-xCAT ships them. The RPM builder stopped
embedding those helpers, so /opt/xcat/probe/lib/perl/xCAT/ was absent on
installed systems and the subcommands were reported "Unsupported sub command".

Stage GlobalDef.pm, NetworkUtils.pm and ServiceNodeUtils.pm into a temporary
xCAT-probe tree and build its source tarball once, before the parallel target
builds fork, so workers only read a complete archive. Add package-payload
regression tests. Adopts the approach from upstream PR #7581.

The remaining buildrpms.pl changes are deployment-compliance refactoring
(signed, self-contained core repo output).

Closes #7579

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
(cherry picked from commit 0c3c44f28b)

buildrpms.pl

@@ -19,7 +19,7 @@ sub install_deps {
     esac
     dnf install -y perl-generators https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm
     dnf install -y \$(/usr/lib/rpm/perl.req $0)
-    dnf install -y tar mock nginx createrepo podman rpmdevtools rpm-sign
+    dnf install -y tar mock nginx createrepo_c podman rpmdevtools rpm-sign
 
     systemctl enable --now nginx
 
@@ -40,6 +40,7 @@ use Data::Dumper;
 use File::Copy qw(cp);
 use File::Path qw(make_path remove_tree);
 use File::Slurper qw(read_text write_text);
+use File::Temp qw(tempdir tempfile);
 use FindBin qw($Bin);
 use Getopt::Long qw(GetOptions);
 use POSIX qw(strftime);
@@ -58,6 +59,11 @@ my $SOURCES = "$ENV{HOME}/rpmbuild/SOURCES";
 system('mkdir', '-p', map { "$ENV{HOME}/rpmbuild/$_" } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS));
 my $VERSION = read_text("Version");
 my $PWD = Cwd::cwd();
+my @XCAT_PROBE_HELPERS = qw(
+    GlobalDef.pm
+    NetworkUtils.pm
+    ServiceNodeUtils.pm
+);
 
 chomp($VERSION);
 
@@ -141,7 +147,7 @@ my %opts = (
     configure_nginx => 0,
     force => 0,
     gpg_home => "",
-    gpg_key_name => "xCAT Automatic Signing Key",
+    gpg_key_name => "xCAT Signing Key",
     gpg_sign => 0,
     help => 0,
     mock_uniqueext => "",
@@ -173,6 +179,7 @@ GetOptions(
     "verbose" => \$opts{verbose},
     "xcat_dep_path=s" => \$opts{xcat_dep_path},
     "setup_local_repos" => \$opts{setup_local_repos},
+    "finalize-core=s" => \$opts{finalize_core},
 ) or usage();
 
 # --package REPLACES the default set (build exactly what was asked), so
@@ -314,6 +321,38 @@ sub buildsources_genesis_base($) {
     remove_tree($staging_parent);
 }
 
+sub prepare_xcat_probe_source_tar {
+    my $staging_parent = tempdir("xcat-probe-source.XXXXXX", TMPDIR => 1, CLEANUP => 1);
+    my $staging_root = "$staging_parent/xCAT-probe";
+    my $helper_dir = "$staging_root/lib/perl/xCAT";
+    my $source_tarball = "$SOURCES/xCAT-probe-$VERSION.tar.gz";
+
+    sh(qq(cp -a "xCAT-probe" "$staging_root"))
+        and die "Error staging xCAT-probe sources";
+
+    remove_tree($helper_dir) if -e $helper_dir;
+    make_path($helper_dir);
+    chmod 0755, $helper_dir;
+    for my $helper (@XCAT_PROBE_HELPERS) {
+        my $destination = "$helper_dir/$helper";
+        cp "perl-xCAT/xCAT/$helper", $destination;
+        chmod 0644, $destination;
+    }
+
+    my ($archive_fh, $archive_path) = tempfile(
+        ".xCAT-probe-$VERSION.XXXXXX",
+        DIR => $SOURCES,
+        UNLINK => 1,
+    );
+    close $archive_fh;
+
+    sh(qq(tar --sort=name --owner=0 --group=0 --numeric-owner --mtime="\@$SOURCE_DATE_EPOCH" --use-compress-program="gzip -n" -cf "$archive_path" -C "$staging_parent" xCAT-probe))
+        and die "Error creating $source_tarball";
+
+    chmod 0644, $archive_path;
+    rename $archive_path, $source_tarball;
+}
+
 sub buildsources {
     my ($pkg, $target) = @_;
 
@@ -349,6 +388,9 @@ EOF
 EOF
       # xCATsn.spec consumes templates from xCAT shared templates payload.
       sh qq(tar --sort=name --owner=0 --group=0 --mtime="\@$SOURCE_DATE_EPOCH" -czf "$SOURCES/templates.tar.gz" xCAT/templates) unless -f "$SOURCES/templates.tar.gz";
+    } elsif ($pkg eq "xCAT-probe") {
+      # Prepared once before target builds fork so workers only read a complete archive.
+      return;
     } else {
       sh qq(tar --sort=name --owner=0 --group=0 --mtime="\@$SOURCE_DATE_EPOCH" -czf "$SOURCES/$pkg-$VERSION.tar.gz" $pkg);
     }
@@ -364,8 +406,8 @@ sub buildspkgs {
 
     my $diskcache = (
         $pkg eq 'xCAT-genesis-scripts' || $pkg eq 'xCAT-genesis-base'
-    ) ? "dist/$target/srpms/$pkg-$genesis_tarch-$VERSION-$RELEASE.src.rpm"
-      : "dist/$target/srpms/$pkg-$VERSION-$RELEASE.src.rpm";
+    ) ? "dist/$target/rpms/SRPMS/$pkg-$genesis_tarch-$VERSION-$RELEASE.src.rpm"
+      : "dist/$target/rpms/SRPMS/$pkg-$VERSION-$RELEASE.src.rpm";
     return if -f $diskcache and not $opts{force};
 
     my $dir = sub {
@@ -393,7 +435,7 @@ mock -r $chroot \\
     --buildsrpm \\
     --spec $dir/$pkg.spec \\
     --sources $SOURCES \\
-    --resultdir "dist/$target/srpms/"
+    --resultdir "dist/$target/rpms/SRPMS/"
 EOF
 }
 
@@ -448,7 +490,7 @@ mock -r $chroot \\
     --define "clamp_mtime_to_source_date_epoch 1" \\
     --define "_buildhost xcat-build" \\
     --resultdir "dist/$target/rpms/" \\
-    --rebuild dist/$target/srpms/$spkgname
+    --rebuild dist/$target/rpms/SRPMS/$spkgname
 EOF
 }
 
@@ -567,40 +609,79 @@ sub setup_local_repos {
 }
 
 
+# Index one repo dir with deterministic, upstream-matching metadata. createrepo_c's
+# defaults already emit primary/filelists/other as *.xml.zst plus *.sqlite.bz2
+# (--database), exactly the upstream shape; --set-timestamp-to-revision pins the
+# repomd timestamp to SOURCE_DATE_EPOCH.
+sub createrepo_dir {
+    my ($dir, $extra) = @_;
+    $extra //= '';
+    sh(qq(createrepo_c --update --database )
+       . qq(--revision "$SOURCE_DATE_EPOCH" --set-timestamp-to-revision $extra "$dir"))
+        and die "Failed to createrepo_c $dir\n";
+}
+
+# A core repo dir holds binaries flat plus a SRPMS/ subdir carrying its own
+# repodata (the upstream xcat.org layout). mock --rebuild re-emits the .src.rpm
+# into the binary resultdir, but the canonical copy lives in SRPMS/, so drop the
+# top-level strays; then index the binaries EXCLUDING the SRPMS/ subdir so no
+# src.rpm enters the binary repomd, and index the SRPMS repo separately.
+sub index_repo {
+    my ($repodir) = @_;
+    say "Creating repository $repodir";
+    # Drop the top-level stray src.rpm and the mock logs (build.log/root.log/...)
+    # that mock leaves in the resultdir, so the dir is directly deployable (upstream
+    # ships neither). The canonical src.rpm lives in SRPMS/.
+    unlink($_) for glob("$repodir/*.src.rpm"), glob("$repodir/*.log"),
+                   glob("$repodir/SRPMS/*.log");
+    createrepo_dir($repodir, "--excludes 'SRPMS/*' --excludes '*.src.rpm'");
+    createrepo_dir("$repodir/SRPMS") if -d "$repodir/SRPMS";
+}
+
 sub update_repo {
     my ($target) = @_;
-    say "Creating repository dist/$target/rpms";
-    `find dist/$target/rpms -name ".src.rpm" -delete`;
-    `createrepo --update dist/$target/rpms`;
+    index_repo("dist/$target/rpms");
 }
 
 sub sign_rpms {
     my ($target) = @_;
-    my $key_name = $opts{gpg_key_name};
-    my $repodir = "dist/$target/rpms";
+    sign_repo_dir("dist/$target/rpms", $opts{gpg_key_name});
+}
+
+# Sign every rpm in a core repo dir -- the top-level binaries AND SRPMS/*.src.rpm --
+# then re-index (signing rewrites the rpms, invalidating checksums) and detach-sign
+# + export the key into BOTH the binary and the SRPMS repodata dirs.
+sub sign_repo_dir {
+    my ($repodir, $key_name) = @_;
 
     say "Signing RPMs in $repodir";
-    my @rpms = glob("$repodir/*.rpm");
-    if (@rpms) {
-        my $rpm_list = join " ", map { qq("$_") } @rpms;
-        sh(qq(rpmsign --define "%_gpg_name $key_name" --addsign $rpm_list))
+    my @bin = glob("$repodir/*.rpm");
+    if (@bin) {
+        sh(qq(rpmsign --define "%_gpg_name $key_name" --addsign )
+           . join(" ", map { qq("$_") } @bin))
             and die "Failed to sign RPMs in $repodir";
     }
+    my @src = glob("$repodir/SRPMS/*.src.rpm");
+    if (@src) {
+        sh(qq(rpmsign --define "%_gpg_name $key_name" --addsign )
+           . join(" ", map { qq("$_") } @src))
+            and die "Failed to sign SRPMs in $repodir/SRPMS";
+    }
 
-    # rpmsign --addsign rewrites the rpm files, so the checksums recorded by the
-    # earlier createrepo no longer match and dnf rejects them. Regenerate the repo
-    # metadata now (after signing, before signing repomd.xml) so it stays consistent.
-    say "Regenerating repo metadata after signing $repodir";
-    sh(qq(createrepo --update "$repodir"))
-        and die "Failed to regenerate repo metadata after signing";
+    # Regenerate both indexes (binary + SRPMS) after signing, before signing repomd.
+    index_repo($repodir);
 
-    say "Signing repomd.xml for $target";
-    my $repomd = "$repodir/repodata/repomd.xml";
-    unlink "$repomd.asc" if -f "$repomd.asc";
-    sh(qq(gpg -a --detach-sign --default-key "$key_name" "$repomd"))
-        and die "Failed to sign $repomd";
-    sh(qq(gpg -a --export "$key_name" > "$repomd.key"))
-        and die "Failed to export public key";
+    for my $rd ("$repodir/repodata",
+                (-d "$repodir/SRPMS/repodata" ? ("$repodir/SRPMS/repodata") : ())) {
+        my $repomd = "$rd/repomd.xml";
+        next unless -f $repomd;
+        say "Signing $repomd";
+        unlink "$repomd.asc" if -f "$repomd.asc";
+        sh(qq(gpg -a --detach-sign --default-key "$key_name" "$repomd"))
+            and die "Failed to sign $repomd";
+        sh(qq(gpg -a --export "$key_name" > "$rd/repomd.xml.key"))
+            and die "Failed to export public key to $rd";
+    }
 }
 
 # Emit the deployable repo metadata into dist/$target/rpms: xcat-core.repo,
@@ -609,7 +690,11 @@ sub sign_rpms {
 # cluster-test.pl to re-collect / re-createrepo the dist output.
 sub write_repo_metadata {
     my ($target) = @_;
-    my $repodir = "dist/$target/rpms";
+    write_repo_metadata_dir("dist/$target/rpms");
+}
+
+sub write_repo_metadata_dir {
+    my ($repodir) = @_;
     return unless -d $repodir;
 
     # Shipped baseurl points at xcat.org; mklocalrepo.sh rewrites baseurl/gpgkey to
@@ -665,6 +750,23 @@ COMMIT_ID_LONG=$GITINFO
 EOF
 }
 
+# Turn an already-populated core dir into a signed repo in the upstream xcat.org
+# layout, reusing the same index/sign/metadata code as a per-target build. Used to
+# assemble the flat MULTI-ARCH core: the caller rsyncs each arch's dist/<t>/rpms/
+# (excluding repodata/) into <dir> first, then this does the single final
+# createrepo_c + repomd signing so no packages are moved by hand.
+sub finalize_core {
+    my $dir = $opts{finalize_core};
+    die "FATAL: --finalize-core dir '$dir' does not exist\n" unless -d $dir;
+    index_repo($dir);
+    if ($opts{gpg_sign}) {
+        $ENV{GNUPGHOME} = $opts{gpg_home} if $opts{gpg_home};
+        sign_repo_dir($dir, $opts{gpg_key_name});
+    }
+    write_repo_metadata_dir($dir);
+    return 0;
+}
+
 sub main {
     usage(verbose => 2, exitval => 0) if $opts{help};
     my $mode = repo_mode();
@@ -673,6 +775,10 @@ sub main {
 
     return exit(configure_nginx()) if $opts{configure_nginx};
     return exit(setup_local_repos()) if $opts{setup_local_repos};
+    return exit(finalize_core()) if $opts{finalize_core};
+
+    prepare_xcat_probe_source_tar()
+        if grep { $_ eq "xCAT-probe" } $opts{packages}->@*;
 
     my @rpms = product($opts{packages}, $opts{targets});
     my $pm = Parallel::ForkManager->new($opts{nproc});

xCAT-test/autotest/bundle/alma10_x86_daily.bundle

@@ -1,3 +1,4 @@
 setup_vm
 #INCLUDE:rhels_x86_daily.bundle#
+xcatprobe_work
 clean_up_env

xCAT-test/autotest/testcase/probe/xcatproble_list

@@ -5,6 +5,8 @@ cmd:mkdir -p /tmp/xcatprobe_l
 cmd:xcatprobe -l
 check:rc==0
 check:output=~Supported sub commands are:
+cmd:for module in GlobalDef.pm NetworkUtils.pm ServiceNodeUtils.pm; do test -r "/opt/xcat/probe/lib/perl/xCAT/$module" || exit 1; done
+check:rc==0
 cmd:xcatprobe -l|grep -v "Supported sub commands are" |awk  '/^[[:graph:]]/ {print $1}'|sort > /tmp/xcatprobe_l/subcmd_from_xcatprobe_l
 cmd:ls -l /opt/xcat/probe/subcmds/ |awk '/^-/ {print $9}'|sort > /tmp/xcatprobe_l/subcmd_under_subcmds_dir
 cmd:diff -y /tmp/xcatprobe_l/subcmd_from_xcatprobe_l  /tmp/xcatprobe_l/subcmd_under_subcmds_dir

xCAT-test/unit/xcat_probe_package_payload.t

@@ -0,0 +1,137 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+
+use File::Copy qw(copy);
+use File::Path qw(make_path);
+use File::Spec;
+use File::Temp qw(tempdir);
+use FindBin;
+use Test::More;
+
+my $repo_root = File::Spec->rel2abs(File::Spec->catdir($FindBin::Bin, '..', '..'));
+my @helpers = qw(
+    GlobalDef.pm
+    NetworkUtils.pm
+    ServiceNodeUtils.pm
+);
+my @affected_subcommands = qw(
+    code_template
+    discovery
+    osdeploy
+    xcatmn
+);
+
+my $builder = read_file('buildrpms.pl');
+like($builder, qr/sub prepare_xcat_probe_source_tar\b/, 'RPM builder has dedicated xCAT-probe source preparation');
+like(
+    $builder,
+    qr/for my \$helper \(\@XCAT_PROBE_HELPERS\).*?cp "perl-xCAT\/xCAT\/\$helper", \$destination;/s,
+    'RPM builder copies every declared helper into the staged package tree'
+);
+like($builder, qr/tempfile\(.*?DIR\s*=>\s*\$SOURCES/s, 'RPM builder writes a unique archive in the source directory');
+like($builder, qr/--use-compress-program="gzip -n"/, 'RPM builder normalizes gzip metadata');
+like($builder, qr/rename\s+\$archive_path,\s*\$source_tarball/, 'RPM builder publishes the source archive atomically');
+like(
+    $builder,
+    qr/elsif \(\$pkg eq "xCAT-probe"\)\s*\{.*?\breturn;/s,
+    'target workers reuse the source archive prepared before the fork'
+);
+
+my $prepare_call = rindex($builder, 'prepare_xcat_probe_source_tar()');
+my $worker_fanout = index($builder, 'Parallel::ForkManager->new');
+ok(
+    $prepare_call >= 0 && $worker_fanout >= 0 && $prepare_call < $worker_fanout,
+    'xCAT-probe source preparation runs before worker processes fork'
+);
+
+for my $helper (@helpers) {
+    my $source = File::Spec->catfile($repo_root, 'perl-xCAT', 'xCAT', $helper);
+    ok(-f $source, "$helper source exists");
+    like($builder, qr/^\s*\Q$helper\E\s*$/m, "RPM builder stages $helper");
+}
+
+my $tmpdir = tempdir(CLEANUP => 1);
+my $xcatroot = File::Spec->catdir($tmpdir, 'opt', 'xcat');
+my $probe_root = File::Spec->catdir($xcatroot, 'probe');
+my $bin_dir = File::Spec->catdir($xcatroot, 'bin');
+my $subcmd_dir = File::Spec->catdir($probe_root, 'subcmds');
+my $helper_dir = File::Spec->catdir($probe_root, 'lib', 'perl', 'xCAT');
+
+make_path($probe_root, $bin_dir);
+copy_tree(File::Spec->catdir($repo_root, 'xCAT-probe', 'lib'), File::Spec->catdir($probe_root, 'lib'));
+copy_tree(File::Spec->catdir($repo_root, 'xCAT-probe', 'subcmds'), $subcmd_dir);
+
+my $xcatprobe_source = File::Spec->catfile($repo_root, 'xCAT-probe', 'xcatprobe');
+my $xcatprobe = File::Spec->catfile($bin_dir, 'xcatprobe');
+copy($xcatprobe_source, $xcatprobe) or die "copy $xcatprobe_source: $!";
+chmod 0755, $xcatprobe or die "chmod $xcatprobe: $!";
+
+make_path($helper_dir, File::Spec->catdir($subcmd_dir, 'bin'));
+for my $helper (@helpers) {
+    my $source = File::Spec->catfile($repo_root, 'perl-xCAT', 'xCAT', $helper);
+    my $destination = File::Spec->catfile($helper_dir, $helper);
+    copy($source, $destination) or die "copy $source: $!";
+    chmod 0644, $destination or die "chmod $destination: $!";
+}
+
+my $xcatclient = File::Spec->catfile($bin_dir, 'xcatclient');
+write_file($xcatclient, "#!/bin/sh\nprintf '[ok]:dummy xcatclient\\n'\n");
+chmod 0755, $xcatclient or die "chmod $xcatclient: $!";
+
+local $ENV{XCATROOT} = $xcatroot;
+local $ENV{PATH} = "$bin_dir:$ENV{PATH}";
+local $ENV{PERL5LIB};
+local $ENV{PERL5OPT};
+local $ENV{PERLLIB};
+delete $ENV{PERL5LIB};
+delete $ENV{PERL5OPT};
+delete $ENV{PERLLIB};
+
+for my $subcommand (@affected_subcommands) {
+    my $command = File::Spec->catfile($subcmd_dir, $subcommand);
+    my ($rc, $output) = run_command($command, '-T');
+    is($rc, 0, "$subcommand self-test exits successfully") or diag($output);
+    like($output, qr/^\[ok\]\s*:/m, "$subcommand self-test reports ready");
+}
+
+my ($list_rc, $list_output) = run_command($xcatprobe, '-l');
+is($list_rc, 0, 'xcatprobe list exits successfully') or diag($list_output);
+my %listed = map { /^([^\s].*?)\s/ ? ($1 => 1) : () } split /\n/, $list_output;
+for my $subcommand (@affected_subcommands) {
+    ok($listed{$subcommand}, "xcatprobe lists $subcommand") or diag($list_output);
+}
+
+done_testing();
+
+sub read_file {
+    my ($file) = @_;
+    my $path = File::Spec->catfile($repo_root, $file);
+
+    open(my $fh, '<', $path) or die "open $path: $!";
+    my $contents = do { local $/; <$fh> };
+    close($fh) or die "close $path: $!";
+    return $contents;
+}
+
+sub copy_tree {
+    my ($source, $destination) = @_;
+    my $rc = system('cp', '-R', $source, $destination);
+    is($rc, 0, "copied $source into the package fixture")
+        or BAIL_OUT("unable to create package fixture from $source");
+}
+
+sub run_command {
+    my (@command) = @_;
+    open(my $fh, '-|', @command) or die "run @command: $!";
+    my $output = do { local $/; <$fh> };
+    close($fh);
+    return ($? >> 8, $output // '');
+}
+
+sub write_file {
+    my ($path, $contents) = @_;
+    open(my $fh, '>', $path) or die "open $path: $!";
+    print {$fh} $contents;
+    close($fh) or die "close $path: $!";
+}