From bfe52d03a7d698f8f40d265e1450a98c5c175316 Mon Sep 17 00:00:00 2001 From: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> Date: Wed, 3 Dec 2025 15:50:05 -0300 Subject: [PATCH 1/9] fix: Fix build with mock Add buildrpms.pl to build RPMs in parallel using mock Add xCAT-buildkit to the build list Fix build dependency in xCAT-buildkit.spec Add fallback in /etc/init.d/xcatd for /etc/rc.d/init.d/functions Signed-off-by: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> --- buildrpms.pl | 270 +++++++++++++++++++++++++++++++ buildrpms.sh | 170 ------------------- xCAT-buildkit/xCAT-buildkit.spec | 2 + xCAT-server/etc/init.d/xcatd | 7 + 4 files changed, 279 insertions(+), 170 deletions(-) create mode 100644 buildrpms.pl delete mode 100755 buildrpms.sh diff --git a/buildrpms.pl b/buildrpms.pl new file mode 100644 index 000000000..0f8a3336a --- /dev/null +++ b/buildrpms.pl @@ -0,0 +1,270 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use feature 'say'; + +use Data::Dumper; +use File::Copy (); +use File::Slurper qw(read_text write_text); +use Parallel::ForkManager; +use Getopt::Long qw(GetOptions); +use Cwd qw(); + +my $SOURCES = "$ENV{HOME}/rpmbuild/SOURCES"; +my $VERSION = read_text("Version"); +my $RELEASE = read_text("Release"); +my $GITINFO = read_text("Gitinfo"); +my $PWD = Cwd::cwd(); + +chomp($VERSION); +chomp($RELEASE); +chomp($GITINFO); + +my @PACKAGES = qw( + perl-xCAT + xCAT + xCAT-buildkit + xCAT-client + xCAT-confluent + xCAT-openbmc-py + xCAT-probe + xCAT-rmc + xCAT-server + xCAT-test + xCAT-vlan); + +my @TARGETS = qw( + rhel+epel-8-x86_64 + rhel+epel-9-x86_64 + rhel+epel-10-x86_64); + +my $NPROC = `nproc --all`; + +# cp $src, $dst copies $src to $dst or aborts with an error message +sub cp { + my ($src, $dst) = @_; + File::Copy::copy($src, $dst) or die "copy $src, $dst failed: $!"; +} + +# sed { s/foo/bar/ } $filepath applies s/foo/bar/ to the file at $filepath +sub sed (&$) { + my ($block, $path) = @_; + my $content = read_text($path); + local $_ = $content; + $block->(); + $content = $_; + write_text($path, $content); +} + +# product(\@A, \@B) returns the catersian product of \@A and \@B +sub product { + my ($a, $b) = @_; + return map { + my $x = $_; + map [ $x, $_ ], @$b; + } @$a +} + +sub createmockconfig { + my ($pkg, $target) = @_; + my $chroot = "$pkg-$target"; + my $cfgfile = "/etc/mock/$chroot.cfg"; + return if -f $cfgfile; + cp "/etc/mock/$target.cfg", $cfgfile; + my $contents = read_text($cfgfile); + $contents =~ s/config_opts\['root'\]\s\+=.*/config_opts['root'] = \"$chroot\"/; + if ($pkg eq "perl-xCAT") { + # perl-generators is required for having perl(xCAT::...) symbols + # exported by the RPM + $contents .= "config_opts['chroot_additional_packages'] = 'perl-generators'\n"; + } + write_text($cfgfile, $contents); +} + +sub buildsources { + my ($pkg, $target) = @_; + + if ($pkg eq "xCAT") { + my @files = ("bmcsetup", "getipmi"); + for my $f (@files) { + cp "xCAT-genesis-scripts/usr/bin/$f", "$pkg/postscripts/$f"; + sed { s/xcat.genesis.$f/$f/ } "${pkg}/postscripts/$f"; + } + qx {bash -c ' + cd xCAT + tar --exclude upflag -czf $SOURCES/postscripts.tar.gz postscripts LICENSE.html + tar -czf $SOURCES/prescripts.tar.gz prescripts + tar -czf $SOURCES/templates.tar.gz templates + tar -czf $SOURCES/winpostscripts.tar.gz winpostscripts + tar -czf $SOURCES/etc.tar.gz etc + cp xcat.conf $SOURCES + cp xcat.conf.apach24 $SOURCES + cp xCATMN $SOURCES + '}; + } else { + `tar -czf "$SOURCES/$pkg-$VERSION.tar.gz" $pkg`; + } +} + +sub buildspkgs { + my ($pkg, $target, $optsref) = @_; + my $chroot = "$pkg-$target"; + + my $diskcache = "dist/$target/srpms/$pkg-$VERSION-$RELEASE.src.rpm"; + return if -f $diskcache and not $optsref->{force}; + + my @opts; + push @opts, "--quiet" unless $optsref->{verbose}; + + say "Building $diskcache"; + + `mock -r $chroot \\ + -N \\ + @{[ join " ", @opts ]} \\ + --define "version $VERSION" \\ + --define "release $RELEASE" \\ + --define "gitinfo $GITINFO" \\ + --buildsrpm \\ + --spec $pkg/$pkg.spec \\ + --sources $SOURCES \\ + --resultdir "dist/$target/srpms/"`; +} + +sub buildpkgs { + my ($pkg, $target, $optsref) = @_; + my $chroot = "$pkg-$target"; + + my $targetarch = (split /-/, $target, 3)[2]; + my $arch = $pkg eq "xCAT" ? $targetarch : "noarch"; + + my $diskcache = "dist/$target/rpms/$pkg-$VERSION-$RELEASE.$arch.rpm"; + return if -f $diskcache and not $optsref->{force}; + + my @opts; + push @opts, "--quiet" unless $optsref->{verbose}; + + say "Building $diskcache"; + + `mock -r $chroot \\ + -N \\ + @{[ join " ", @opts ]} \\ + --define "version $VERSION" \\ + --define "release $RELEASE" \\ + --define "gitinfo $GITINFO" \\ + --resultdir "dist/$target/rpms/" \\ + --rebuild dist/$target/srpms/$pkg-${VERSION}-${RELEASE}.src.rpm`; +} + +sub buildall { + my ($pkg, $target, $optsref) = @_; + createmockconfig($pkg, $target, $optsref); + buildsources($pkg, $target, $optsref); + buildspkgs($pkg, $target, $optsref); + buildpkgs($pkg, $target, $optsref); +} + +sub configure_nginx { + my ($args) = @_; + my @targets = $args->{targets}->@*; + my $deps = $args->{deps}; + my $conf = <<"EOF"; +server { + listen 8080; + listen [::]:8080; +EOF + for my $target (@targets) { + my $fullpath = "$PWD/dist/$target/rpms"; + $conf .= <<"EOF"; + location /$target/ { + alias $fullpath/; + autoindex on; + index off; + allow all; + } +EOF + } + # TODO:I need one xcat-dep for each target + $conf .= <<"EOF"; + location /xcat-dep/ { + alias $deps; + autoindex on; + index off; + allow all; + } +} +EOF + write_text("/etc/nginx/conf.d/xcat-repos.conf", $conf); +} + +sub update_repo { + my ($target) = @_; + say "Creating repository dist/$target/rpms"; + `find dist/$target/rpms -name ".src.rpm" -delete`; + `createrepo --update dist/$target/rpms`; +} + + +sub usage { + my ($errmsg) = @_; + say STDERR "Usage: $0 [--package=] [--target=] [--package=] [--target=] ..."; + + say STDERR $errmsg if $errmsg; + exit -1; +} + +sub parseopts { + my %opts = ( + targets => \@TARGETS, + packages => \@PACKAGES, + nproc => int(`nproc --all`), + force => 0, + verbose => 0, + deps => "$PWD/../xcat-dep/", + ); + GetOptions( + "target=s@" => \$opts{targets}, + "package=s@" => \$opts{packages}, + "nproc=i" => \$opts{nproc}, + "verbose" => \$opts{verbose}, + "force" => \$opts{force}, + "deps=s" => \$opts{deps}, + ) or usage(); + + return \%opts; + +}; + +sub main { + my $opts = parseopts(); + my @rpms = product($opts->{packages}, $opts->{targets}); + my $pm = Parallel::ForkManager->new($opts->{nproc}); + + for my $pair (@rpms) { + my ($pkg, $target) = $pair->@*; + $pm->start and next; + + buildall($pkg, $target, $opts); + + $pm->finish; + } + + $pm->wait_all_children; + + for my $target ($opts->{targets}->@*) { + $pm->start and next; + + update_repo($target, $opts); + + $pm->finish; + } + $pm->wait_all_children; + + configure_nginx($opts); + + `systemctl restart nginx`; +} + +main(); + diff --git a/buildrpms.sh b/buildrpms.sh deleted file mode 100755 index 8d5cc198b..000000000 --- a/buildrpms.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/bin/bash -e - -# Build xCAT rpms for multiple targets using mock for -# chroot encapsulation. There are multiple packages and -# multiple targets. A package is a xCAT package, like -# xCAT-server, a target is a mock chroot, like rhel+epel-9-x86_64. -# -# This script has two modes: -# ./buildrpms.sh all -# ./buildrpms.sh single -# -# In 'all' mode it builds ALL packages for ALL targets, this -# is, a cartesian product PKGS x TARGETS, a RPM is a pair -# (PKG, TARGET). -# -# In 'single' mode a single RPM is built. It is intented for -# reproducing failing builds and for debugging. -# -# In order to make 'all' mode faster the builds are parallelized. Since mock -# lock chroots a distinct chroot is created for each RPM with name -# -, then multiple mock processes are spawn, each with its own -# chroot. -# -# The output is generated at dist/ folder, both .src.rpm and .rpm files are -# in this folder. -# -# Notes: -# - `xargs` requires shell functions and variables to be exported -# - Because `buildall` is executed by xargs, I updated it to accept a single -# argument : intead of two arguments - -# pkgs built by default -PACKAGES=(xCAT-{server,client,probe,openbmc-py,rmc,test,vlan,confluent} perl-xCAT xCAT) - -# All targets -TARGETS=(rhel+epel-{8,9,10}-x86_64) - -SOURCES=${SOURCES:-/root/rpmbuild/SOURCES/} -VERSION=$(cat Version) -RELEASE=$(cat Release) -GITINFO=$(cat Gitinfo) - -NPROC=${NPROC:-$(nproc --all)} - -export SOURCES VERSION RELEASE GITINFO - -# Holds all RPMS to be built -declare -a RPMS=() -for target in ${TARGETS[@]}; do - for pkg in ${PACKAGES[@]}; do - RPMS+=("$pkg:$target") - done -done - -# Create the mock chroot configurations for each $pkg $target -function createmockchroot() { - local pkg=$1 - local target=$2 - local chroot="$pkg-$target" - if [ ! -f "/etc/mock/$chroot.cfg" ]; then - cp "/etc/mock/$target.cfg" "/etc/mock/$chroot.cfg" - sed -e "s/config_opts\['root'\]\s\+=.*/config_opts['root'] = \"$chroot\"/" \ - -i "/etc/mock/$chroot.cfg" - fi -} -export -f createmockchroot - -# Create a tarball of the source code into $SOURCES/. These tarballs -# are then read by the .spec files -function buildsources() { - local pkg=$1 - - case $1 in - xCAT) - # shipping bmcsetup and getipmi scripts as part of postscripts - files=("bmcsetup" "getipmi") - for f in "${files[@]}"; do - cp "xCAT-genesis-scripts/usr/bin/"$f ${pkg}/postscripts/$f - sed -i "s/xcat.genesis.$f/$f/g" ${pkg}/postscripts/$f - done - cd xCAT - tar --exclude upflag -czf $SOURCES/postscripts.tar.gz postscripts LICENSE.html - tar -czf $SOURCES/prescripts.tar.gz prescripts - tar -czf $SOURCES/templates.tar.gz templates - tar -czf $SOURCES/winpostscripts.tar.gz winpostscripts - tar -czf $SOURCES/etc.tar.gz etc - cp xcat.conf $SOURCES - cp xcat.conf.apach24 $SOURCES - cp xCATMN $SOURCES - cd .. - ;; - *) - tar -czf "$SOURCES/$pkg-$VERSION.tar" $pkg - ;; - esac -} -export -f buildsources - -# Build the .src.rpm files -function buildspkgs() { - local pkg=$1 - local target=$2 - local chroot="$pkg-$target" - mock -r $chroot \ - -N \ - --quiet \ - --define "version $VERSION" \ - --define "release $RELEASE" \ - --define "gitinfo $GITINFO" \ - --buildspkg \ - --spec $pkg/$pkg.spec \ - --sources $SOURCES \ - --resultdir "dist/$target/$pkg/" -} -export -f buildspkgs - -# Build the .noarch.rpm files -function buildpkgs() { - local pkg=$1 - local target=$2 - local chroot="$pkg-$target" - mock -r $chroot \ - -N \ - --quiet \ - --define "version $VERSION" \ - --define "release $RELEASE" \ - --define "gitinfo $GITINFO" \ - --resultdir "dist/$target/$pkg/" \ - dist/$target/$pkg/$pkg-${VERSION}-${RELEASE}.src.rpm -} -export -f buildpkgs - -# Receive a single argument with the format : -# Call each step required to build for -function buildall() { - IFS=: read -r pkg target <<< "$1" - createmockchroot $pkg $target - buildsources $pkg $target - buildspkgs $pkg $target - buildpkgs $pkg $target -} -export -f buildall - -function usage() { - echo "usage:. $0 single " - echo "usage:. $0 all" - echo " where:" - echo " PKG = one of xCAT-server,xCAT-client,.." - echo ' TARGET = one of `mock --list-chroots`' - echo ' all = build all combinations' - exit -1 -} - -test -d dist/ || mkdir dist/ - -if [ $# -ne 1 ] && [ $# -ne 3 ]; then - usage -fi - -case $1 in - 'single') - buildall "$2:$3" - ;; - 'all') - echo -n ${RPMS[@]} | xargs -d ' ' -I% -P $NPROC -t bash -euc "buildall %" - ;; - *) - usage - ;; -esac diff --git a/xCAT-buildkit/xCAT-buildkit.spec b/xCAT-buildkit/xCAT-buildkit.spec index 8531dac38..037dfcfc4 100644 --- a/xCAT-buildkit/xCAT-buildkit.spec +++ b/xCAT-buildkit/xCAT-buildkit.spec @@ -12,6 +12,8 @@ Distribution: %{?_distribution:%{_distribution}}%{!?_distribution:%{_vendor}} Prefix: /opt/xcat BuildRoot: /var/tmp/%{name}-%{version}-%{release}-root +BuildRequires: perl-Pod-Html + #%ifnos linux AutoReqProv: no #%endif diff --git a/xCAT-server/etc/init.d/xcatd b/xCAT-server/etc/init.d/xcatd index f92bd214d..0f40a5fee 100755 --- a/xCAT-server/etc/init.d/xcatd +++ b/xCAT-server/etc/init.d/xcatd @@ -61,6 +61,13 @@ if [ -f /etc/init.d/functions ]; then LOG_SUCCESS=RHSuccess LOG_FAILURE=RHFailure LOG_WARNING=passed +elif [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions + START_DAEMON=daemon + STATUS=MStatus + LOG_SUCCESS=RHSuccess + LOG_FAILURE=RHFailure + LOG_WARNING=passed elif [ -f /lib/lsb/init-functions ]; then . /lib/lsb/init-functions START_DAEMON=start_daemon From 83f6b7430233a3f508563149d8c5e9f402b8f881 Mon Sep 17 00:00:00 2001 From: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> Date: Fri, 5 Dec 2025 13:33:57 -0300 Subject: [PATCH 2/9] fix!: Skip settunnables if running inside a container This commit adds an early return to xcatconfig settunnables function. This function set parameters at /proc/sys/net/ipv4/neigh/default/gc_thresh1 /proc/sys/net/ipv4/neigh/default/gc_thresh2 /proc/sys/net/ipv4/neigh/default/gc_thresh3 And set sysctl attributes by writing to /etc/sysctl.d/ and /etc/sysctl.conf These are tunning network parameters for running on production and should not affect the overall function for testing purposes. Signed-off-by: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> --- xCAT-server/sbin/xcatconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/xCAT-server/sbin/xcatconfig b/xCAT-server/sbin/xcatconfig index e5a219e37..6b5120922 100755 --- a/xCAT-server/sbin/xcatconfig +++ b/xCAT-server/sbin/xcatconfig @@ -843,6 +843,9 @@ sub genSSHRootKeys sub settunables { + # tunnables are not settable inside containers, we use + # container for integration testing + return if $ENV{container}; # tuning ARP on Linux # set for right now From 5b7f5f45f97a5ead9823d3823ab43d6b9c3b48f9 Mon Sep 17 00:00:00 2001 From: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> Date: Fri, 5 Dec 2025 13:38:47 -0300 Subject: [PATCH 3/9] build: Add build and testing logic to be used in the CI Signed-off-by: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> --- buildrpms.pl | 140 ++++++++++++++++++---------------- test/Containerfile | 10 +++ test/scripts/setuptesthost.pl | 105 +++++++++++++++++++++++++ test/scripts/testxcat.pl | 112 +++++++++++++++++++++++++++ 4 files changed, 303 insertions(+), 64 deletions(-) mode change 100644 => 100755 buildrpms.pl create mode 100644 test/Containerfile create mode 100755 test/scripts/setuptesthost.pl create mode 100755 test/scripts/testxcat.pl diff --git a/buildrpms.pl b/buildrpms.pl old mode 100644 new mode 100755 index 0f8a3336a..337d501a5 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -22,6 +22,7 @@ chomp($VERSION); chomp($RELEASE); chomp($GITINFO); + my @PACKAGES = qw( perl-xCAT xCAT @@ -40,7 +41,38 @@ my @TARGETS = qw( rhel+epel-9-x86_64 rhel+epel-10-x86_64); -my $NPROC = `nproc --all`; + +my %opts = ( + targets => \@TARGETS, + packages => \@PACKAGES, + nproc => int(`nproc --all`), + force => 0, + verbose => 0, + deps => "$PWD/../xcat-dep/", +); + +GetOptions( + "target=s@" => \$opts{targets}, + "package=s@" => \$opts{packages}, + "nproc=i" => \$opts{nproc}, + "verbose" => \$opts{verbose}, + "force" => \$opts{force}, + "deps=s" => \$opts{deps}, +) or usage(); + +sub sh { + my ($cmd) = @_; + say "Running: $cmd" + if $opts{quiet}; + open my $fh, "-|", "bash -lc '$cmd'" or die "cannot run $cmd: $!"; + + while (my $line = <$fh>) { + print $line + if $opts{verbose}; + } + close $fh; + return $? >> 8; +} # cp $src, $dst copies $src to $dst or aborts with an error message sub cp { @@ -71,7 +103,7 @@ sub createmockconfig { my ($pkg, $target) = @_; my $chroot = "$pkg-$target"; my $cfgfile = "/etc/mock/$chroot.cfg"; - return if -f $cfgfile; + return if -f $cfgfile && ! $opts{force}; cp "/etc/mock/$target.cfg", $cfgfile; my $contents = read_text($cfgfile); $contents =~ s/config_opts\['root'\]\s\+=.*/config_opts['root'] = \"$chroot\"/; @@ -109,72 +141,75 @@ sub buildsources { } sub buildspkgs { - my ($pkg, $target, $optsref) = @_; + my ($pkg, $target) = @_; my $chroot = "$pkg-$target"; my $diskcache = "dist/$target/srpms/$pkg-$VERSION-$RELEASE.src.rpm"; - return if -f $diskcache and not $optsref->{force}; + return if -f $diskcache and not $opts{force}; my @opts; - push @opts, "--quiet" unless $optsref->{verbose}; + push @opts, "--quiet" unless $opts{verbose}; say "Building $diskcache"; - `mock -r $chroot \\ - -N \\ - @{[ join " ", @opts ]} \\ - --define "version $VERSION" \\ - --define "release $RELEASE" \\ - --define "gitinfo $GITINFO" \\ - --buildsrpm \\ - --spec $pkg/$pkg.spec \\ - --sources $SOURCES \\ - --resultdir "dist/$target/srpms/"`; + sh(<<"EOF"); +mock -r $chroot \\ + -N \\ + @{[ join " ", @opts ]} \\ + --define "version $VERSION" \\ + --define "release $RELEASE" \\ + --define "gitinfo $GITINFO" \\ + --buildsrpm \\ + --spec $pkg/$pkg.spec \\ + --sources $SOURCES \\ + --resultdir "dist/$target/srpms/" +EOF } sub buildpkgs { - my ($pkg, $target, $optsref) = @_; + my ($pkg, $target) = @_; + my $optsref = \%opts; my $chroot = "$pkg-$target"; my $targetarch = (split /-/, $target, 3)[2]; my $arch = $pkg eq "xCAT" ? $targetarch : "noarch"; my $diskcache = "dist/$target/rpms/$pkg-$VERSION-$RELEASE.$arch.rpm"; - return if -f $diskcache and not $optsref->{force}; + return if -f $diskcache and not $opts{force}; my @opts; - push @opts, "--quiet" unless $optsref->{verbose}; + push @opts, "--quiet" unless $opts{verbose}; say "Building $diskcache"; - `mock -r $chroot \\ - -N \\ - @{[ join " ", @opts ]} \\ - --define "version $VERSION" \\ - --define "release $RELEASE" \\ - --define "gitinfo $GITINFO" \\ - --resultdir "dist/$target/rpms/" \\ - --rebuild dist/$target/srpms/$pkg-${VERSION}-${RELEASE}.src.rpm`; + sh(<<"EOF"); +mock -r $chroot \\ + -N \\ + @{[ join " ", @opts ]} \\ + --define "version $VERSION" \\ + --define "release $RELEASE" \\ + --define "gitinfo $GITINFO" \\ + --resultdir "dist/$target/rpms/" \\ + --rebuild dist/$target/srpms/$pkg-${VERSION}-${RELEASE}.src.rpm +EOF } sub buildall { - my ($pkg, $target, $optsref) = @_; - createmockconfig($pkg, $target, $optsref); - buildsources($pkg, $target, $optsref); - buildspkgs($pkg, $target, $optsref); - buildpkgs($pkg, $target, $optsref); + my ($pkg, $target) = @_; + createmockconfig($pkg, $target); + buildsources($pkg, $target); + buildspkgs($pkg, $target); + buildpkgs($pkg, $target); } sub configure_nginx { - my ($args) = @_; - my @targets = $args->{targets}->@*; - my $deps = $args->{deps}; + my $deps = $opts{deps}; my $conf = <<"EOF"; server { listen 8080; listen [::]:8080; EOF - for my $target (@targets) { + for my $target ($opts{targets}->@*) { my $fullpath = "$PWD/dist/$target/rpms"; $conf .= <<"EOF"; location /$target/ { @@ -214,54 +249,31 @@ sub usage { exit -1; } -sub parseopts { - my %opts = ( - targets => \@TARGETS, - packages => \@PACKAGES, - nproc => int(`nproc --all`), - force => 0, - verbose => 0, - deps => "$PWD/../xcat-dep/", - ); - GetOptions( - "target=s@" => \$opts{targets}, - "package=s@" => \$opts{packages}, - "nproc=i" => \$opts{nproc}, - "verbose" => \$opts{verbose}, - "force" => \$opts{force}, - "deps=s" => \$opts{deps}, - ) or usage(); - - return \%opts; - -}; - sub main { - my $opts = parseopts(); - my @rpms = product($opts->{packages}, $opts->{targets}); - my $pm = Parallel::ForkManager->new($opts->{nproc}); + my @rpms = product($opts{packages}, $opts{targets}); + my $pm = Parallel::ForkManager->new($opts{nproc}); for my $pair (@rpms) { my ($pkg, $target) = $pair->@*; $pm->start and next; - buildall($pkg, $target, $opts); + buildall($pkg, $target); $pm->finish; } $pm->wait_all_children; - for my $target ($opts->{targets}->@*) { + for my $target ($opts{targets}->@*) { $pm->start and next; - update_repo($target, $opts); + update_repo($target); $pm->finish; } $pm->wait_all_children; - configure_nginx($opts); + configure_nginx(); `systemctl restart nginx`; } diff --git a/test/Containerfile b/test/Containerfile new file mode 100644 index 000000000..21e88fe3d --- /dev/null +++ b/test/Containerfile @@ -0,0 +1,10 @@ +ARG RELEASEVER=9 +FROM quay.io/rockylinux/rockylinux:${RELEASEVER}-ubi-init + +RUN dnf install -y epel-release +RUN dnf config-manager --set-enabled epel crb +RUN dnf makecache +RUN dnf install -y hostname perl perl-core perl-generators iproute net-tools bind-utils openssh-clients python3 perl-Net-IP perl-Sys-Syslog wget perl-IO perl-File-Copy perl-File-Basename perl-File-Slurper perl-Parallel-ForkManager perl-JSON perl-Data-Dumper +RUN dnf clean all + +WORKDIR /workspace diff --git a/test/scripts/setuptesthost.pl b/test/scripts/setuptesthost.pl new file mode 100755 index 000000000..88bd88675 --- /dev/null +++ b/test/scripts/setuptesthost.pl @@ -0,0 +1,105 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use feature 'say'; +use Getopt::Long qw(GetOptions); +use Cwd qw(cwd); + +my $PWD = cwd(); + +sub sh { + my ($cmd) = @_; + say "Running: $cmd"; + open my $fh, "-|", $cmd or die "cannot run $cmd: $!"; + + while (my $line = <$fh>) { + print $line; + } + close $fh; + return $? >> 8; +} + +sub usage { + say STDERR "Usage: $0 [--setupcontainer --target rhel+epel-9-x86_64]"; + exit -1; +} + +sub parseopts { + my %opts = ( + target => "", + setupcontainer => 0, + force => 0, + ); + + GetOptions( + "target=s" => \$opts{target}, + "setupcontainer" => \$opts{setupcontainer}, + "force" => \$opts{force}, + ) or usage(); + + usage() + if $opts{setupcontainer} + and not $opts{target}; + + + return \%opts; +}; + +sub containerexists { + my ($name) = @_; + return sh("podman container exists $name") == 0; +} + +sub imageexists { + my ($imagename) = @_; + return sh("podman image exists $imagename") == 0; +} + +sub cleanupcontainerandimage { + my ($container, $image) = @_; + return sh(<<"EOF"); +podman kill $container +podman rm -f $container +podman rmi -f $image +EOF +} + +sub setupcontainer { + my ($opts) = @_; + my $target = $opts->{target}; + + + my $releasever = int((split /-/, $target, 3)[1]); + my $name = "xcattest-el$releasever"; + + cleanupcontainerandimage($name, "$name-image") + if $opts->{force}; + + sh("podman build -t $name-image --build-arg RELEASEVER=$releasever test/") + unless imageexists("$name-image"); + my $script = <<"EOF"; +podman create --name $name \\ + --privileged \\ + --systemd=true \\ + --cap-add=ALL \\ + -v "$PWD/test/scripts:/workspace/scripts:Z" \\ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \\ + --tmpfs /tmp \\ + --tmpfs /run \\ + $name-image +podman start $name +EOF + sh($script) unless containerexists($name); +} + +sub main { + my $opts = parseopts(); + + return setupcontainer($opts) if $opts->{setupcontainer}; + + + usage(); +}; + +main(); diff --git a/test/scripts/testxcat.pl b/test/scripts/testxcat.pl new file mode 100755 index 000000000..e69f43201 --- /dev/null +++ b/test/scripts/testxcat.pl @@ -0,0 +1,112 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use feature 'say'; +use Data::Dumper qw(Dumper); +use Getopt::Long qw(GetOptions); +use File::Slurper qw(read_text write_text); + +my %opts = ( + releasever => int(`rpm --eval '%{rhel}'`), + verbose => 0, + setuprepos => 0, + installxcat => 0, + uninstallxcat => 0, + reinstallxcat => 0, + validatexcat => 0, + quiet => 0, + all => 0, +); + +GetOptions( + 'releasever=i' => \$opts{releasever}, + verbose => \$opts{verbose}, + quiet => \$opts{quiet}, + quiet => \$opts{quiet}, + setuprepos => \$opts{setuprepos}, + installxcat => \$opts{installxcat}, + uninstallxcat => \$opts{uninstallxcat}, + reinstallxcat => \$opts{reinstallxcat}, + validatexcat => \$opts{validatexcat}, + all => \$opts{all}, +) or usage(); + +sub sh { + my ($cmd) = @_; + say "Running: $cmd" + unless $opts{quiet}; + open my $fh, "-|", "bash -lc '$cmd'" or die "cannot run $cmd: $!"; + + while (my $line = <$fh>) { + print $line + unless $opts{quiet}; + } + close $fh; + return $? >> 8; +} + +sub usage { + say STDERR "usage $0: [--releasever=9] [--verbose] [--quiet] {--setuprepos|--isntallxcat|--uninstallxcat|--reinstallxcat|--validatexcat|--all}"; +} + +sub setuprepos { + say "Setting up repositories" + unless $opts{quiet}; + my $releasever = $opts{releasever}; + my $content = <<"EOF"; +[xcat3] +name=xcat3 +baseurl=http://host.containers.internal:8080/rhel+epel-$releasever-x86_64/ +gpgcheck=0 +enabled=1 + +[xcat3-deps] +name=xcat3-deps +baseurl=http://host.containers.internal:8080/xcat-dep/el$releasever/x86_64/ +gpgcheck=0 +enabled=1 +EOF + write_text("/etc/yum.repos.d/xcat-repos.repo", $content); + sh("dnf makecache --repo=xcat3 --repo=xcat3-deps"); +} + +sub uninstallxcat { + sh("dnf remove -y xCAT"); + sh("rm -rf /opt/xcat /etc/xcat /var/run/xcat /root/.xcat /install /tftpboot"); +} + +sub installxcat { + sh("dnf install -y xCAT"); +} + +sub validatexcat { + # Put commands to validate xcat installation here + sh("systemctl is-active xcatd") == 0 or die("xcatd not running?"); + sh("lsdef") == 0 or die("lsdef not working"); +} + +sub main { + return setuprepos() if $opts{setuprepos}; + return installxcat() if $opts{installxcat}; + return uninstallxcat() if $opts{uninstallxcat}; + return validatexcat() if $opts{validatexcat}; + + return do { + setuprepos(); + uninstallxcat() + if ($opts{reinstallxcat} or $opts{uninstallxcat}); + installxcat(); + validatexcat(); + } if $opts{all}; + + return do { + uninstallxcat(); + installxcat(); + } if $opts{reinstallxcat}; + + usage(); +} + +main(); + From f920ae4b9e02d00cee0bb770078bf62b1ba90d6f Mon Sep 17 00:00:00 2001 From: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:09:22 -0300 Subject: [PATCH 4/9] doc: Add test/README.md Signed-off-by: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> --- test/README.md | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 test/README.md diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..7489f174e --- /dev/null +++ b/test/README.md @@ -0,0 +1,96 @@ +# Testing Workflow for buildrpms.pl + +## Overview + +Requirements: + +_The following software are assumed to be instaled in the host responsible for building the RPMs and running the tests._ + +* [mock](https://rpm-software-management.github.io/mock/) for building the RPMs +* nginx for serving RPM repositories +* [podman](https://podman.io) +* This was tested on a RHEL9.7 host. + + +_Note: Before start, ensure nginx is listening 8080. (the script will not do that for you, you need to change the nginx.conf and restart it), more on this below_ + +Workflow: + +`buildrpms.pl` is the entry point for generating all xCAT RPMs required for a specific [mock](https://rpm-software-management.github.io/mock/) target (for example `rhel+epel-9-x86_64`, `rhel+epel-10-x86_64`). Use `mock --list-chroots` to see all possible postions. _But note that only `rhle+epel-{8,9,10}` are being tested now_. Each target corresponds to a mock chroot and eventually to a container image used by the test harness. The high-level flow is: + +1. Prepare the dependency repositories (`../xcat-dep`) for the target (more on this below) +2. Run `buildrpms.pl` to build or reuse RPMs for the target and export a repository. +3. Provision the testing container for the same target. +4. Execute the automated test suite inside the container. + +All scripts are idempotent. They only rebuild or reinstall what is missing. Pass `--force` to any step when you want to overwrite existing content. + +## xCAT Runtime Dependencies (`../xcat-dep`) +- The dependency repositories must live one directory above this repo, in `../xcat-dep`. +- We keep a tarball that already includes the EL 8, EL 9 and EL 10 dependency RPMs. Extract the tarball so that the directory tree looks like `../xcat-dep///`. +- Dependencies are only required to run xCAT. Building does not require xcat-dep. + +The expected structure is like this + + xcat-dep/ + ├── el10 + │   └── x86_64 + │   └── repodata + | └── *.rpm + ├── el9 + │   └── x86_64 + │   └── repodata + | └── *.rpm + └── el8 + └── x86_64 + └── repodata + └── *.rpm + + +## Building RPMs for a Target +```bash +./buildrpms.pl --target [--force] +``` + +- The script invokes [mock](https://rpm-software-management.github.io/mock/) for the requested target to build *all* RPMs listed for that target. The target name maps to the chroot definition under `/etc/mock/`. +- Before building `X.rpm`, the script checks whether the file already exists and skips it; use `--force` to rebuild the RPM even if the file is present **this is important if the source code changed since the last build!**. +- When the build finishes, the script collects the RPMs, runs `createrepo` and writes an nginx configuration snippet so the repo can be served. + +You can build a single package by specifying in the command line for example + +```bash +./buildrpms.pl --target rhel+epel-9-x86_64 --package perl-xCAT --force +``` + +All builds run in parallel, use `--nproc N` to control the number of jobs. Also note that this will use a lot of disk space, at last 50G is required since the last run. The directories that grow are `/var/lib/mock` and `/var/cache/mock`. + +### nginx on Port 8080 +- `buildrpms.pl` assumes nginx exposes the generated repository on port 8080. Manually update the main nginx configuration (for example `/etc/nginx/nginx.conf`) so that it listens on `8080` port. The scripts will create a `/etc/nginx/conf.d/xcat-repos.conf` file with all the repositories configured and restart +nginx at each run. + +## Preparing the Test Container +```bash +test/scripts/setuptesthost.pl --setupcontainer --target [--force] +``` + +- This builds a container image and creates a container named `xcattest-elX` (X is 8, 9 or 10 depending on the target). +- The container setup is idempotent: it checks for an existing container/image and skips recreation unless you provide `--force`. +- The EL `RELEASEVER` (8, 9 or 10) is deduced from the target which is expected to be a triple in the format `--` and it is used to as `--build-arg RELEASEVER=` during the container image building. + +## Running the Automated Tests +Once the container exists and nginx is serving the repo: + +```bash +podman exec -it xcattest-el10 scripts/testxcat.pl --all +``` + +- Replace `xcattest-el10` with the appropriate container name. The script configures the repository inside the container, installs xCAT, ensures `xcatd` is running, and finally runs `lsdef` to verify daemon connectivity. +- You can safely rerun the command; it will reuse the container state unless `--force` is supplied to the helper scripts. +- In this case you call combine `--force` with `--reinstallxcat` to make it remove xCAT completely and reinstalling again. + +## End-to-End Checklist +- Dependencies extracted to `../xcat-dep` for all EL versions you plan to build. +- `buildrpms.pl` completes successfully for the target and produces a repository. +- nginx listens on port 8080 and serves the generated repository. +- `test/scripts/setuptesthost.pl --setupcontainer --target ` creates or updates the `xcattest-elX` container. +- `podman exec -it xcattest-elX scripts/testxcat.pl --all` runs to completion and verifies the xCAT stack inside the container. From ff6828f2527748cc5a81e23de1eb38596b01ce83 Mon Sep 17 00:00:00 2001 From: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:55:53 -0300 Subject: [PATCH 5/9] fix: Fix verbosity bug in buildrpms.pl Signed-off-by: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> --- buildrpms.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildrpms.pl b/buildrpms.pl index 337d501a5..96f331a9f 100755 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -63,7 +63,7 @@ GetOptions( sub sh { my ($cmd) = @_; say "Running: $cmd" - if $opts{quiet}; + if $opts{verbose}; open my $fh, "-|", "bash -lc '$cmd'" or die "cannot run $cmd: $!"; while (my $line = <$fh>) { From a0261a907a0fb2af2691f6c61dbb7f627760d57b Mon Sep 17 00:00:00 2001 From: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:11:25 -0300 Subject: [PATCH 6/9] fix: Fix buildrpms.pl mock configuration bug Signed-off-by: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> --- buildrpms.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildrpms.pl b/buildrpms.pl index 96f331a9f..5ff7d2769 100755 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -106,7 +106,7 @@ sub createmockconfig { return if -f $cfgfile && ! $opts{force}; cp "/etc/mock/$target.cfg", $cfgfile; my $contents = read_text($cfgfile); - $contents =~ s/config_opts\['root'\]\s\+=.*/config_opts['root'] = \"$chroot\"/; + $contents =~ s/config_opts\['root'\]\s+=.*/config_opts['root'] = \"$chroot\"/; if ($pkg eq "perl-xCAT") { # perl-generators is required for having perl(xCAT::...) symbols # exported by the RPM From 698ed4bed624f15ebda85b68a866314e2ae533d1 Mon Sep 17 00:00:00 2001 From: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> Date: Fri, 5 Dec 2025 17:11:16 -0300 Subject: [PATCH 7/9] fix: Fix container setup for EL8 Also add --regennginxconfig to reset nginx config, and ensure nginx configuration is generated for all targets not only the ones provided in the command line. This avoid the anoying effect of overriding the repositories of previous builds. Signed-off-by: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> --- buildrpms.pl | 10 ++++++++-- test/Containerfile | 14 ++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/buildrpms.pl b/buildrpms.pl index 5ff7d2769..60932ce3c 100755 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -49,6 +49,7 @@ my %opts = ( force => 0, verbose => 0, deps => "$PWD/../xcat-dep/", + regennginxconf => 0, ); GetOptions( @@ -58,6 +59,7 @@ GetOptions( "verbose" => \$opts{verbose}, "force" => \$opts{force}, "deps=s" => \$opts{deps}, + "regennginxconf" => \$opts{regennginxconf}, ) or usage(); sub sh { @@ -209,7 +211,10 @@ server { listen 8080; listen [::]:8080; EOF - for my $target ($opts{targets}->@*) { + + # We always generate the nginx config for all + # the targets, not $opts{targets} + for my $target (@TARGETS) { my $fullpath = "$PWD/dist/$target/rpms"; $conf .= <<"EOF"; location /$target/ { @@ -231,6 +236,7 @@ EOF } EOF write_text("/etc/nginx/conf.d/xcat-repos.conf", $conf); + `systemctl restart nginx`; } sub update_repo { @@ -250,6 +256,7 @@ sub usage { } sub main { + return configure_nginx() if $opts{regennginxconf}; my @rpms = product($opts{packages}, $opts{targets}); my $pm = Parallel::ForkManager->new($opts{nproc}); @@ -275,7 +282,6 @@ sub main { configure_nginx(); - `systemctl restart nginx`; } main(); diff --git a/test/Containerfile b/test/Containerfile index 21e88fe3d..7731dbdd1 100644 --- a/test/Containerfile +++ b/test/Containerfile @@ -1,10 +1,20 @@ ARG RELEASEVER=9 FROM quay.io/rockylinux/rockylinux:${RELEASEVER}-ubi-init +ARG RELEASEVER RUN dnf install -y epel-release -RUN dnf config-manager --set-enabled epel crb + +RUN echo "RELEASEVER=${RELEASEVER}" +RUN if [[ "${RELEASEVER}" == "8" ]]; then \ + dnf config-manager --set-enabled powertools epel; \ + elif [[ "${RELEASEVER}" =~ /(9|10)/ ]]; then \ + dnf config-manager --set-enabled crb epel; \ + else \ + echo "Invalid RELEASEVER=${RELEASEVER}"; \ + exit -1; \ + fi RUN dnf makecache -RUN dnf install -y hostname perl perl-core perl-generators iproute net-tools bind-utils openssh-clients python3 perl-Net-IP perl-Sys-Syslog wget perl-IO perl-File-Copy perl-File-Basename perl-File-Slurper perl-Parallel-ForkManager perl-JSON perl-Data-Dumper +RUN dnf install -y hostname perl perl-core RUN dnf clean all WORKDIR /workspace From 01f508da09f4b6deceadbb48934c0df9f2aa7335 Mon Sep 17 00:00:00 2001 From: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> Date: Fri, 5 Dec 2025 17:59:32 -0300 Subject: [PATCH 8/9] fix: Fix the help messages in all the scripts Signed-off-by: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> --- buildrpms.pl | 44 +++++++++++++++++++------ test/README.md | 2 +- test/scripts/setuptesthost.pl | 12 +++---- test/scripts/testxcat.pl | 61 ++++++++++++++++++----------------- 4 files changed, 74 insertions(+), 45 deletions(-) diff --git a/buildrpms.pl b/buildrpms.pl index 60932ce3c..8cf7b5361 100755 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -48,8 +48,10 @@ my %opts = ( nproc => int(`nproc --all`), force => 0, verbose => 0, - deps => "$PWD/../xcat-dep/", - regennginxconf => 0, + xcat_dep_path => "$PWD/../xcat-dep/", + configure_nginx => 0, + help => 0, + nginx_port => 8080, ); GetOptions( @@ -58,8 +60,10 @@ GetOptions( "nproc=i" => \$opts{nproc}, "verbose" => \$opts{verbose}, "force" => \$opts{force}, - "deps=s" => \$opts{deps}, - "regennginxconf" => \$opts{regennginxconf}, + "xcat_dep_path=s" => \$opts{xcat_dep_path}, + "configure_nginx" => \$opts{configure_nginx}, + "help" => \$opts{help}, + "nginx_port" => \$opts{nginx_port}, ) or usage(); sub sh { @@ -205,11 +209,12 @@ sub buildall { } sub configure_nginx { - my $deps = $opts{deps}; + my $xcat_dep_path = $opts{xcat_dep_path}; + my $port = $opts{nginx_port}; my $conf = <<"EOF"; server { - listen 8080; - listen [::]:8080; + listen $port; + listen [::]:$port; EOF # We always generate the nginx config for all @@ -228,7 +233,7 @@ EOF # TODO:I need one xcat-dep for each target $conf .= <<"EOF"; location /xcat-dep/ { - alias $deps; + alias $xcat_dep_path; autoindex on; index off; allow all; @@ -250,13 +255,34 @@ sub update_repo { sub usage { my ($errmsg) = @_; say STDERR "Usage: $0 [--package=] [--target=] [--package=] [--target=] ..."; + say STDERR ""; + say STDERR " RPM builder script"; + say STDERR " .. build xCAT RPMs for these targets:"; + say STDERR map { " $_\n" } @TARGETS; + say STDERR ""; + say STDERR " Options:"; + say STDERR ""; + say STDERR " --target .................. build only these targets"; + say STDERR " --package ................. build only these packages"; + say STDERR " --force ......................... override built RPMS"; + say STDERR " --configure_nginx ............... update nginx configuration"; + say STDERR " --nginx_port=8080 ............... change the nginx port in"; + say STDERR " (use with --configure_nginx)"; + say STDERR " --nproc ..................... run up to N jobs in parallel"; + say STDERR " --xcat_dep_path=../xcat-dep ..... path to xcat-dep repositories"; + say STDERR ""; + say STDERR " If no --target or --package is given all combinations are built"; + say STDERR ""; + say STDERR " See test/README.md for more information"; say STDERR $errmsg if $errmsg; exit -1; } sub main { - return configure_nginx() if $opts{regennginxconf}; + return usage() if $opts{help}; + return configure_nginx() if $opts{configure_nginx}; + my @rpms = product($opts{packages}, $opts{targets}); my $pm = Parallel::ForkManager->new($opts{nproc}); diff --git a/test/README.md b/test/README.md index 7489f174e..cf397d9c2 100644 --- a/test/README.md +++ b/test/README.md @@ -86,7 +86,7 @@ podman exec -it xcattest-el10 scripts/testxcat.pl --all - Replace `xcattest-el10` with the appropriate container name. The script configures the repository inside the container, installs xCAT, ensures `xcatd` is running, and finally runs `lsdef` to verify daemon connectivity. - You can safely rerun the command; it will reuse the container state unless `--force` is supplied to the helper scripts. -- In this case you call combine `--force` with `--reinstallxcat` to make it remove xCAT completely and reinstalling again. +- In this case you call combine `--force` with `--reinstall` to make it remove xCAT completely and reinstalling again. ## End-to-End Checklist - Dependencies extracted to `../xcat-dep` for all EL versions you plan to build. diff --git a/test/scripts/setuptesthost.pl b/test/scripts/setuptesthost.pl index 88bd88675..905503c46 100755 --- a/test/scripts/setuptesthost.pl +++ b/test/scripts/setuptesthost.pl @@ -21,25 +21,25 @@ sub sh { } sub usage { - say STDERR "Usage: $0 [--setupcontainer --target rhel+epel-9-x86_64]"; + say STDERR "Usage: $0 --setup_container --target [--force]"; exit -1; } sub parseopts { my %opts = ( target => "", - setupcontainer => 0, + setup_container => 0, force => 0, ); GetOptions( "target=s" => \$opts{target}, - "setupcontainer" => \$opts{setupcontainer}, + "setup_container" => \$opts{setup_container}, "force" => \$opts{force}, ) or usage(); usage() - if $opts{setupcontainer} + if $opts{setup_container} and not $opts{target}; @@ -65,7 +65,7 @@ podman rmi -f $image EOF } -sub setupcontainer { +sub setup_container { my ($opts) = @_; my $target = $opts->{target}; @@ -96,7 +96,7 @@ EOF sub main { my $opts = parseopts(); - return setupcontainer($opts) if $opts->{setupcontainer}; + return setup_container($opts) if $opts->{setup_container}; usage(); diff --git a/test/scripts/testxcat.pl b/test/scripts/testxcat.pl index e69f43201..a135c2021 100755 --- a/test/scripts/testxcat.pl +++ b/test/scripts/testxcat.pl @@ -10,13 +10,14 @@ use File::Slurper qw(read_text write_text); my %opts = ( releasever => int(`rpm --eval '%{rhel}'`), verbose => 0, - setuprepos => 0, - installxcat => 0, - uninstallxcat => 0, - reinstallxcat => 0, - validatexcat => 0, + setup_repos => 0, + install => 0, + uninstall => 0, + reinstall => 0, + validate => 0, quiet => 0, all => 0, + nginx_port => 8080, ); GetOptions( @@ -24,12 +25,13 @@ GetOptions( verbose => \$opts{verbose}, quiet => \$opts{quiet}, quiet => \$opts{quiet}, - setuprepos => \$opts{setuprepos}, - installxcat => \$opts{installxcat}, - uninstallxcat => \$opts{uninstallxcat}, - reinstallxcat => \$opts{reinstallxcat}, - validatexcat => \$opts{validatexcat}, + setup_repos => \$opts{setup_repos}, + install => \$opts{install}, + uninstall => \$opts{uninstall}, + reinstall => \$opts{reinstall}, + validate => \$opts{validate}, all => \$opts{all}, + nginx_port => \$opts{nginx_port}, ) or usage(); sub sh { @@ -47,23 +49,24 @@ sub sh { } sub usage { - say STDERR "usage $0: [--releasever=9] [--verbose] [--quiet] {--setuprepos|--isntallxcat|--uninstallxcat|--reinstallxcat|--validatexcat|--all}"; + say STDERR "usage $0: [--releasever=9] [--verbose] [--quiet] {--setup_repos|--install|--uninstall|--reinstall|--validate|--all} [--nginx-port=8080]"; } -sub setuprepos { +sub setup_repos { say "Setting up repositories" unless $opts{quiet}; my $releasever = $opts{releasever}; + my $port = $opts{nginx_port}; my $content = <<"EOF"; [xcat3] name=xcat3 -baseurl=http://host.containers.internal:8080/rhel+epel-$releasever-x86_64/ +baseurl=http://host.containers.internal:$port/rhel+epel-$releasever-x86_64/ gpgcheck=0 enabled=1 [xcat3-deps] name=xcat3-deps -baseurl=http://host.containers.internal:8080/xcat-dep/el$releasever/x86_64/ +baseurl=http://host.containers.internal:$port/xcat-dep/el$releasever/x86_64/ gpgcheck=0 enabled=1 EOF @@ -71,39 +74,39 @@ EOF sh("dnf makecache --repo=xcat3 --repo=xcat3-deps"); } -sub uninstallxcat { +sub uninstall { sh("dnf remove -y xCAT"); sh("rm -rf /opt/xcat /etc/xcat /var/run/xcat /root/.xcat /install /tftpboot"); } -sub installxcat { +sub install { sh("dnf install -y xCAT"); } -sub validatexcat { +sub validate { # Put commands to validate xcat installation here sh("systemctl is-active xcatd") == 0 or die("xcatd not running?"); sh("lsdef") == 0 or die("lsdef not working"); } sub main { - return setuprepos() if $opts{setuprepos}; - return installxcat() if $opts{installxcat}; - return uninstallxcat() if $opts{uninstallxcat}; - return validatexcat() if $opts{validatexcat}; + return setup_repos() if $opts{setup_repos}; + return install() if $opts{install}; + return uninstall() if $opts{uninstall}; + return validate() if $opts{validate}; return do { - setuprepos(); - uninstallxcat() - if ($opts{reinstallxcat} or $opts{uninstallxcat}); - installxcat(); - validatexcat(); + setup_repos(); + uninstall() + if ($opts{reinstall} or $opts{uninstall}); + install(); + validate(); } if $opts{all}; return do { - uninstallxcat(); - installxcat(); - } if $opts{reinstallxcat}; + uninstall(); + install(); + } if $opts{reinstall}; usage(); } From 86250a4165bb00cf2873998c5f4cd66da96148b3 Mon Sep 17 00:00:00 2001 From: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> Date: Mon, 8 Dec 2025 11:12:49 -0300 Subject: [PATCH 9/9] doc: Update build/test scripts options in test/README.md Signed-off-by: Daniel Hilst Selli <392820+dhilst@users.noreply.github.com> --- test/README.md | 24 +++++++++++++++--------- test/scripts/testxcat.pl | 1 - 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/test/README.md b/test/README.md index cf397d9c2..a61217b06 100644 --- a/test/README.md +++ b/test/README.md @@ -49,20 +49,26 @@ The expected structure is like this ## Building RPMs for a Target ```bash -./buildrpms.pl --target [--force] +./buildrpms.pl [--target ] [--package ] [--force] [--nproc N] [--xcat_dep_path /path/to/xcat-dep] [--nginx_port 8080] ``` -- The script invokes [mock](https://rpm-software-management.github.io/mock/) for the requested target to build *all* RPMs listed for that target. The target name maps to the chroot definition under `/etc/mock/`. -- Before building `X.rpm`, the script checks whether the file already exists and skips it; use `--force` to rebuild the RPM even if the file is present **this is important if the source code changed since the last build!**. -- When the build finishes, the script collects the RPMs, runs `createrepo` and writes an nginx configuration snippet so the repo can be served. +- `--target` may be repeated; by default all supported targets (`rhel+epel-{8,9,10}-x86_64`) are built. +- `--package` narrows the build to specific RPM names; the default list contains every xCAT component. +- `--force` rebuilds even if the corresponding SRPM/RPM already exists under `dist//`. +- `--nproc N` controls the amount of parallel mock jobs (defaults to `nproc --all`). +- `--xcat_dep_path` tells the nginx helper where the dependency repos live (defaults to `../xcat-dep`). +- `--nginx_port` changes the port used when generating `/etc/nginx/conf.d/xcat-repos.conf`. +- Passing `--configure_nginx` alone regenerates the nginx configuration without building. -You can build a single package by specifying in the command line for example +Example – rebuild only `perl-xCAT` for EL9 and overwrite previous artifacts: ```bash ./buildrpms.pl --target rhel+epel-9-x86_64 --package perl-xCAT --force ``` -All builds run in parallel, use `--nproc N` to control the number of jobs. Also note that this will use a lot of disk space, at last 50G is required since the last run. The directories that grow are `/var/lib/mock` and `/var/cache/mock`. +Under the hood the script invokes [mock](https://rpm-software-management.github.io/mock/) to build SRPMs and RPMs, skips work when files already exist (unless `--force` is used), runs `createrepo dist//rpms`, and finally rewrites the nginx configuration so the repo is served automatically. + +All builds run in parallel according to `--nproc`. Be aware that mock consumes significant disk space (plan for at least ~50 GB between `/var/lib/mock` and `/var/cache/mock`). ### nginx on Port 8080 - `buildrpms.pl` assumes nginx exposes the generated repository on port 8080. Manually update the main nginx configuration (for example `/etc/nginx/nginx.conf`) so that it listens on `8080` port. The scripts will create a `/etc/nginx/conf.d/xcat-repos.conf` file with all the repositories configured and restart @@ -70,7 +76,7 @@ nginx at each run. ## Preparing the Test Container ```bash -test/scripts/setuptesthost.pl --setupcontainer --target [--force] +test/scripts/setuptesthost.pl --setup_container --target [--force] ``` - This builds a container image and creates a container named `xcattest-elX` (X is 8, 9 or 10 depending on the target). @@ -85,8 +91,8 @@ podman exec -it xcattest-el10 scripts/testxcat.pl --all ``` - Replace `xcattest-el10` with the appropriate container name. The script configures the repository inside the container, installs xCAT, ensures `xcatd` is running, and finally runs `lsdef` to verify daemon connectivity. -- You can safely rerun the command; it will reuse the container state unless `--force` is supplied to the helper scripts. -- In this case you call combine `--force` with `--reinstall` to make it remove xCAT completely and reinstalling again. +- The tester exposes additional knobs: use `--setup_repos`, `--install`, `--uninstall`, `--reinstall`, `--validate`, or `--all` (default in the example) plus `--nginx_port` if your repo is served on a different port. Combine these as needed to drive just one phase or the entire install/validate pass. +- You can safely rerun the command; it will reuse the container state unless `--force` is supplied to the helper scripts, and `--reinstall` forces a clean uninstall/install cycle inside the container. ## End-to-End Checklist - Dependencies extracted to `../xcat-dep` for all EL versions you plan to build. diff --git a/test/scripts/testxcat.pl b/test/scripts/testxcat.pl index a135c2021..c206d608d 100755 --- a/test/scripts/testxcat.pl +++ b/test/scripts/testxcat.pl @@ -24,7 +24,6 @@ GetOptions( 'releasever=i' => \$opts{releasever}, verbose => \$opts{verbose}, quiet => \$opts{quiet}, - quiet => \$opts{quiet}, setup_repos => \$opts{setup_repos}, install => \$opts{install}, uninstall => \$opts{uninstall},