2
0
mirror of https://github.com/xcat2/xcat-core.git synced 2026-11-22 02:12:16 +00:00

Merge pull request #7979 from VersatusHPC/refactor/debian-profile-lookup

refactor(netboot): share profile asset selection
This commit is contained in:
Daniel Hilst
2026-10-29 14:58:18 -03:00
committed by GitHub
15 changed files with 338 additions and 43 deletions
+1
View File
@@ -62,6 +62,7 @@ jobs:
xCAT-test/native/ubuntu_genimage_debootstrap_arch.t
xCAT-test/native/ubuntu_genimage_apt_mirror.t
xCAT-test/native/ubuntu_genimage_resolver_libs.t
xCAT-test/native/debian_genimage_profile_assets.t
- name: Exercise the liteimg ramdisk archive
if: ${{ !cancelled() }}
run: prove xCAT-test/native/liteimg_ramdisk_archive.t
@@ -55,6 +55,9 @@ If this does not work or the management node is not the same distro level, copy
/opt/xcat/share/xcat/netboot/<os> to a system that is the proper architecture, and mount /install from
the management node to that system. Then change directory to /opt/xcat/share/xcat/netboot/<os> and run ./genimage.
When copying Debian scripts from a source checkout, use \ **cp -RL**\ to include
the shared profile helper. Installed packages include the helper in the Debian directory.
**********
Parameters
+3
View File
@@ -38,6 +38,9 @@ If this does not work or the management node is not the same distro level, copy
/opt/xcat/share/xcat/netboot/<os> to a system that is the proper architecture, and mount /install from
the management node to that system. Then change directory to /opt/xcat/share/xcat/netboot/<os> and run ./genimage.
When copying Debian scripts from a source checkout, use B<cp -RL> to include
the shared profile helper. Installed packages include the helper in the Debian directory.
=head1 Parameters
+3
View File
@@ -44,6 +44,9 @@ binary-arch:
dh_installdocs
# dh_installexamples
dh_install -X".svn"
# Debian scripts must remain self-contained when copied from an installed package.
rm -f $(rootdir)/share/xcat/netboot/debian/xCAT/ProfileUtils.pm
install -m 0644 share/xcat/netboot/imgutils/xCAT/ProfileUtils.pm $(rootdir)/share/xcat/netboot/debian/xCAT/ProfileUtils.pm
chmod 755 $(rootdir)/bin/*
chmod 755 $(rootdir)/sbin/*
chmod 644 $(rootdir)/share/xcat/ca/*
+5 -15
View File
@@ -11,6 +11,9 @@ use File::Copy;
use File::Find;
use Getopt::Long;
use Cwd qw(realpath);
use FindBin;
use lib "$FindBin::Bin";
use xCAT::ProfileUtils;
#use strict;
Getopt::Long::Configure("bundling");
@@ -1302,21 +1305,8 @@ sub get_profile_def_filename {
my $ext = shift;
my $dotpos = rindex($osver, ".");
my $osbase = substr($osver, 0, $dotpos);
if (-r "$base/$profile.$osver.$arch.$ext") {
return "$base/$profile.$osver.$arch.$ext";
} elsif (-r "$base/$profile.$osbase.$arch.$ext") {
return "$base/$profile.$osbase.$arch.$ext";
} elsif (-r "$base/$profile.$arch.$ext") {
return "$base/$profile.$arch.$ext";
} elsif (-r "$base/$profile.$osver.$ext") {
return "$base/$profile.$osver.$ext";
} elsif (-r "$base/$profile.$osbase.$ext") {
return "$base/$profile.$osbase.$ext";
} elsif (-r "$base/$profile.$ext") {
return "$base/$profile.$ext";
}
return "";
return xCAT::ProfileUtils::find_profile_asset(
$base, $profile, $arch, $ext, $osver, $osbase);
}
@@ -0,0 +1 @@
../../imgutils/xCAT/ProfileUtils.pm
@@ -11,6 +11,8 @@ use File::Copy qw(move);
use File::Find;
use File::Path;
use Cwd qw(realpath);
use lib dirname(__FILE__);
use xCAT::ProfileUtils;
use xCAT::SvrUtils;
sub el_major_version {
@@ -241,33 +243,9 @@ sub get_profile_def_filename {
if ($osver =~ /^leap15/) {
$fallbackos = "sle15";
}
if (-r "$base/$profile.$osver.$arch.$ext") {
return "$base/$profile.$osver.$arch.$ext";
}
foreach my $osbase (@osbase) {
if (-r "$base/$profile.$osbase.$arch.$ext") {
return "$base/$profile.$osbase.$arch.$ext";
}
}
if ($fallbackos && -r "$base/$profile.$fallbackos.$arch.$ext") {
return "$base/$profile.$fallbackos.$arch.$ext";
} elsif (-r "$base/$profile.$arch.$ext") {
return "$base/$profile.$arch.$ext";
} elsif (-r "$base/$profile.$osver.$ext") {
return "$base/$profile.$osver.$ext";
}
foreach my $osbase (@osbase) {
if (-r "$base/$profile.$osbase.$ext") {
return "$base/$profile.$osbase.$ext";
}
}
if ($fallbackos && -r "$base/$profile.$fallbackos.$ext") {
return "$base/$profile.$fallbackos.$ext";
} elsif (-r "$base/$profile.$ext") {
return "$base/$profile.$ext";
}
return "";
return xCAT::ProfileUtils::find_profile_asset(
$base, $profile, $arch, $ext, $osver, @osbase,
($fallbackos ? ($fallbackos) : ()));
}
sub include_file
@@ -0,0 +1,17 @@
package xCAT::ProfileUtils;
use strict;
use warnings;
sub find_profile_asset {
my ($base, $profile, $arch, $ext, @versions) = @_;
my @suffixes = ((map { "$_.$arch" } @versions), $arch, @versions, '');
foreach my $suffix (@suffixes) {
my $file = "$base/$profile" . (length($suffix) ? ".$suffix" : '') . ".$ext";
return $file if -r $file;
}
return '';
}
1;
+5
View File
@@ -183,6 +183,11 @@ cp -hpR share/xcat/install/* $RPM_BUILD_ROOT/%{prefix}/share/xcat/install/
cp -hpR share/xcat/netboot/* $RPM_BUILD_ROOT/%{prefix}/share/xcat/netboot/
%endif
# Debian scripts must remain self-contained when copied from an installed package.
rm -f $RPM_BUILD_ROOT/%{prefix}/share/xcat/netboot/debian/xCAT/ProfileUtils.pm
cp share/xcat/netboot/imgutils/xCAT/ProfileUtils.pm $RPM_BUILD_ROOT/%{prefix}/share/xcat/netboot/debian/xCAT/ProfileUtils.pm
chmod 644 $RPM_BUILD_ROOT/%{prefix}/share/xcat/netboot/debian/xCAT/ProfileUtils.pm $RPM_BUILD_ROOT/%{prefix}/share/xcat/netboot/imgutils/xCAT/ProfileUtils.pm
%ifos linux
# pwd
cp -d sbin/* $RPM_BUILD_ROOT/%{prefix}/sbin
+9
View File
@@ -45,6 +45,15 @@ packages, execute target-architecture binaries or prove that the generated image
## Other native tests
`debian_genimage_profile_assets.t` runs the Debian genimage command in an
isolated filesystem. It checks custom and shipped package lists,
extra-package lists and postinstall scripts. Package tools and postinstall
scripts stop with deliberate failures after recording the selected input.
Only the Debian script directory is copied, with no installed xCAT libraries.
Source-checkout copies follow symlinks. An optional directory argument checks
a package payload without following symlinks.
It does not install packages or build an initrd.
The pull-request workflow runs these packaging cases explicitly as an ordinary
user:
@@ -0,0 +1,113 @@
use strict;
use warnings;
use FindBin;
use lib "$FindBin::Bin/../lib";
use File::Path qw(make_path);
use Test::More;
use XCAT::Test::File qw(repo_path);
use XCAT::Test::ImageSandbox;
use XCAT::Test::Process qw(run_command);
plan skip_all => 'Linux namespaces are required' unless $^O eq 'linux';
die 'Run image tests as an unprivileged user' unless $>;
my $payload = shift @ARGV;
my $source = $payload // repo_path('xCAT-server/share/xcat/netboot/debian');
sub standalone_image {
my $box = XCAT::Test::ImageSandbox->new();
make_path("$box->{root}/work/share/xcat/netboot");
my @copy = ('-R', '--no-preserve=mode,ownership');
push @copy, '-L' unless defined $payload;
my ($rc, $output) = run_command('cp', @copy, $source, "$box->{root}/work/share/xcat/netboot/");
die "copy Debian scripts: $output" if $rc;
return $box;
}
for my $extension (qw(pkglist otherpkgs.pkglist postinstall)) {
for my $custom (0, 1) {
subtest "$extension " . ($custom ? 'custom' : 'shipped') => sub {
my $box = standalone_image();
my $shipped = 'work/share/xcat/netboot/debian';
my $customdir = 'install/custom/netboot/debian';
$box->write('install/debian12.5/x86_64/Packages.gz', 'index');
make_path("$box->{root}/install/debian12.5/x86_64/dists/bookworm");
$box->write('work/image/rootimg/lib/modules/fixture/modules.dep', '');
my $file = "profile.debian12.5.x86_64.$extension";
my $selected = $custom ? "/$customdir/profile.$extension"
: "share/xcat/netboot/debian/$file";
my $content = $extension eq 'postinstall'
? "#!/bin/sh\nprintf '%s\\n' \"\$0\" \"\$@\" > /work/selected\nexit 23\n"
: "selected-package\n";
my $target = $box->write("$shipped/$file", $custom ? "wrong-package\n" : $content);
chmod(0755, $target) or die "chmod: $!";
if ($custom) {
$target = $box->write("$customdir/profile.$extension", $content);
chmod(0755, $target) or die "chmod: $!";
}
$box->command('debootstrap', $extension eq 'pkglist' ? <<'SH' : 'exit 0');
printf '%s\n' "$@" > /work/selected
exit 23
SH
$box->command('yum', <<'SH');
printf '%s\n' "$@" > /work/selected
exit 23
SH
$box->write("$shipped/profile.pkglist", '') if $extension eq 'otherpkgs.pkglist';
my @args = ('-o', 'debian12.5', '-a', 'x86_64', '-p', 'profile',
'-i', 'eth0', '-n', 'fixture', '-k', 'fixture',
'--rootimgdir', '/work/image', '--noupdate');
push @args, '--onlyinitrd' if $extension eq 'postinstall';
my ($rc, $output, $error) = $box->run('env', 'PERL5LIB=', 'XCATROOT=/nonexistent',
'perl', 'share/xcat/netboot/debian/genimage', @args);
is($rc, 1, 'caller stops at the selected external command failure') or diag($output, $error);
my $observed = -f "$box->{root}/work/selected" ? $box->read('work/selected') : '';
if ($extension eq 'postinstall') {
is($observed, "$selected\n/work/image/rootimg\ndebian12.5\nx86_64\nprofile\n",
'postinstall gets the unchanged path spelling and arguments');
like($output, qr/postinstall script \Q$selected\E failed/, 'caller reports the selected postinstall path');
} elsif ($extension eq 'pkglist') {
like($observed, qr/^--include=selected-package$/m, 'debootstrap receives the selected package');
unlike($observed, qr/wrong-package/, 'custom directory takes precedence over a shipped exact match');
like($output, qr/apt invocation failed/, 'caller reaches the bootstrap failure');
} else {
like($observed, qr/^selected-package$/m, 'extra-package installation consumes the selected list');
unlike($observed, qr/wrong-package/, 'custom directory takes precedence over a shipped exact match');
like($output, qr/yum invocation failed/, 'caller reaches the extra-package failure');
}
};
}
}
subtest 'missing package list' => sub {
my $box = standalone_image();
$box->write('install/debian12.5/x86_64/Packages.gz', 'index');
make_path("$box->{root}/install/debian12.5/x86_64/dists/bookworm");
my ($rc, $output, $error) = $box->run('env', 'PERL5LIB=', 'XCATROOT=/nonexistent',
'perl', 'share/xcat/netboot/debian/genimage',
'-o', 'debian12.5', '-a', 'x86_64', '-p', 'missing', '-n', 'fixture', '-i', 'eth0',
'--rootimgdir', '/work/image');
is($rc, 1, 'missing package list stops image creation') or diag($output, $error);
like($output, qr/Unable to find package list for missing!/, 'original missing-list diagnostic remains');
};
subtest 'standalone explicit image definition' => sub {
my $box = standalone_image();
my $postinstall = $box->write('work/postinstall', "#!/bin/sh\necho explicit > /work/selected\nexit 23\n");
chmod(0755, $postinstall) or die "chmod: $!";
$box->write('work/image/rootimg/lib/modules/fixture/modules.dep', '');
my ($rc, $output, $error) = $box->run('env', 'PERL5LIB=', 'XCATROOT=/nonexistent',
'perl', 'share/xcat/netboot/debian/genimage', '--onlyinitrd',
'-o', 'debian12.5', '-a', 'x86_64', '-p', 'profile', '-n', 'fixture', '-i', 'eth0',
'-k', 'fixture', '--postinstall', '/work/postinstall', '--rootimgdir', '/work/image', 'explicit-image');
is($rc, 1, 'standalone command reaches the explicit postinstall failure') or diag($output, $error);
is(-f "$box->{root}/work/selected" ? $box->read('work/selected') : '', "explicit\n",
'an explicit image definition runs without installed xCAT modules');
like($output, qr/postinstall script \/work\/postinstall failed/, 'failure comes from the explicit postinstall');
};
done_testing();
+22 -1
View File
@@ -48,6 +48,21 @@ my ($status, $output) = run_command($^X,
repo_path('xCAT-test/unit/profile_customization.t'), "$root/opt/xcat");
is($status, 0, 'packaged profiles resolve and remain independently customizable') or diag($output);
my $netboot = "$root/opt/xcat/share/xcat/netboot";
for my $directory (qw(debian imgutils)) {
my $helper = "$netboot/$directory/xCAT/ProfileUtils.pm";
ok(-f $helper && !-l $helper, "$directory ships a regular profile helper") or next;
is(sprintf('%04o', S_IMODE((stat($helper))[2])), '0644', "$directory helper is readable without write access for others");
}
for my $case (
['unit/imgutils_standalone.t', 'imgutils'],
['native/debian_genimage_profile_assets.t', 'debian'],
) {
my ($test, $directory) = @$case;
my ($rc, $result) = run_command($^X, repo_path("xCAT-test/$test"), "$netboot/$directory");
is($rc, 0, "packaged $directory works when copied without sibling directories") or diag($result);
}
if (!-f '/etc/debian_version') {
local $ENV{fsm} = 1;
my $reduced = tempdir(CLEANUP => 1);
@@ -65,7 +80,7 @@ if (!-f '/etc/debian_version') {
my $data = 'install/rh/compute.rhels10.riscv64.pkglist';
my $postinstall = 'netboot/rocky/compute.rocky10.riscv64.postinstall';
chmod(oct('0666'), "$profiles/$data") == 1 or die "chmod: $!";
for my $mask ('0022', '0002') {
for my $mask ('0022', '0002', '0077') {
subtest "Debian writable sources with umask $mask" => sub {
my $previous = umask oct($mask);
my $deb = XCAT::Test::Lifecycle::build_deb('xCAT-server', $fixture);
@@ -80,6 +95,12 @@ if (!-f '/etc/debian_version') {
my $payload = tempdir(CLEANUP => 1);
XCAT::Test::Lifecycle::checked('dpkg-deb', '-x', $deb, $payload);
check_assets($payload, '', @assets);
for my $directory (qw(debian imgutils)) {
my $helper = "$payload/opt/xcat/share/xcat/netboot/$directory/xCAT/ProfileUtils.pm";
ok(-f $helper && !-l $helper, "$directory ships a regular helper with umask $mask");
is(sprintf('%04o', S_IMODE((stat($helper))[2])), '0644',
"$directory helper retains readable permissions with umask $mask");
}
for my $path (qw(install install/rh netboot netboot/rocky netboot/ubuntu/genimage)) {
is(sprintf('%04o', S_IMODE((stat("$payload/opt/xcat/share/xcat/$path"))[2])),
'0755', "$path remains executable without group or other write access");
@@ -0,0 +1,20 @@
use strict;
use warnings;
use JSON::PP qw(encode_json);
BEGIN {
*CORE::GLOBAL::exit = sub { die "genimage exit $_[0]\n" };
}
my ($script, $osver, $profile, $arch, $base, $ext) = @ARGV;
@ARGV = ('-o', $osver, '-p', $profile, '-a', $arch);
my $startup;
{
local $0 = $script;
local *STDOUT;
open STDOUT, '>', \$startup or die "capture stdout: $!";
do $script;
die "genimage did not stop at argument validation: $@ $!\n"
unless $@ eq "genimage exit 1\n";
}
print encode_json([main::get_profile_def_filename($base, $ext)]);
+36
View File
@@ -0,0 +1,36 @@
use strict;
use warnings;
use FindBin;
use lib "$FindBin::Bin/../lib";
use Cwd qw(realpath);
use File::Path qw(make_path);
use File::Slurper qw(write_text);
use File::Temp qw(tempdir);
use Test::More;
use XCAT::Test::File qw(repo_path);
use XCAT::Test::Process qw(run_command);
my $source = shift @ARGV // repo_path('xCAT-server/share/xcat/netboot/imgutils');
my $root = tempdir(CLEANUP => 1);
my ($status, $output) = run_command('cp', '-R', $source, "$root/imgutils");
die "copy imgutils: $output" if $status;
make_path("$root/assets");
my $assets = realpath("$root/assets");
local $ENV{PERL5LIB} = '';
local $ENV{XCATROOT} = '/nonexistent';
for my $extension (qw(pkglist otherpkgs.pkglist postinstall)) {
my $selected = "$assets/compute.debian12.x86_64.$extension";
write_text($selected, "selected\n");
write_text("$root/assets/compute.$extension", "fallback\n");
my ($rc, $result) = run_command($^X,
'-I' . repo_path('perl-xCAT'), '-I' . repo_path('xCAT-server/lib/perl'),
'-e', 'require shift @ARGV; print imgutils::get_profile_def_filename(@ARGV)',
"$root/imgutils/imgutils.pm",
'debian12.5', 'compute', 'x86_64', "$root/assets", $extension);
is($rc, 0, "copied imgutils loads without any sibling distro directory") or diag($result);
is($result, $selected, "copied imgutils selects the $extension version fallback");
}
done_testing();
+95
View File
@@ -0,0 +1,95 @@
use strict;
use warnings;
use FindBin;
use lib "$FindBin::Bin/../lib", "$FindBin::Bin/../../perl-xCAT",
"$FindBin::Bin/../../xCAT-server/lib/perl",
"$FindBin::Bin/../../xCAT-server/share/xcat/netboot/imgutils";
use Cwd qw(getcwd realpath);
use File::Path qw(make_path);
use File::Slurper qw(write_text);
use File::Temp qw(tempdir);
use JSON::PP qw(decode_json);
use Test::More;
use XCAT::Test::File qw(repo_path);
use XCAT::Test::Process qw(run_command);
use imgutils;
my $root = tempdir(CLEANUP => 1);
my $original_dir = getcwd();
chdir($root) or die "chdir $root: $!";
make_path('assets');
symlink('assets', 'alias') or die "symlink: $!";
sub lookup {
my ($owner, $version, $arch, $base, $extension) = @_;
return imgutils::get_profile_def_filename($version, 'compute', $arch, $base, $extension)
if $owner eq 'imgutils';
local $ENV{PERL5LIB} = '';
local $ENV{XCATROOT} = '/nonexistent';
my ($rc, $output) = run_command($^X,
repo_path('xCAT-test/unit/fixtures/debian_profile_lookup.pl'),
repo_path('xCAT-server/share/xcat/netboot/debian/genimage'),
$version, 'compute', $arch, $base, $extension);
die "Debian lookup failed ($rc): $output" if $rc;
return decode_json($output)->[0];
}
for my $owner (qw(debian imgutils)) {
for my $arch (qw(x86_64 ppc64le s390x)) {
for my $extension (qw(pkglist otherpkgs.pkglist postinstall)) {
subtest "$owner $arch $extension priority" => sub {
my @names = (
"compute.debian12.5.$arch.$extension",
"compute.debian12.$arch.$extension",
"compute.$arch.$extension",
"compute.debian12.5.$extension",
"compute.debian12.$extension",
"compute.$extension",
);
write_text("assets/$_", "fixture\n") for @names;
for my $name (@names) {
for my $base ('assets', './alias', realpath('assets')) {
my $expected_base = $owner eq 'debian' ? $base : realpath($base);
is(lookup($owner, 'debian12.5', $arch, $base, $extension),
"$expected_base/$name", "selects $name through $base");
}
unlink("assets/$name") or die "unlink $name: $!";
}
is(lookup($owner, 'debian12.5', $arch, 'assets', $extension), '', 'miss returns an empty string');
is(lookup($owner, 'debian12.5', $arch, 'missing', $extension), '', 'missing directory returns an empty string');
SKIP: {
skip 'root can read mode-0000 files', 1 unless $>;
write_text("assets/$names[0]", 'unreadable');
chmod(0000, "assets/$names[0]") or die "chmod: $!";
write_text("assets/$names[-1]", 'readable');
is(lookup($owner, 'debian12.5', $arch, 'assets', $extension),
($owner eq 'debian' ? 'assets' : realpath('assets')) . "/$names[-1]",
'an unreadable exact match does not hide the generic asset');
unlink(map { "assets/$_" } @names[0, -1]) == 2 or die "unlink: $!";
}
};
}
}
}
for my $case (
['debian', 'debian12.5', 'compute.debian12.x86_64.pkglist', 'compute.debian12.4.x86_64.pkglist'],
['debian', 'debian12.5.1', 'compute.debian12.5.x86_64.pkglist', 'compute.debian12.x86_64.pkglist'],
['imgutils', 'rocky9.6', 'compute.rocky9.5.x86_64.pkglist', 'compute.rocky9.x86_64.pkglist'],
['imgutils', 'ubuntu24.04', 'compute.pkglist', 'compute.ubuntu24.x86_64.pkglist'],
['imgutils', 'sles11sp1', 'compute.sles11.x86_64.pkglist', 'compute.pkglist'],
['imgutils', 'leap15.6', 'compute.sle15.x86_64.pkglist', 'compute.x86_64.pkglist'],
['imgutils', 'leap15.6', 'compute.sle15.pkglist', 'compute.pkglist'],
['debian', 'debian12', 'compute.debian12.x86_64.pkglist', 'compute.pkglist'],
) {
my ($owner, $version, $wanted, $other) = @$case;
write_text("assets/$_", "fixture\n") for ($wanted, $other);
is(lookup($owner, $version, 'x86_64', 'assets', 'pkglist'),
($owner eq 'debian' ? 'assets' : realpath('assets')) . "/$wanted",
"$owner retains its $version fallback policy");
unlink("assets/$wanted", "assets/$other") == 2 or die "unlink: $!";
}
chdir($original_dir) or die "chdir $original_dir: $!";
done_testing();