Merge pull request #7820 from VersatusHPC/fix/ubuntu-install-pkglists

Make ospkgs and detect_dhcpd work on current Ubuntu releases
2026-09-09 05:00:44 +00:00 · 2026-09-08 14:13:17 -03:00
parent 796103c300 107f7b14ec
commit cd4d903879
20 changed files with 412 additions and 11 deletions
@@ -5,6 +5,7 @@ BEGIN { $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : -d '/opt/xcat' ? '/o

 use lib "$::XCATROOT/probe/lib/perl";
 use probe_utils;
+use xCAT::CommandUtils;
 use File::Basename;
 use IO::Socket::INET;
 use Time::HiRes qw(gettimeofday sleep);
@@ -63,7 +64,8 @@ if ($::TEST) {
    exit 0;
 }

-unless (-x "/usr/sbin/tcpdump") {
+my $tcpdump = xCAT::CommandUtils::find_executable('tcpdump');
+unless ($tcpdump) {
    probe_utils->send_msg("$output", "f", "Tool 'tcpdump' is installed on current server");
    probe_utils->send_msg("$output", "d", "$program_name needs to leverage 'tcpdump', please install 'tcpdump' first");
    exit 1;
@@ -158,7 +160,7 @@ if (!defined $pid) {
 } elsif ($pid == 0) {

    # Child process
-    my $cmd = "tcpdump -i $nic port 68 -n -vvvvvv > $dumpfile 2>/dev/null";
+    my $cmd = "$tcpdump -i $nic port 68 -n -vvvvvv > $dumpfile 2>/dev/null";
    `$cmd`;
    exit 0;
 }
@@ -379,7 +381,7 @@ sub packdhcppkg {

 sub kill_child {
    kill 15, $pid;
-    my @pidoftcpdump = `ps -ef | grep -E "[0-9]+:[0-9]+:[0-9]+ tcpdump -i $nic" | awk -F' ' '{print \$2}'`;
+    my @pidoftcpdump = `ps -ef | grep -E "[0-9]+:[0-9]+:[0-9]+ $tcpdump -i $nic" | awk -F' ' '{print \$2}'`;
    foreach my $cpid (@pidoftcpdump) {
        kill 15, $cpid;
    }
@@ -0,0 +1,5 @@
+openssh-server
+chrony
+gawk
+nfs-common
+snmpd
@@ -3,8 +3,10 @@ ntp
 gawk
 nfs-common
 snmpd
-qemu-kvm
-libvirt-bin
+qemu-system
+qemu-utils
+libvirt-daemon-system
+libvirt-clients
 bridge-utils
 libcap2-bin
 vlan
@@ -0,0 +1,13 @@
+openssh-server
+ntp
+gawk
+nfs-common
+snmpd
+qemu-system-ppc
+qemu-utils
+libvirt-daemon-system
+libvirt-clients
+bridge-utils
+libcap2-bin
+vlan
+tmux
@@ -0,0 +1 @@
+kvm.ppc64el.pkglist
@@ -0,0 +1,11 @@
+openssh-server
+ntp
+gawk
+nfs-common
+snmpd
+qemu-kvm
+libvirt-bin
+bridge-utils
+libcap2-bin
+vlan
+tmux
@@ -0,0 +1,11 @@
+openssh-server
+ntp
+gawk
+nfs-common
+snmpd
+qemu-kvm
+libvirt-bin
+bridge-utils
+libcap2-bin
+vlan
+tmux
@@ -0,0 +1,11 @@
+openssh-server
+ntp
+gawk
+nfs-common
+snmpd
+qemu-kvm
+libvirt-bin
+bridge-utils
+libcap2-bin
+vlan
+tmux
@@ -0,0 +1,13 @@
+openssh-server
+ntp
+gawk
+nfs-common
+snmpd
+qemu-system-misc
+qemu-utils
+libvirt-daemon-system
+libvirt-clients
+bridge-utils
+libcap2-bin
+vlan
+tmux
@@ -0,0 +1,13 @@
+openssh-server
+chrony
+gawk
+nfs-common
+snmpd
+qemu-system
+qemu-utils
+libvirt-daemon-system
+libvirt-clients
+bridge-utils
+libcap2-bin
+vlan
+tmux
@@ -0,0 +1,13 @@
+openssh-server
+chrony
+gawk
+nfs-common
+snmpd
+qemu-system-ppc
+qemu-utils
+libvirt-daemon-system
+libvirt-clients
+bridge-utils
+libcap2-bin
+vlan
+tmux
@@ -0,0 +1 @@
+kvm.ubuntu26.04.ppc64el.pkglist
@@ -0,0 +1,13 @@
+openssh-server
+chrony
+gawk
+nfs-common
+snmpd
+qemu-system-x86
+qemu-utils
+libvirt-daemon-system
+libvirt-clients
+bridge-utils
+libcap2-bin
+vlan
+tmux
@@ -0,0 +1,13 @@
+openssh-server
+ntp
+gawk
+nfs-common
+snmpd
+qemu-system-x86
+qemu-utils
+libvirt-daemon-system
+libvirt-clients
+bridge-utils
+libcap2-bin
+vlan
+tmux
@@ -4,5 +4,5 @@ gawk
 nfs-common
 snmpd
 libdbd-mysql-perl
-libodbc1
-
+libdbd-pg-perl
+unixodbc
@@ -0,0 +1,8 @@
+openssh-server
+chrony
+gawk
+nfs-common
+snmpd
+libdbd-mysql-perl
+libdbd-pg-perl
+unixodbc
@@ -0,0 +1,17 @@
+bash
+nfs-common
+openssl
+isc-dhcp-client
+linux-image-generic
+openssh-server
+openssh-client
+wget
+vim
+chrony
+rsyslog
+rsync
+busybox-static
+gawk
+tar
+gzip
+xz-utils
@@ -1,4 +1,9 @@
 #!/usr/bin/perl
+BEGIN {
+    $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat';
+}
+use lib "$::XCATROOT/lib/perl";
+use xCAT::CommandUtils;
 use IO::Socket::INET;
 use Time::HiRes qw(gettimeofday sleep);
 use Getopt::Long;
@@ -27,7 +32,8 @@ if (!GetOptions(

 if ($::HELP) { print $::USAGE; exit 0; }

-unless (-x "/usr/sbin/tcpdump") {
+my $tcpdump = xCAT::CommandUtils::find_executable('tcpdump');
+unless ($tcpdump) {
    print "Error: Please install tcpdump before the detecting.\n";
    exit 1;
 }
@@ -92,7 +98,7 @@ my $dumpfile = "/tmp/dhcpdumpfile.log";
 if ($pid == 0) {

    # Child process
-    my $cmd = "tcpdump -i $IF port 68 -n -vvvvvv > $dumpfile 2>/dev/null";
+    my $cmd = "$tcpdump -i $IF port 68 -n -vvvvvv > $dumpfile 2>/dev/null";
    `$cmd`;
    exit 0;
 }
@@ -146,7 +152,7 @@ kill_child();

 #kill the child process
 kill 15, $pid;
-my @pidoftcpdump = `ps -ef | grep -E "[0-9]+:[0-9]+:[0-9]+ tcpdump -i $IF" | awk -F' ' '{print \$2}'`;
+my @pidoftcpdump = `ps -ef | grep -E "[0-9]+:[0-9]+:[0-9]+ $tcpdump -i $IF" | awk -F' ' '{print \$2}'`;
 foreach my $cpid (@pidoftcpdump) {
    kill 15, $cpid;

@@ -341,7 +347,7 @@ sub packdhcppkg {

 sub kill_child {
    kill 15, $pid;
-    my @pidoftcpdump = `ps -ef | grep -E "[0-9]+:[0-9]+:[0-9]+ tcpdump -i $IF" | awk -F' ' '{print \$2}'`;
+    my @pidoftcpdump = `ps -ef | grep -E "[0-9]+:[0-9]+:[0-9]+ $tcpdump -i $IF" | awk -F' ' '{print \$2}'`;
    foreach my $cpid (@pidoftcpdump) {
        kill 15, $cpid;

@@ -0,0 +1,145 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+
+use Config;
+use File::Path qw(make_path);
+use File::Spec;
+use File::Temp qw(tempdir);
+use FindBin;
+use Test::More;
+
+use lib "$FindBin::Bin/../../perl-xCAT";
+use xCAT::CommandUtils;
+
+# Both detect_dhcpd copies refused to run unless /usr/sbin/tcpdump existed. Debian and Ubuntu
+# install tcpdump as /usr/bin/tcpdump, so the rogue-DHCP detector refused to run on every Ubuntu
+# management node whether tcpdump was installed or not, and the probe reported its tcpdump check
+# as failed. The scripts are driven for real with a tcpdump that PATH alone can reach and that
+# records how it was started. Outside the namespace below, a host that also carries
+# /usr/sbin/tcpdump lets the previous guard pass as well.
+
+my $repo  = File::Spec->rel2abs( File::Spec->catdir( $FindBin::Bin, '..', '..' ) );
+my $tools = "$repo/xCAT-server/share/xcat/tools/detect_dhcpd";
+my $probe = "$repo/xCAT-probe/subcmds/detect_dhcpd";
+plan skip_all => 'detect_dhcpd not found' unless -f $tools && -f $probe;
+
+my $perl = $Config{perlpath};
+my $mac  = '02:00:5e:00:53:01';
+
+# The capture run masks /tmp, so every fixture lives outside it.
+my @fixture_dir = ( -d '/var/tmp' && -w '/var/tmp' ) ? ( DIR => '/var/tmp' ) : ();
+
+# An XCATROOT whose lib/perl and probe/lib/perl are this checkout, so the scripts load the
+# libraries they would load on a management node.
+my $root = tempdir( @fixture_dir, CLEANUP => 1 );
+make_path("$root/lib/perl", "$root/probe/lib");
+symlink( "$repo/perl-xCAT/xCAT",      "$root/lib/perl/xCAT" )   or die "symlink: $!";
+symlink( "$repo/xCAT-probe/lib/perl", "$root/probe/lib/perl" ) or die "symlink: $!";
+
+sub write_script {
+    my ( $path, $body ) = @_;
+    open( my $fh, '>', $path ) or die "$path: $!";
+    print {$fh} "#!/bin/sh\n$body";
+    close($fh);
+    chmod 0755, $path;
+}
+
+# PATH will hold one directory: a tcpdump that records its invocation and the tools the scripts
+# pipe through. The plain run gets an ip that answers nothing, so the scripts stop at the
+# interface step before they open a socket or fork the capture.
+my $marker = File::Spec->catfile( tempdir( @fixture_dir, CLEANUP => 1 ), 'tcpdump.ran' );
+sub fixture_bin {
+    my (%with) = @_;
+    my $bindir = tempdir( @fixture_dir, CLEANUP => 1 );
+    write_script( "$bindir/tcpdump", qq{printf '%s\\n' "\$0" "\$*" > '$marker'\nexit 0\n} );
+    foreach my $tool (qw(awk head grep ps)) {
+        my $real = xCAT::CommandUtils::find_executable($tool) or next;
+        symlink( $real, "$bindir/$tool" ) or die "symlink $tool: $!";
+    }
+    if ( $with{real_ip} ) {
+        symlink( $with{real_ip}, "$bindir/ip" ) or die "symlink ip: $!";
+    } else {
+        write_script( "$bindir/ip", "exit 0\n" );
+    }
+    return $bindir;
+}
+
+sub script_command {
+    my ( $script, @args ) = @_;
+    return "$perl -I '$repo/perl-xCAT' -I '$repo/xCAT-probe/lib/perl' '$script' @args";
+}
+
+sub run_with_path {
+    my ( $bindir, $script, @args ) = @_;
+    local $ENV{PATH}     = $bindir;
+    local $ENV{XCATROOT} = $root;
+    unlink $marker;
+    return `@{[ script_command( $script, @args ) ]} 2>&1`;
+}
+
+sub recorded_invocation {
+    open( my $fh, '<', $marker ) or return;
+    chomp( my @lines = <$fh> );
+    close($fh);
+    return @lines;
+}
+
+my $plain = fixture_bin();
+my $out = run_with_path( $plain, $tools, '-i', 'lo', '-m', $mac, '-t', '1' );
+unlike( $out, qr/install tcpdump/, 'the tool accepts a tcpdump found through PATH' );
+like( $out, qr/IP\/MAC/, '... and gets as far as the interface step' );
+$out = run_with_path( $plain, $probe, '-i', 'lo', '-m', $mac, '-d', '1' );
+unlike( $out, qr/please install 'tcpdump' first/, 'the probe accepts a tcpdump found through PATH' );
+like( $out, qr/IP\/MAC/, '... and gets as far as the interface step' );
+
+# The capture itself runs only inside a private network and mount namespace: the loopback
+# interface is the only one, its default route keeps the DHCP discover on the host, and a tmpfs
+# over /tmp keeps the dump file out of the shared one. The scripts then reach tcpdump as they do
+# on a management node. The loopback interface has no Ethernet address, so the MAC is given.
+# /usr/sbin/tcpdump is hidden there, so the previous guard fails on every host.
+sub isolation {
+    my %bin = map { $_ => xCAT::CommandUtils::find_executable($_) } qw(unshare mount ip);
+    return unless $bin{unshare} && $bin{mount} && $bin{ip};
+    my $setup = "$bin{mount} -t tmpfs tmpfs /tmp && $bin{ip} link set lo up && $bin{ip} route add default dev lo"
+      . " && { [ ! -e /usr/sbin/tcpdump ] || $bin{mount} --bind /dev/null /usr/sbin/tcpdump; }";
+    foreach my $flags (qw(-mn -rmn)) {
+        next unless system("$bin{unshare} $flags sh -c '$setup' >/dev/null 2>&1") == 0;
+        return { unshare => $bin{unshare}, flags => $flags, setup => $setup, ip => $bin{ip} };
+    }
+    return;
+}
+
+sub run_isolated {
+    my ( $ns, $bindir, $script, @args ) = @_;
+    unlink $marker;
+    my $command = "env PATH='$bindir' XCATROOT='$root' " . script_command( $script, @args );
+    my $shell   = "$ns->{unshare} $ns->{flags} sh -c \"$ns->{setup} && exec $command\" 2>&1";
+    my $out     = `$shell`;
+    diag($out) if $?;
+    return $out;
+}
+
+SKIP: {
+    skip 'the private /tmp would hide this checkout or its fixtures', 8
+      if index( $repo, '/tmp/' ) == 0 || !@fixture_dir;
+    my $ns = isolation();
+    skip 'no private network namespace on this host', 8 unless $ns;
+    my $isolated = fixture_bin( real_ip => $ns->{ip} );
+
+    $out = run_isolated( $ns, $isolated, $tools, '-i', 'lo', '-m', $mac, '-t', '1' );
+    like( $out, qr/servers reply/, 'the tool runs its capture window in the namespace' );
+    ok( -f $marker, '... and starts tcpdump' );
+    my ( $ran, $args ) = recorded_invocation();
+    is( $ran,  "$isolated/tcpdump",         '... by the path it resolved' );
+    is( $args, "-i lo port 68 -n -vvvvvv", '... with the capture arguments' );
+
+    $out = run_isolated( $ns, $isolated, $probe, '-i', 'lo', '-m', $mac, '-d', '1' );
+    like( $out, qr/servers replied/, 'the probe runs its capture window in the namespace' );
+    ok( -f $marker, '... and starts tcpdump' );
+    ( $ran, $args ) = recorded_invocation();
+    is( $ran,  "$isolated/tcpdump",         '... by the path it resolved' );
+    is( $args, "-i lo port 68 -n -vvvvvv", '... with the capture arguments' );
+}
+
+done_testing();
@@ -0,0 +1,103 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+
+use File::Basename qw(basename);
+use File::Spec;
+use FindBin;
+use Test::More;
+
+use lib "$FindBin::Bin/../../perl-xCAT", "$FindBin::Bin/../../xCAT-server/lib/perl";
+use xCAT::SvrUtils;
+use xCAT::Postage;
+
+# The arch-neutral Ubuntu package lists are what every release and architecture without a list
+# of its own falls back to. Postage exports the resolved list as OSPKGS and ospkgs hands it to
+# one apt-get install, so one name the archive no longer carries loses every package on the
+# list. ntp and ntpdate are gone from 26.04, libodbc1 (the unixODBC runtime the odbcsetup
+# postscript needs) was renamed on 24.04, libvirt-bin is gone from 20.04 on and qemu-kvm from
+# 22.04 on. Older releases keep the names, and the daemon, they had.
+
+my $repo_root = File::Spec->rel2abs( File::Spec->catdir( $FindBin::Bin, '..', '..' ) );
+my $share     = File::Spec->catdir( $repo_root, 'xCAT-server', 'share', 'xcat' );
+my $install   = File::Spec->catdir( $share, 'install', 'ubuntu' );
+my $netboot   = File::Spec->catdir( $share, 'netboot', 'ubuntu' );
+
+# ospkgs receives the list as get_pkglist_tex renders it for OSPKGS, includes expanded.
+sub packages_in {
+    my ($path) = @_;
+    return {} unless $path && -f $path;
+    my %p = map { $_ => 1 } grep { length } split /,/, xCAT::Postage::get_pkglist_tex($path);
+    return \%p;
+}
+
+# The list an osimage of this profile, release and architecture resolves to, and its packages.
+sub resolved {
+    my ( $dir, $profile, $os, $arch ) = @_;
+    my $file = xCAT::SvrUtils->get_pkglist_file_name( $dir, $profile, $os, $arch );
+    return ( basename( $file || '' ), packages_in($file) );
+}
+
+# The shared lists keep ntp for the releases that still carry it. aarch64 has no list of its own.
+foreach my $case ( [ $install, 'compute' ], [ $install, 'service' ], [ $install, 'kvm' ], [ $netboot, 'compute' ] ) {
+    my ( $dir, $profile ) = @$case;
+    my ( $file, $p ) = resolved( $dir, $profile, 'ubuntu24.04.4', 'aarch64' );
+    is( $file, "$profile.pkglist", "$profile on 24.04 without a list of its own resolves to the shared list" );
+    ok( $p->{ntp}, "... which keeps ntp" );
+    ok( !$p->{$_}, "... and no longer names $_" ) for qw(libodbc1 qemu-kvm libvirt-bin);
+}
+ok( packages_in("$install/service.pkglist")->{unixodbc}, 'the shared service list names unixodbc' );
+ok( packages_in("$install/service.pkglist")->{'libdbd-pg-perl'}, '... and the PostgreSQL driver beside the MySQL one' );
+
+# 26.04 dropped ntp, so each shared list has a 26.04 counterpart that carries chrony.
+foreach my $case ( [ $install, 'compute' ], [ $install, 'service' ], [ $install, 'kvm' ], [ $netboot, 'compute' ] ) {
+    my ( $dir, $profile ) = @$case;
+    my ( $file, $p ) = resolved( $dir, $profile, 'ubuntu26.04.1', 'aarch64' );
+    is( $file, "$profile.ubuntu26.04.pkglist", "$profile on 26.04 without a list of its own resolves to the 26.04 list" );
+    ok( $p->{chrony},                 "... which names chrony" );
+    ok( !$p->{ntp} && !$p->{ntpdate}, "... and neither ntp nor ntpdate" );
+}
+ok( packages_in("$install/service.ubuntu26.04.pkglist")->{unixodbc}, 'the 26.04 service list names unixodbc' );
+ok( packages_in("$install/service.ubuntu26.04.pkglist")->{'libdbd-pg-perl'}, '... and the PostgreSQL driver beside the MySQL one' );
+
+# qemu-kvm was a transitional name for the emulator of the host architecture. No current release
+# has one name for that, so a per-architecture list names the native one and the shared kvm lists
+# fall back to qemu-system, which carries every emulator. ospkgs installs without recommends, so
+# every list names qemu-utils for the qcow2 volumes kvm.pm creates. 12.04, 14.04 and 16.04 keep
+# the names they shipped with.
+my @kvm = (
+    # os              arch       list                              emulator            libvirt
+    [ 'ubuntu12.04.5', 'x86_64',  'kvm.ubuntu12.04.pkglist',         'qemu-kvm',         'libvirt-bin' ],
+    [ 'ubuntu14.04.4', 'x86_64',  'kvm.ubuntu14.04.pkglist',         'qemu-kvm',         'libvirt-bin' ],
+    [ 'ubuntu16.04',   'x86_64',  'kvm.ubuntu16.04.pkglist',         'qemu-kvm',         'libvirt-bin' ],
+    [ 'ubuntu18.04',   'x86_64',  'kvm.x86_64.pkglist',              'qemu-system-x86',  'libvirt-daemon-system' ],
+    [ 'ubuntu24.04.4', 'x86_64',  'kvm.x86_64.pkglist',              'qemu-system-x86',  'libvirt-daemon-system' ],
+    [ 'ubuntu24.04.4', 'ppc64el', 'kvm.ppc64el.pkglist',             'qemu-system-ppc',  'libvirt-daemon-system' ],
+    [ 'ubuntu24.04.4', 'ppc64le', 'kvm.ppc64le.pkglist',             'qemu-system-ppc',  'libvirt-daemon-system' ],
+    [ 'ubuntu24.04.4', 'riscv64', 'kvm.ubuntu24.04.riscv64.pkglist', 'qemu-system-misc', 'libvirt-daemon-system' ],
+    [ 'ubuntu26.04.1', 'x86_64',  'kvm.ubuntu26.04.x86_64.pkglist',  'qemu-system-x86',  'libvirt-daemon-system' ],
+    [ 'ubuntu26.04.1', 'ppc64el', 'kvm.ubuntu26.04.ppc64el.pkglist', 'qemu-system-ppc',  'libvirt-daemon-system' ],
+    [ 'ubuntu26.04.1', 'ppc64le', 'kvm.ubuntu26.04.ppc64le.pkglist', 'qemu-system-ppc',  'libvirt-daemon-system' ],
+);
+foreach my $case (@kvm) {
+    my ( $os, $arch, $list, $emulator, $libvirt ) = @$case;
+    my ( $file, $p ) = resolved( $install, 'kvm', $os, $arch );
+    is( $file, $list, "kvm on $os $arch resolves to $list" );
+    ok( $p->{$emulator}, "... which installs $emulator" );
+    ok( $p->{$libvirt},  "... and $libvirt" );
+    ok( !$p->{'qemu-system'}, "... and not the qemu-system fallback" );
+    ok( $p->{'qemu-utils'}, "... and qemu-utils" ) if $libvirt eq 'libvirt-daemon-system';
+}
+
+# An architecture without a list of its own gets the qemu-system fallback. On 26.04 riscv64
+# libvirt-daemon-system depends on qemu-kvm or qemu-system and nothing provides qemu-kvm, so apt
+# installs that fallback there whatever the list names, and that release keeps its 26.04 list.
+foreach my $case ( [ 'ubuntu24.04.4', 'aarch64', 'kvm.pkglist' ], [ 'ubuntu26.04.1', 'aarch64', 'kvm.ubuntu26.04.pkglist' ],
+                   [ 'ubuntu26.04.1', 'riscv64', 'kvm.ubuntu26.04.pkglist' ] ) {
+    my ( $os, $arch, $list ) = @$case;
+    my ( $file, $p ) = resolved( $install, 'kvm', $os, $arch );
+    is( $file, $list, "kvm on $os $arch resolves to $list" );
+    ok( $p->{'qemu-system'} && $p->{'qemu-utils'} && $p->{'libvirt-daemon-system'}, '... which names the fallback emulator, qemu-utils and libvirt' );
+}
+
+done_testing();