xCAT/xcat-core
2
0
Fork 0
mirror of https://github.com/xcat2/xcat-core.git synced 2026-09-05 04:27:55 +00:00
Code Issues Releases Wiki Activity

test(xcat-core): nothing checks that xcatd respawns the monitor at all

Browse Source 
The unit tests drive xCAT::RespawnUtils, which is where the pacing lives, but nothing
connects that to the daemon. Replacing the respawn condition in xcatd's service loop with
"if (0)" -- so a dead install monitor is never re-forked -- leaves the whole suite green.
The behaviour the PR exists to deliver is unverified.

That gap cannot be closed in a unit test: xcatd needs the database, SSL, the plugin tree
and /var/run/xcat before it will start, which is why the pacing was extracted in the first
place. It belongs in xCAT-test, where there is a running daemon to kill things in. Add a
case that kills the install monitor and requires that a new one appears, that it reclaims
xcatiport rather than merely existing, and that the SSL listener keeps its pid throughout
-- surviving without a restart being the entire point.

The process titles are matched anchored. An unanchored "pgrep -f xcatd: install monitor"
also matches the running test's own command line, and the kill would then take out the
test; that was observed on a live MN, not guessed.

Two smaller test defects go with it. The fork test kept every pid it forked in @spawned
and had its END block signal all of them, including ones it had already reaped -- verified
as 3 of 3 -- so a recycled pid would take a signal meant for a process that no longer
exists, and the suite runs as root in CI. Reaped pids now leave the list. And the tunables
reach policy() straight from %ENV, where they can be empty or misspelt; assert that none
of those shapes produces a Perl warning, since xcatd runs under use warnings and would put
one in the daemon log on every start.

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
This commit is contained in:
Daniel Hilst
2026-08-26 17:37:49 -03:00
parent 49b0c26efb
commit e364172b99
2 changed files with 60 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
xCAT-test/autotest/testcase/xcatd/case0
+26
View File
@@ -187,3 +187,29 @@ cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then
check:rc==0
check:output!~Error|ERROR
end
start:xcatd_install_monitor_respawns
description:a killed install monitor comes back and reclaims xcatiport, without restarting xcatd
label:mn_only,ci_test,xcatd
#the process titles are matched anchored: an unanchored "pgrep -f" also matches this
#test's own command line, and the kill below would then take out the test itself
cmd:mkdir -p /tmp/xcatd_monitor_respawn; pgrep -f "^xcatd: install monitor$" > /tmp/xcatd_monitor_respawn/before.pid; test -s /tmp/xcatd_monitor_respawn/before.pid
check:rc==0
cmd:pgrep -f "^xcatd: SSL listener$" > /tmp/xcatd_monitor_respawn/listener.pid; test -s /tmp/xcatd_monitor_respawn/listener.pid
check:rc==0
cmd:kill -9 $(cat /tmp/xcatd_monitor_respawn/before.pid)
check:rc==0
#it must return on its own. the respawn floor is 5s by default, so poll well past that
cmd:for i in $(seq 1 30); do sleep 2; pgrep -f "^xcatd: install monitor$" > /tmp/xcatd_monitor_respawn/after.pid; test -s /tmp/xcatd_monitor_respawn/after.pid && break; done; test -s /tmp/xcatd_monitor_respawn/after.pid
check:rc==0
cmd:test "$(cat /tmp/xcatd_monitor_respawn/before.pid)" != "$(cat /tmp/xcatd_monitor_respawn/after.pid)"
check:rc==0
#and it must have reclaimed the port, not merely be running again
cmd:XCATIPORT=$(lsdef -t site -i xcatiport 2>/dev/null | grep xcatiport | cut -d= -f2); XCATIPORT=${XCATIPORT:-3002}; ss -lntp | grep ":$XCATIPORT " | grep -q "pid=$(cat /tmp/xcatd_monitor_respawn/after.pid)"
check:rc==0
#xcatd itself must not have been restarted -- surviving without a restart is the point
cmd:pgrep -f "^xcatd: SSL listener$" | diff -q - /tmp/xcatd_monitor_respawn/listener.pid
check:rc==0
cmd:rm -rf /tmp/xcatd_monitor_respawn
check:rc==0
end
xCAT-test/unit/xcatd_monitor_respawn.t
+34 -5
View File
@@ -115,6 +115,20 @@ subtest 'a policy cannot be built with a delay that fails to back off' => sub {
is( $default->{min_interval}, 5, 'unset options fall back to the default floor' );
is( $default->{max_interval}, 300, '...and the default ceiling' );
is( $default->{healthy}, 60, '...and the default healthy uptime' );
# These arrive straight from %ENV, so they can be empty or misspelt. xcatd runs under
# use warnings: comparing a non-numeric one would put "Argument isn't numeric" in the
# daemon log on every start. Anything that is not a plain non-negative integer is
# treated as unset.
my @warnings;
local $SIG{__WARN__} = sub { push @warnings, @_ };
for my $junk ( '', 'abc', '-5', '3.5' ) {
is( xCAT::RespawnUtils::policy( min_interval => $junk )->{min_interval}, 5,
"a min_interval of '$junk' falls back to the default" );
}
is( xCAT::RespawnUtils::policy( min_interval => ' 7 ' )->{min_interval}, 7,
'a padded value is still read as a number' );
is_deeply( \@warnings, [], 'no tunable produces a Perl warning' );
};
subtest 'hitting the ceiling is reported once per failure streak' => sub {
@@ -203,6 +217,7 @@ subtest 'the monitor comes back on its own once the port is released' => sub {
my $pump = sub {
if ($mon_pid) {
if ( waitpid( $mon_pid, WNOHANG ) == $mon_pid ) {
@spawned = grep { $_ != $mon_pid } @spawned; # reaped: not ours to signal
push @deaths, [ $mon_pid, $mon_forked_at, time() ];
$pace = exited( $pace, time() );
$mon_pid = 0;
@@ -234,10 +249,19 @@ subtest 'the monitor comes back on its own once the port is released' => sub {
select( undef, undef, undef, 0.05 );
};
# This subtest is the one place in the file that depends on real elapsed time. It
# normally finishes in well under ten seconds; the deadline is a runaway guard, not a
# timing assertion. Say so when it fires, so a loaded runner reports a timeout rather
# than an assertion that looks like a logic failure.
my $deadline = time() + 60;
my $timed_out = sub { time() >= $deadline };
# (1) the port is held: monitors must fail repeatedly, without a fork storm
$pump->() while ( @deaths < 3 && time() < $deadline );
$pump->() while ( @deaths < 3 && !$timed_out->() );
if ( $timed_out->() && @deaths < 3 ) {
diag( "timed out waiting for three failed monitors (got "
. scalar(@deaths) . "); the runner is too loaded for this subtest" );
}
cmp_ok( scalar(@deaths), '>=', 3,
'the monitor is retried several times while the port is held' )
or return;
@@ -254,7 +278,9 @@ subtest 'the monitor comes back on its own once the port is released' => sub {
# (3) it must recover by itself
$pump->()
while ( !( $mon_pid && time() - $mon_forked_at >= $healthy + 1 )
&& time() < $deadline );
&& !$timed_out->() );
diag("timed out waiting for the monitor to reclaim the freed port")
if $timed_out->() && !$mon_pid;
ok( $mon_pid && time() - $mon_forked_at >= $healthy + 1,
'a respawned monitor binds the freed port and stays up -- no xcatd restart' )
@@ -265,15 +291,18 @@ subtest 'the monitor comes back on its own once the port is released' => sub {
# (4) and after that healthy run the pacing is back to prompt
my $forks_before = scalar(@forks);
kill 'TERM', $mon_pid;
$pump->() while ( @forks == $forks_before && time() < $deadline );
$pump->() while ( @forks == $forks_before && !$timed_out->() );
cmp_ok( scalar(@forks), '>', $forks_before,
'killing the healthy monitor gets it replaced again' );
cmp_ok( $forks[-1] - $deaths[-1][2], '<=', 2,
'that replacement is prompt: the healthy run reset the backoff' );
kill 'TERM', $mon_pid if $mon_pid;
waitpid( $mon_pid, 0 ) if $mon_pid;
if ($mon_pid) {
kill 'TERM', $mon_pid;
waitpid( $mon_pid, 0 );
@spawned = grep { $_ != $mon_pid } @spawned;
}
};
done_testing();