xCAT/xcat-core
2
0
Fork 0
mirror of https://github.com/xcat2/xcat-core.git synced 2026-09-05 04:27:55 +00:00
Code Issues Releases Wiki Activity

fix(xcat-core): the install monitor's respawn pacing cannot be tested inside xcatd

Browse Source 
The backoff that decides when to re-fork the install monitor is arithmetic over a handful
of counters, but it lives inline in xcatd among the daemon's globals, its signal handlers
and its fork. xcatd needs the database, SSL, the plugin tree and /var/run/xcat before it
will run, so nothing in a unit test can execute that arithmetic; a test can only match
patterns against the script's source and hope the shape it finds behaves. That is how a
retry budget which ran out and could never be refilled passed a green test run.

Move the pacing to xCAT::RespawnUtils as pure functions: each takes the current state and
the current time and returns the next state, reading no clock, no globals and no files.
Passing the time in is what makes the schedule checkable over a virtual clock instead of
in real seconds, and returning a new state rather than mutating one is what makes it safe
to call from the SIGCHLD handler -- the result is built before the caller installs it, so
a signal arriving partway through cannot leave the pacing half-updated.

The behaviour is unchanged from the previous commit and stays covered by
xCAT-test/unit/xcatd_monitor_respawn.t, which now executes these functions instead of
grepping for them: the delay doubles from XCATD_MON_RESPAWN_MIN_INTERVAL (5s) to
XCATD_MON_RESPAWN_MAX_INTERVAL (300s) and holds there without ever refusing a retry, and a
monitor that stayed up XCATD_MON_RESPAWN_HEALTHY seconds (60s) resets the backoff when it
later dies. policy() now also refuses a floor below one second, which would double to
itself and give a fork storm rather than a backoff, and a ceiling under the floor.

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
This commit is contained in:
Daniel Hilst
2026-08-26 17:07:59 -03:00
parent 173ce5c6a3
commit 49b0c26efb
2 changed files with 98 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
perl-xCAT/xCAT/RespawnUtils.pm
+74
View File
@@ -0,0 +1,74 @@
# IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html
package xCAT::RespawnUtils;
use strict;
use warnings;
sub policy {
my (%opt) = @_;
my $min = defined($opt{min_interval}) ? $opt{min_interval} : 5;
my $max = defined($opt{max_interval}) ? $opt{max_interval} : 300;
my $healthy = defined($opt{healthy}) ? $opt{healthy} : 60;
$min = 1 if $min < 1;
$max = $min if $max < $min;
return {
min_interval => $min,
max_interval => $max,
healthy => $healthy,
delay => $min,
next_at => 0,
started_at => undef,
streak => 0,
reported => 0,
};
}
sub due {
my ($state, $now) = @_;
return $now >= $state->{next_at} ? 1 : 0;
}
sub forked {
my ($state, $now) = @_;
return { %$state, started_at => $now };
}
sub exited {
my ($state, $now) = @_;
my %next = (%$state, started_at => undef);
if (defined($state->{started_at})
and ($now - $state->{started_at}) >= $state->{healthy}) {
$next{delay} = $state->{min_interval};
$next{next_at} = $now;
$next{streak} = 0;
$next{reported} = 0;
} else {
$next{streak} = $state->{streak} + 1;
$next{next_at} = $now + $state->{delay};
$next{delay} = ($state->{delay} * 2 > $state->{max_interval})
? $state->{max_interval}
: $state->{delay} * 2;
}
return \%next;
}
sub should_report {
my ($state) = @_;
return 0 if $state->{reported};
return 0 if $state->{streak} < 1;
return $state->{delay} >= $state->{max_interval} ? 1 : 0;
}
sub reported {
my ($state) = @_;
return { %$state, reported => 1 };
}
1;
xCAT-server/sbin/xcatd
+24 -90
View File
@@ -30,85 +30,16 @@ my $udpctl;
my $pid_UDP;
my $pid_MON;
# BEGIN mon-respawn-policy
# When the install monitor dies, the main service loop below re-forks it. This decides
# WHEN. It is deliberately free of forking and of daemon state so that
# xCAT-test/unit/xcatd_monitor_respawn.t can extract this region verbatim and drive it.
#
# Respawning has to be paced. do_installm_service dies when it cannot bind xcatiport, so
# while something else holds that port every respawn is a fast, futile fork that also
# re-enters that function's USR2 socket-takeover handshake against whatever holds it.
#
# But pacing must never become giving up. A retry budget that runs out cannot be refilled:
# with no monitor alive, nothing is left to reset it. The port would stay dead until xcatd
# is restarted -- which is exactly the failure this respawn exists to remove, just reached
# more slowly -- and a port that frees up a minute later would never be picked back up.
#
# So the delay doubles from $mon_respawn_min to $mon_respawn_max and then stays there.
# A monitor that cannot start costs one fork per $mon_respawn_max seconds for as long as
# that lasts, and comes back within that bound once the port is free again.
#
# A monitor that survived $mon_respawn_healthy seconds plainly got the socket and served,
# so its eventual death resets the delay: an isolated death is retried at once, and the
# backoff only builds up during a real streak of failures to start.
my $mon_respawn_min = defined($ENV{XCATD_MON_RESPAWN_MIN_INTERVAL}) ? $ENV{XCATD_MON_RESPAWN_MIN_INTERVAL} : 5;
my $mon_respawn_max = defined($ENV{XCATD_MON_RESPAWN_MAX_INTERVAL}) ? $ENV{XCATD_MON_RESPAWN_MAX_INTERVAL} : 300;
my $mon_respawn_healthy = defined($ENV{XCATD_MON_RESPAWN_HEALTHY}) ? $ENV{XCATD_MON_RESPAWN_HEALTHY} : 60;
$mon_respawn_min = 1 if $mon_respawn_min < 1; # 0 would be a fork storm
$mon_respawn_max = $mon_respawn_min if $mon_respawn_max < $mon_respawn_min;
my $mon_respawn_delay = $mon_respawn_min; # delay to apply after the NEXT failure
my $mon_respawn_next = 0; # earliest time() at which to try again
my $mon_respawn_started; # time() the running monitor was forked
my $mon_respawn_streak = 0; # consecutive monitors that died young
my $mon_respawn_capped = 0; # has this streak already reported the ceiling?
# Is a respawn allowed yet? This only ever answers "not yet" -- never "no more".
sub mon_respawn_due {
my ($now) = @_;
return $now >= $mon_respawn_next;
}
# Note that a monitor is being forked. Called BEFORE the fork: the child can die, and be
# reaped, before xfork even returns to the parent.
sub mon_respawn_forked {
my ($now) = @_;
$mon_respawn_started = $now;
return;
}
# Note that the monitor exited, and schedule the next attempt. Returns the number of
# consecutive monitors that have died young (0 once one of them managed to serve).
# Safe to call from the SIGCHLD handler: arithmetic only, no I/O.
sub mon_respawn_exited {
my ($now) = @_;
if (defined($mon_respawn_started) and ($now - $mon_respawn_started) >= $mon_respawn_healthy) {
$mon_respawn_delay = $mon_respawn_min; # it served; this is a fresh start
$mon_respawn_next = $now;
$mon_respawn_streak = 0;
$mon_respawn_capped = 0;
} else {
$mon_respawn_streak++;
$mon_respawn_next = $now + $mon_respawn_delay;
$mon_respawn_delay = ($mon_respawn_delay * 2 > $mon_respawn_max)
? $mon_respawn_max
: $mon_respawn_delay * 2;
}
$mon_respawn_started = undef;
return $mon_respawn_streak;
}
# True once per failure streak, when the backoff first reaches its ceiling. So a monitor
# that cannot start is reported once rather than on every attempt, and is reported afresh
# if it starts failing again after having served.
sub mon_respawn_hit_ceiling {
return 0 if $mon_respawn_capped;
return 0 if $mon_respawn_streak < 1;
return 0 if $mon_respawn_delay < $mon_respawn_max;
$mon_respawn_capped = 1;
return 1;
}
# END mon-respawn-policy
# Pacing for re-forking the install monitor; see the main service loop below. The pacing
# itself is pure arithmetic in xCAT::RespawnUtils -- a backoff with a ceiling but no end,
# so a held xcatiport costs one fork per ceiling interval instead of a fork storm, and the
# monitor is back within that bound once the port is free. Each function returns a NEW
# state, which is why the reaper can safely install one from inside the signal handler.
my $mon_respawn = xCAT::RespawnUtils::policy(
min_interval => $ENV{XCATD_MON_RESPAWN_MIN_INTERVAL},
max_interval => $ENV{XCATD_MON_RESPAWN_MAX_INTERVAL},
healthy => $ENV{XCATD_MON_RESPAWN_HEALTHY},
);
my $numofnodes=0;
@@ -130,6 +61,7 @@ use xCAT::TLSPolicy qw(resolve_xcatd_tls_settings);
use xCAT::TableUtils;
use xCAT::NetworkUtils;
use xCAT::MsgUtils;
use xCAT::RespawnUtils;
use xCAT::xcatd;
use xCAT::CmdLog;
use xCAT::State;
@@ -1144,7 +1076,7 @@ sub ssl_reaper {
}
if ($CHILDPID == $pid_MON) {
$pid_MON = 0;
mon_respawn_exited(time()); # paces the re-fork the main service loop will do
$mon_respawn = xCAT::RespawnUtils::exited($mon_respawn, time());
}
}
$SIG{CHLD} = \&ssl_reaper;
@@ -1270,7 +1202,8 @@ if (!(socketpair($rescanreadpipe, $rescanwritepipe, AF_UNIX, SOCK_STREAM, PF_UNS
}
$rescanrselect = new IO::Select;
$rescanrselect->add($rescanreadpipe);
mon_respawn_forked(time()); # so this monitor's uptime counts towards the respawn policy too
# record this monitor too, so its uptime counts when it eventually dies
$mon_respawn = xCAT::RespawnUtils::forked($mon_respawn, time());
$pid_MON = xCAT::Utils->xfork;
if (!defined $pid_MON) {
xCAT::MsgUtils->message("S", "Unable to fork installmonitor");
@@ -1551,21 +1484,22 @@ until ($quit) {
# leave xcatiport permanently dead while this daemon kept running, so installing nodes could
# no longer report status or request the boot flip until the WHOLE daemon was restarted.
#
# The mon-respawn-policy region near the top of this file paces the attempts, so a port that
# stays held costs one fork per ceiling interval instead of a fork storm. It never stops
# saying yes, so the monitor also comes back on its own once whatever held the port lets go.
if (!$pid_MON && !$quit && $sport && mon_respawn_due(time())) {
if (mon_respawn_hit_ceiling()) {
# xCAT::RespawnUtils paces the attempts, and never stops allowing them, so the monitor also
# comes back on its own once whatever held the port lets go of it.
if (!$pid_MON && !$quit && $sport && xCAT::RespawnUtils::due($mon_respawn, time())) {
if (xCAT::RespawnUtils::should_report($mon_respawn)) {
xCAT::MsgUtils->message("S",
"xcatd: install monitor is not staying up (${mon_respawn_streak} attempts);"
. " still retrying xcatiport $sport every $mon_respawn_max seconds");
"xcatd: install monitor is not staying up ($mon_respawn->{streak} attempts);"
. " still retrying xcatiport $sport every $mon_respawn->{max_interval} seconds");
$mon_respawn = xCAT::RespawnUtils::reported($mon_respawn);
}
mon_respawn_forked(time());
$mon_respawn = xCAT::RespawnUtils::forked($mon_respawn, time());
$pid_MON = xCAT::Utils->xfork;
if (!defined $pid_MON) {
xCAT::MsgUtils->message("S", "xcatd: unable to re-fork install monitor");
$pid_MON = 0;
mon_respawn_exited(time()); # count the failed fork and back off before retrying
# count the failed fork and back off before retrying
$mon_respawn = xCAT::RespawnUtils::exited($mon_respawn, time());
} elsif (!$pid_MON) { # child: serve only the install monitor
$$progname = "xcatd: install monitor";
$pid_UDP = 0;