From 49b0c26efb3e30435eb99ba451721ca5cc4c2955 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:07:59 -0300 Subject: [PATCH] fix(xcat-core): the install monitor's respawn pacing cannot be tested inside xcatd The backoff that decides when to re-fork the install monitor is arithmetic over a handful of counters, but it lives inline in xcatd among the daemon's globals, its signal handlers and its fork. xcatd needs the database, SSL, the plugin tree and /var/run/xcat before it will run, so nothing in a unit test can execute that arithmetic; a test can only match patterns against the script's source and hope the shape it finds behaves. That is how a retry budget which ran out and could never be refilled passed a green test run. Move the pacing to xCAT::RespawnUtils as pure functions: each takes the current state and the current time and returns the next state, reading no clock, no globals and no files. Passing the time in is what makes the schedule checkable over a virtual clock instead of in real seconds, and returning a new state rather than mutating one is what makes it safe to call from the SIGCHLD handler -- the result is built before the caller installs it, so a signal arriving partway through cannot leave the pacing half-updated. The behaviour is unchanged from the previous commit and stays covered by xCAT-test/unit/xcatd_monitor_respawn.t, which now executes these functions instead of grepping for them: the delay doubles from XCATD_MON_RESPAWN_MIN_INTERVAL (5s) to XCATD_MON_RESPAWN_MAX_INTERVAL (300s) and holds there without ever refusing a retry, and a monitor that stayed up XCATD_MON_RESPAWN_HEALTHY seconds (60s) resets the backoff when it later dies. policy() now also refuses a floor below one second, which would double to itself and give a fork storm rather than a backoff, and a ceiling under the floor. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- perl-xCAT/xCAT/RespawnUtils.pm | 74 +++++++++++++++++++++ xCAT-server/sbin/xcatd | 114 +++++++-------------------------- 2 files changed, 98 insertions(+), 90 deletions(-) create mode 100644 perl-xCAT/xCAT/RespawnUtils.pm diff --git a/perl-xCAT/xCAT/RespawnUtils.pm b/perl-xCAT/xCAT/RespawnUtils.pm new file mode 100644 index 000000000..cb802e5fa --- /dev/null +++ b/perl-xCAT/xCAT/RespawnUtils.pm @@ -0,0 +1,74 @@ +# IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html +package xCAT::RespawnUtils; + +use strict; +use warnings; + +sub policy { + my (%opt) = @_; + + my $min = defined($opt{min_interval}) ? $opt{min_interval} : 5; + my $max = defined($opt{max_interval}) ? $opt{max_interval} : 300; + my $healthy = defined($opt{healthy}) ? $opt{healthy} : 60; + + $min = 1 if $min < 1; + $max = $min if $max < $min; + + return { + min_interval => $min, + max_interval => $max, + healthy => $healthy, + delay => $min, + next_at => 0, + started_at => undef, + streak => 0, + reported => 0, + }; +} + +sub due { + my ($state, $now) = @_; + return $now >= $state->{next_at} ? 1 : 0; +} + +sub forked { + my ($state, $now) = @_; + return { %$state, started_at => $now }; +} + +sub exited { + my ($state, $now) = @_; + + my %next = (%$state, started_at => undef); + + if (defined($state->{started_at}) + and ($now - $state->{started_at}) >= $state->{healthy}) { + + $next{delay} = $state->{min_interval}; + $next{next_at} = $now; + $next{streak} = 0; + $next{reported} = 0; + } else { + $next{streak} = $state->{streak} + 1; + $next{next_at} = $now + $state->{delay}; + $next{delay} = ($state->{delay} * 2 > $state->{max_interval}) + ? $state->{max_interval} + : $state->{delay} * 2; + } + + return \%next; +} + +sub should_report { + my ($state) = @_; + return 0 if $state->{reported}; + return 0 if $state->{streak} < 1; + return $state->{delay} >= $state->{max_interval} ? 1 : 0; +} + +sub reported { + my ($state) = @_; + return { %$state, reported => 1 }; +} + +1; diff --git a/xCAT-server/sbin/xcatd b/xCAT-server/sbin/xcatd index 7af459ff2..7279fcab0 100755 --- a/xCAT-server/sbin/xcatd +++ b/xCAT-server/sbin/xcatd @@ -30,85 +30,16 @@ my $udpctl; my $pid_UDP; my $pid_MON; -# BEGIN mon-respawn-policy -# When the install monitor dies, the main service loop below re-forks it. This decides -# WHEN. It is deliberately free of forking and of daemon state so that -# xCAT-test/unit/xcatd_monitor_respawn.t can extract this region verbatim and drive it. -# -# Respawning has to be paced. do_installm_service dies when it cannot bind xcatiport, so -# while something else holds that port every respawn is a fast, futile fork that also -# re-enters that function's USR2 socket-takeover handshake against whatever holds it. -# -# But pacing must never become giving up. A retry budget that runs out cannot be refilled: -# with no monitor alive, nothing is left to reset it. The port would stay dead until xcatd -# is restarted -- which is exactly the failure this respawn exists to remove, just reached -# more slowly -- and a port that frees up a minute later would never be picked back up. -# -# So the delay doubles from $mon_respawn_min to $mon_respawn_max and then stays there. -# A monitor that cannot start costs one fork per $mon_respawn_max seconds for as long as -# that lasts, and comes back within that bound once the port is free again. -# -# A monitor that survived $mon_respawn_healthy seconds plainly got the socket and served, -# so its eventual death resets the delay: an isolated death is retried at once, and the -# backoff only builds up during a real streak of failures to start. -my $mon_respawn_min = defined($ENV{XCATD_MON_RESPAWN_MIN_INTERVAL}) ? $ENV{XCATD_MON_RESPAWN_MIN_INTERVAL} : 5; -my $mon_respawn_max = defined($ENV{XCATD_MON_RESPAWN_MAX_INTERVAL}) ? $ENV{XCATD_MON_RESPAWN_MAX_INTERVAL} : 300; -my $mon_respawn_healthy = defined($ENV{XCATD_MON_RESPAWN_HEALTHY}) ? $ENV{XCATD_MON_RESPAWN_HEALTHY} : 60; -$mon_respawn_min = 1 if $mon_respawn_min < 1; # 0 would be a fork storm -$mon_respawn_max = $mon_respawn_min if $mon_respawn_max < $mon_respawn_min; - -my $mon_respawn_delay = $mon_respawn_min; # delay to apply after the NEXT failure -my $mon_respawn_next = 0; # earliest time() at which to try again -my $mon_respawn_started; # time() the running monitor was forked -my $mon_respawn_streak = 0; # consecutive monitors that died young -my $mon_respawn_capped = 0; # has this streak already reported the ceiling? - -# Is a respawn allowed yet? This only ever answers "not yet" -- never "no more". -sub mon_respawn_due { - my ($now) = @_; - return $now >= $mon_respawn_next; -} - -# Note that a monitor is being forked. Called BEFORE the fork: the child can die, and be -# reaped, before xfork even returns to the parent. -sub mon_respawn_forked { - my ($now) = @_; - $mon_respawn_started = $now; - return; -} - -# Note that the monitor exited, and schedule the next attempt. Returns the number of -# consecutive monitors that have died young (0 once one of them managed to serve). -# Safe to call from the SIGCHLD handler: arithmetic only, no I/O. -sub mon_respawn_exited { - my ($now) = @_; - if (defined($mon_respawn_started) and ($now - $mon_respawn_started) >= $mon_respawn_healthy) { - $mon_respawn_delay = $mon_respawn_min; # it served; this is a fresh start - $mon_respawn_next = $now; - $mon_respawn_streak = 0; - $mon_respawn_capped = 0; - } else { - $mon_respawn_streak++; - $mon_respawn_next = $now + $mon_respawn_delay; - $mon_respawn_delay = ($mon_respawn_delay * 2 > $mon_respawn_max) - ? $mon_respawn_max - : $mon_respawn_delay * 2; - } - $mon_respawn_started = undef; - return $mon_respawn_streak; -} - -# True once per failure streak, when the backoff first reaches its ceiling. So a monitor -# that cannot start is reported once rather than on every attempt, and is reported afresh -# if it starts failing again after having served. -sub mon_respawn_hit_ceiling { - return 0 if $mon_respawn_capped; - return 0 if $mon_respawn_streak < 1; - return 0 if $mon_respawn_delay < $mon_respawn_max; - $mon_respawn_capped = 1; - return 1; -} -# END mon-respawn-policy +# Pacing for re-forking the install monitor; see the main service loop below. The pacing +# itself is pure arithmetic in xCAT::RespawnUtils -- a backoff with a ceiling but no end, +# so a held xcatiport costs one fork per ceiling interval instead of a fork storm, and the +# monitor is back within that bound once the port is free. Each function returns a NEW +# state, which is why the reaper can safely install one from inside the signal handler. +my $mon_respawn = xCAT::RespawnUtils::policy( + min_interval => $ENV{XCATD_MON_RESPAWN_MIN_INTERVAL}, + max_interval => $ENV{XCATD_MON_RESPAWN_MAX_INTERVAL}, + healthy => $ENV{XCATD_MON_RESPAWN_HEALTHY}, +); my $numofnodes=0; @@ -130,6 +61,7 @@ use xCAT::TLSPolicy qw(resolve_xcatd_tls_settings); use xCAT::TableUtils; use xCAT::NetworkUtils; use xCAT::MsgUtils; +use xCAT::RespawnUtils; use xCAT::xcatd; use xCAT::CmdLog; use xCAT::State; @@ -1144,7 +1076,7 @@ sub ssl_reaper { } if ($CHILDPID == $pid_MON) { $pid_MON = 0; - mon_respawn_exited(time()); # paces the re-fork the main service loop will do + $mon_respawn = xCAT::RespawnUtils::exited($mon_respawn, time()); } } $SIG{CHLD} = \&ssl_reaper; @@ -1270,7 +1202,8 @@ if (!(socketpair($rescanreadpipe, $rescanwritepipe, AF_UNIX, SOCK_STREAM, PF_UNS } $rescanrselect = new IO::Select; $rescanrselect->add($rescanreadpipe); -mon_respawn_forked(time()); # so this monitor's uptime counts towards the respawn policy too +# record this monitor too, so its uptime counts when it eventually dies +$mon_respawn = xCAT::RespawnUtils::forked($mon_respawn, time()); $pid_MON = xCAT::Utils->xfork; if (!defined $pid_MON) { xCAT::MsgUtils->message("S", "Unable to fork installmonitor"); @@ -1551,21 +1484,22 @@ until ($quit) { # leave xcatiport permanently dead while this daemon kept running, so installing nodes could # no longer report status or request the boot flip until the WHOLE daemon was restarted. # - # The mon-respawn-policy region near the top of this file paces the attempts, so a port that - # stays held costs one fork per ceiling interval instead of a fork storm. It never stops - # saying yes, so the monitor also comes back on its own once whatever held the port lets go. - if (!$pid_MON && !$quit && $sport && mon_respawn_due(time())) { - if (mon_respawn_hit_ceiling()) { + # xCAT::RespawnUtils paces the attempts, and never stops allowing them, so the monitor also + # comes back on its own once whatever held the port lets go of it. + if (!$pid_MON && !$quit && $sport && xCAT::RespawnUtils::due($mon_respawn, time())) { + if (xCAT::RespawnUtils::should_report($mon_respawn)) { xCAT::MsgUtils->message("S", - "xcatd: install monitor is not staying up (${mon_respawn_streak} attempts);" - . " still retrying xcatiport $sport every $mon_respawn_max seconds"); + "xcatd: install monitor is not staying up ($mon_respawn->{streak} attempts);" + . " still retrying xcatiport $sport every $mon_respawn->{max_interval} seconds"); + $mon_respawn = xCAT::RespawnUtils::reported($mon_respawn); } - mon_respawn_forked(time()); + $mon_respawn = xCAT::RespawnUtils::forked($mon_respawn, time()); $pid_MON = xCAT::Utils->xfork; if (!defined $pid_MON) { xCAT::MsgUtils->message("S", "xcatd: unable to re-fork install monitor"); $pid_MON = 0; - mon_respawn_exited(time()); # count the failed fork and back off before retrying + # count the failed fork and back off before retrying + $mon_respawn = xCAT::RespawnUtils::exited($mon_respawn, time()); } elsif (!$pid_MON) { # child: serve only the install monitor $$progname = "xcatd: install monitor"; $pid_UDP = 0;