From f22aed308acda3e4ed95d501c3001a20789f2ff4 Mon Sep 17 00:00:00 2001
From: Daniel Hilst <392820+dhilst@users.noreply.github.com>
Date: Tue, 25 Aug 2026 18:06:00 -0300
Subject: [PATCH] fix(xcat-core): xcatd stops respawning the install monitor
 and never resumes

The respawn of the install monitor was paced by a retry budget that, once spent, made
the daemon stop trying for good. That put xcatiport back in the state the respawn was
added to fix: with no monitor alive there is nothing left to reset the counter, so the
port stays dead until the whole daemon is restarted, and a port that frees up a minute
later is never picked back up. It only reached that state more slowly than before.

Pacing itself is needed. do_installm_service dies when it cannot bind the port, so an
unguarded re-fork spins as fast as fork allows while something else holds it, and keeps
re-entering that function's USR2 socket-takeover handshake. Replace the budget with an
exponential backoff that has a ceiling but no end: the delay doubles from
XCATD_MON_RESPAWN_MIN_INTERVAL (default 5s) to XCATD_MON_RESPAWN_MAX_INTERVAL (default
300s) and stays there. A monitor that cannot start therefore costs one fork per five
minutes for as long as that lasts, and is back within five minutes of the port becoming
free, with no restart and no operator action.

A monitor that ran for XCATD_MON_RESPAWN_HEALTHY seconds (default 60) plainly got the
socket and served, so its eventual death resets the delay: an isolated death is retried
at once and the backoff only builds up during a real streak of failures to start. The
ceiling is reported once per streak rather than on every attempt, and says that xcatd is
still retrying instead of that it has stopped.

The pacing lives in a marked mon-respawn-policy region, free of forking and of daemon
state, so xCAT-test/unit/xcatd_monitor_respawn.t drives the real code rather than a copy
of it.

Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
---
 xCAT-server/sbin/xcatd | 143 ++++++++++++++++++++++++++++++-----------
 1 file changed, 107 insertions(+), 36 deletions(-)

diff --git a/xCAT-server/sbin/xcatd b/xCAT-server/sbin/xcatd
index de34f23d5..7af459ff2 100755
--- a/xCAT-server/sbin/xcatd
+++ b/xCAT-server/sbin/xcatd
@@ -29,11 +29,86 @@ my $sslctl;
 my $udpctl;
 my $pid_UDP;
 my $pid_MON;
-# Respawn accounting for the install monitor (see the main service loop below).
-my $mon_respawn_last     = 0;
-my $mon_respawn_attempts = 0;
-my $mon_respawn_interval = defined($ENV{XCATD_MON_RESPAWN_INTERVAL}) ? $ENV{XCATD_MON_RESPAWN_INTERVAL} : 5;
-my $mon_respawn_max      = defined($ENV{XCATD_MON_RESPAWN_MAX})      ? $ENV{XCATD_MON_RESPAWN_MAX}      : 10;
+
+# BEGIN mon-respawn-policy
+# When the install monitor dies, the main service loop below re-forks it. This decides
+# WHEN. It is deliberately free of forking and of daemon state so that
+# xCAT-test/unit/xcatd_monitor_respawn.t can extract this region verbatim and drive it.
+#
+# Respawning has to be paced. do_installm_service dies when it cannot bind xcatiport, so
+# while something else holds that port every respawn is a fast, futile fork that also
+# re-enters that function's USR2 socket-takeover handshake against whatever holds it.
+#
+# But pacing must never become giving up. A retry budget that runs out cannot be refilled:
+# with no monitor alive, nothing is left to reset it. The port would stay dead until xcatd
+# is restarted -- which is exactly the failure this respawn exists to remove, just reached
+# more slowly -- and a port that frees up a minute later would never be picked back up.
+#
+# So the delay doubles from $mon_respawn_min to $mon_respawn_max and then stays there.
+# A monitor that cannot start costs one fork per $mon_respawn_max seconds for as long as
+# that lasts, and comes back within that bound once the port is free again.
+#
+# A monitor that survived $mon_respawn_healthy seconds plainly got the socket and served,
+# so its eventual death resets the delay: an isolated death is retried at once, and the
+# backoff only builds up during a real streak of failures to start.
+my $mon_respawn_min = defined($ENV{XCATD_MON_RESPAWN_MIN_INTERVAL}) ? $ENV{XCATD_MON_RESPAWN_MIN_INTERVAL} : 5;
+my $mon_respawn_max = defined($ENV{XCATD_MON_RESPAWN_MAX_INTERVAL}) ? $ENV{XCATD_MON_RESPAWN_MAX_INTERVAL} : 300;
+my $mon_respawn_healthy = defined($ENV{XCATD_MON_RESPAWN_HEALTHY}) ? $ENV{XCATD_MON_RESPAWN_HEALTHY} : 60;
+$mon_respawn_min = 1                if $mon_respawn_min < 1;    # 0 would be a fork storm
+$mon_respawn_max = $mon_respawn_min if $mon_respawn_max < $mon_respawn_min;
+
+my $mon_respawn_delay  = $mon_respawn_min;    # delay to apply after the NEXT failure
+my $mon_respawn_next   = 0;                   # earliest time() at which to try again
+my $mon_respawn_started;                      # time() the running monitor was forked
+my $mon_respawn_streak = 0;                   # consecutive monitors that died young
+my $mon_respawn_capped = 0;                   # has this streak already reported the ceiling?
+
+# Is a respawn allowed yet? This only ever answers "not yet" -- never "no more".
+sub mon_respawn_due {
+    my ($now) = @_;
+    return $now >= $mon_respawn_next;
+}
+
+# Note that a monitor is being forked. Called BEFORE the fork: the child can die, and be
+# reaped, before xfork even returns to the parent.
+sub mon_respawn_forked {
+    my ($now) = @_;
+    $mon_respawn_started = $now;
+    return;
+}
+
+# Note that the monitor exited, and schedule the next attempt. Returns the number of
+# consecutive monitors that have died young (0 once one of them managed to serve).
+# Safe to call from the SIGCHLD handler: arithmetic only, no I/O.
+sub mon_respawn_exited {
+    my ($now) = @_;
+    if (defined($mon_respawn_started) and ($now - $mon_respawn_started) >= $mon_respawn_healthy) {
+        $mon_respawn_delay  = $mon_respawn_min;    # it served; this is a fresh start
+        $mon_respawn_next   = $now;
+        $mon_respawn_streak = 0;
+        $mon_respawn_capped = 0;
+    } else {
+        $mon_respawn_streak++;
+        $mon_respawn_next  = $now + $mon_respawn_delay;
+        $mon_respawn_delay = ($mon_respawn_delay * 2 > $mon_respawn_max)
+          ? $mon_respawn_max
+          : $mon_respawn_delay * 2;
+    }
+    $mon_respawn_started = undef;
+    return $mon_respawn_streak;
+}
+
+# True once per failure streak, when the backoff first reaches its ceiling. So a monitor
+# that cannot start is reported once rather than on every attempt, and is reported afresh
+# if it starts failing again after having served.
+sub mon_respawn_hit_ceiling {
+    return 0 if $mon_respawn_capped;
+    return 0 if $mon_respawn_streak < 1;
+    return 0 if $mon_respawn_delay < $mon_respawn_max;
+    $mon_respawn_capped = 1;
+    return 1;
+}
+# END mon-respawn-policy
 
 my $numofnodes=0;
 
@@ -1069,8 +1144,7 @@ sub ssl_reaper {
         }
         if ($CHILDPID == $pid_MON) {
             $pid_MON = 0;
-            # a monitor that ran for a while was healthy; give the next death a full budget
-            $mon_respawn_attempts = 0 if time() - $mon_respawn_last > $mon_respawn_interval * $mon_respawn_max;
+            mon_respawn_exited(time());    # paces the re-fork the main service loop will do
         }
     }
     $SIG{CHLD} = \&ssl_reaper;
@@ -1196,6 +1270,7 @@ if (!(socketpair($rescanreadpipe, $rescanwritepipe, AF_UNIX, SOCK_STREAM, PF_UNS
 }
 $rescanrselect = new IO::Select;
 $rescanrselect->add($rescanreadpipe);
+mon_respawn_forked(time());    # so this monitor's uptime counts towards the respawn policy too
 $pid_MON = xCAT::Utils->xfork;
 if (!defined $pid_MON) {
     xCAT::MsgUtils->message("S", "Unable to fork installmonitor");
@@ -1476,36 +1551,32 @@ until ($quit) {
     # leave xcatiport permanently dead while this daemon kept running, so installing nodes could
     # no longer report status or request the boot flip until the WHOLE daemon was restarted.
     #
-    # Rate limit it. do_installm_service dies if it cannot bind the port after its own retries,
-    # so an unguarded re-fork here would spin as fast as fork allows while the port stays held,
-    # and would keep re-entering that function's USR2 socket-takeover handshake. Space the
-    # attempts, cap them, and say so when the cap is reached rather than retrying forever.
-    if (!$pid_MON && !$quit && $sport && $mon_respawn_attempts < $mon_respawn_max) {
-        if (time() - $mon_respawn_last >= $mon_respawn_interval) {
-            $mon_respawn_last = time();
-            $mon_respawn_attempts++;
-            $pid_MON = xCAT::Utils->xfork;
-            if (!defined $pid_MON) {
-                xCAT::MsgUtils->message("S", "xcatd: unable to re-fork install monitor");
-                $pid_MON = 0;
-            } elsif (!$pid_MON) {    # child: serve only the install monitor
-                $$progname = "xcatd: install monitor";
-                $pid_UDP = 0;
-                close($listener);
-                close($udpctl); $udpctl = 0;
-                do_installm_service;
-                xexit(0);
-            } else {
-                xCAT::MsgUtils->trace(0, "I",
-                    "xcatd: re-forked install monitor (pid $pid_MON) after it exited"
-                      . " (attempt $mon_respawn_attempts of $mon_respawn_max)");
-            }
+    # The mon-respawn-policy region near the top of this file paces the attempts, so a port that
+    # stays held costs one fork per ceiling interval instead of a fork storm. It never stops
+    # saying yes, so the monitor also comes back on its own once whatever held the port lets go.
+    if (!$pid_MON && !$quit && $sport && mon_respawn_due(time())) {
+        if (mon_respawn_hit_ceiling()) {
+            xCAT::MsgUtils->message("S",
+                "xcatd: install monitor is not staying up (${mon_respawn_streak} attempts);"
+                  . " still retrying xcatiport $sport every $mon_respawn_max seconds");
+        }
+        mon_respawn_forked(time());
+        $pid_MON = xCAT::Utils->xfork;
+        if (!defined $pid_MON) {
+            xCAT::MsgUtils->message("S", "xcatd: unable to re-fork install monitor");
+            $pid_MON = 0;
+            mon_respawn_exited(time());    # count the failed fork and back off before retrying
+        } elsif (!$pid_MON) {    # child: serve only the install monitor
+            $$progname = "xcatd: install monitor";
+            $pid_UDP = 0;
+            close($listener);
+            close($udpctl); $udpctl = 0;
+            do_installm_service;
+            xexit(0);
+        } else {
+            xCAT::MsgUtils->trace(0, "I",
+                "xcatd: re-forked install monitor (pid $pid_MON) after it exited");
         }
-    } elsif (!$pid_MON && !$quit && $sport && $mon_respawn_attempts == $mon_respawn_max) {
-        $mon_respawn_attempts++;    # report once, then stop trying
-        xCAT::MsgUtils->message("S",
-            "xcatd: install monitor failed to stay up after $mon_respawn_max attempts;"
-              . " giving up on xcatiport $sport. Restart xcatd once the port is free.");
     }
     while ($udpalive and $udpwatcher->can_read(0)) { # take an intermission to broker some state requests from udp traffic control
         eval {