diff --git a/perl-xCAT/xCAT/RespawnUtils.pm b/perl-xCAT/xCAT/RespawnUtils.pm index 43a8f73ae..bd432228c 100644 --- a/perl-xCAT/xCAT/RespawnUtils.pm +++ b/perl-xCAT/xCAT/RespawnUtils.pm @@ -1,51 +1,38 @@ # IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html package xCAT::RespawnUtils; -# Pacing for a parent that has to keep a child alive; xcatd's install monitor is the caller. +# Backoff for a parent that re-forks a child when it dies; xcatd's install monitor is the +# caller. The delay doubles from min_interval to max_interval and holds there. # -# The child is re-forked whenever it dies, but not as fast as fork() returns -- it may be -# dying because a resource it needs is held by someone else, and retrying flat out burns CPU -# and interferes with whatever handshake it performs to claim that resource. So attempts back -# off, doubling from min_interval to max_interval and then holding there. +# There is no attempt limit. A spent budget cannot be refilled: with no child alive nothing +# resets it, and the resource stays dead until xcatd restarts. A child that ran `healthy` +# seconds served, so its death resets the delay. # -# It never stops retrying. A retry budget that runs out cannot be refilled, because with no -# child alive nothing is left to reset it, so the resource would stay unserved until the whole -# daemon is restarted -- the failure the respawn exists to prevent. A child that stayed up -# `healthy` seconds evidently did claim its resource and serve, so its death resets the delay -# and only a real streak of failures to start builds the backoff up. +# Every sub returns a new state and leaves its argument alone, so exited() can run in a +# SIGCHLD handler. # -# Every function returns a NEW state and never mutates the one it is handed. That is what -# makes them safe to call from a SIGCHLD handler: the result is complete before the caller's -# assignment installs it, so a signal cannot catch the pacing half-written. +# The state is a plain hash; call the subs below for the next state. # -# The state is a plain hash. Callers may read these; use the functions below to get the next -# state rather than writing to them. -# -# min_interval shortest wait between attempts, and what a healthy run resets the delay to -# max_interval longest wait -- the delay doubles up to this and then stays here -# healthy how long a child must survive before we count it as having served -# delay how long to wait after the NEXT failure +# min_interval shortest wait, and what a healthy run resets the delay to +# max_interval longest wait; the delay doubles up to this +# healthy seconds a child must survive to count as having served +# delay the wait after the next failure # next_at earliest time() at which another attempt is allowed -# started_at when the running child was forked, or undef when none is running -# streak how many children in a row have died young -# reported whether we have already logged that this streak reached the ceiling +# started_at when the running child was forked, undef when none is running +# streak children in a row that died young +# reported whether the ceiling was already logged for this streak use strict; use warnings; -# Everything up to the "Forking" section below is pure arithmetic: it reads only the state it -# is handed, returns a new one, and touches no clock, no globals and no processes. Keep it -# that way -- that is what lets the pacing be tested on a made-up clock instead of in real -# seconds, and what makes exited() safe to call from a signal handler. - -# Read one tunable, falling back to the default unless it really looks like a whole number. +# The tunables reach policy() straight from %ENV, so they can be empty or misspelt. Anything +# that is not a whole number is treated as unset. sub _tunable { my ($value, $default) = @_; return $default unless defined($value) && $value =~ /^\s*\d+\s*$/; return $value + 0; } -# Start pacing a child from scratch. Anything the caller leaves out gets a sensible default. sub policy { my (%opt) = @_; @@ -53,7 +40,7 @@ sub policy { my $max = _tunable($opt{max_interval}, 300); my $healthy = _tunable($opt{healthy}, 60); - $min = 1 if $min < 1; + $min = 1 if $min < 1; # 0 doubles to 0, which is a fork storm $max = $min if $max < $min; return { @@ -68,21 +55,17 @@ sub policy { }; } -# Is it time to try again yet? This can say "not yet", but it never says "no more". sub due { my ($state, $now) = @_; return $now >= $state->{next_at} ? 1 : 0; } -# Note that we are about to fork, so we can tell later how long the child lasted. Call this -# before forking: the child can die and be reaped before fork() even returns to us. +# Call before forking: the child can die and be reaped before fork() returns to the parent. sub forked { my ($state, $now) = @_; return { %$state, started_at => $now }; } -# Note that the child died, and decide when to try again -- straight away if it had been up -# long enough to have served, later and later if it keeps failing to start. sub exited { my ($state, $now) = @_; @@ -106,8 +89,7 @@ sub exited { return \%next; } -# Has this run of failures just hit the ceiling, and not been mentioned yet? Keeps the log to -# one line per streak instead of one per attempt. +# Keeps the log to one line per streak rather than one per attempt. sub should_report { my ($state) = @_; return 0 if $state->{reported}; @@ -115,44 +97,35 @@ sub should_report { return $state->{delay} >= $state->{max_interval} ? 1 : 0; } -# Remember that we have already logged the ceiling for this streak. sub reported { my ($state) = @_; return { %$state, reported => 1 }; } # --- Forking ----------------------------------------------------------------------------- -# The one impure sub. Everything above only does arithmetic; this actually forks. +# The one impure sub. Everything above is arithmetic. -# Fork a child and keep the pacing straight while doing it. Takes the child's body as a -# block, then `state` and `pid` -- REFERENCES to the caller's own variables -- and `now`: +# Fork a child, taking its body as a block: # # xCAT::RespawnUtils::supervise { ...child... } # state => \$state, pid => \$pid, now => time(); # -# The (&@) prototype is what allows the leading block. It needs this module loaded with -# `use`, not `require`: under `require` the sub is unknown when the call is compiled, the -# block is then read as a bare block, and its value arrives as the first argument. +# `state` and `pid` are references to the caller's own variables. The reaper matches the dead +# child against that pid and folds the death into that state, so both have to be in place +# while SIGCHLD is still blocked; values assigned from a return would land after it is let +# back in, and a child dying in the gap would be compared against a pid still holding 0. # -# Two orderings in here are easy to get wrong and are the reason this is not left to callers. -# The attempt is recorded before the fork, because the child can die and be reaped before -# fork() returns to us. And the pid and the state are installed in the caller's variables -# while SIGCHLD is still blocked -- which is why they are passed by reference rather than -# handed back as a return value. The reaper matches the dead child against that pid and folds -# the death into that state; had the caller assigned them from a return value, the assignment -# would land after the signal was let back in, so a child dying in the gap would be compared -# against a pid still holding 0, missed, and the caller would then write a dead pid back over -# the reaper's work -- believing a dead child alive, and never respawning it. +# The (&@) prototype needs this module loaded with `use`. Under `require` the sub is unknown +# when the call is compiled, the block is read as a bare block, and its value arrives as the +# first argument. # -# The block is only ever entered in the child and is not expected to return; if it does, the -# child exits quietly rather than falling back into the parent's code. Passing a live `pid` -# is a no-op, so a caller that forgets to check is not punished with a second child. The new -# pid is also returned, for a caller that wants it inline. +# The block runs only in the child and is not expected to return. Passing a live `pid` is a +# no-op. Returns the new pid. sub supervise (&@) { my ($child, %arg) = @_; my ($stateref, $pidref, $now) = @arg{qw(state pid now)}; - return $$pidref if $$pidref; # already running; nothing to do + return $$pidref if $$pidref; require POSIX; require xCAT::Utils; @@ -170,13 +143,13 @@ sub supervise (&@) { return 0; } - unless ($pid) { # child: it must not go on to serve with SIGCHLD blocked + unless ($pid) { # the child must not serve with SIGCHLD blocked POSIX::sigprocmask(POSIX::SIG_UNBLOCK(), $mask); $child->(); POSIX::_exit(0); } - $$pidref = $pid; # in place before the reaper can run, or it matches a stale pid + $$pidref = $pid; # in place before the reaper runs, or it matches a stale pid POSIX::sigprocmask(POSIX::SIG_UNBLOCK(), $mask); return $pid; } diff --git a/xCAT-server/sbin/xcatd b/xCAT-server/sbin/xcatd index 669a8a67e..c8d08d887 100755 --- a/xCAT-server/sbin/xcatd +++ b/xCAT-server/sbin/xcatd @@ -55,11 +55,7 @@ use xCAT::xcatd; use xCAT::CmdLog; use xCAT::State; -# Pacing for re-forking the install monitor; see the main service loop below. The pacing -# itself is pure arithmetic in xCAT::RespawnUtils -- a backoff with a ceiling but no end, -# so a held xcatiport costs one fork per ceiling interval instead of a fork storm, and the -# monitor is back within that bound once the port is free. Each function returns a NEW -# state, which is why the reaper can safely install one from inside the signal handler. +# Pacing for re-forking the install monitor; see the main service loop below. my $mon_respawn = xCAT::RespawnUtils::policy( min_interval => $ENV{XCATD_MON_RESPAWN_MIN_INTERVAL}, max_interval => $ENV{XCATD_MON_RESPAWN_MAX_INTERVAL}, @@ -1035,10 +1031,9 @@ wait_db_process(); my $CHILDPID = 0; # Global for reapers my %immediatechildren; -# Fold the death of the install monitor into the respawn pacing. Every reaper that can be -# the handler when it dies has to call this: generic_reaper covers startup and the throttled -# path, ssl_reaper the rest of the service loop. A death reaped without this leaves $pid_MON -# holding a dead pid, and the service loop only re-forks when $pid_MON is clear. +# Both reapers call this. generic_reaper is the handler at startup and while connections are +# throttled, ssl_reaper for the rest of the service loop. A death reaped without this leaves +# $pid_MON holding a dead pid, and the service loop re-forks only when $pid_MON is clear. sub reap_install_monitor { my ($pid) = @_; @@ -1214,10 +1209,6 @@ if (!(socketpair($rescanreadpipe, $rescanwritepipe, AF_UNIX, SOCK_STREAM, PF_UNS } $rescanrselect = new IO::Select; $rescanrselect->add($rescanreadpipe); -# supervise() records the attempt, blocks SIGCHLD across the fork, and installs $pid_MON and -# $mon_respawn itself while it is still blocked -- which is why they are passed by reference; -# see the sub. This monitor's uptime counts towards the pacing too, so an unrelated death much -# later is retried promptly. xCAT::RespawnUtils::supervise { $$progname = "xcatd: install monitor"; $pid_UDP = 0; @@ -1492,14 +1483,9 @@ my $udpalive = 1; until ($quit) { $SIG{CHLD} = \&ssl_reaper; # set here to ensure that signal handler is not corrupted during loop - # Respawn the install monitor if it has died. It is forked exactly once at startup, and the - # SIGCHLD reaper only clears $pid_MON when it exits -- nothing re-forks it. A single death - # of that child (a stray signal, or a lost socket takeover during an xcatd restart) used to - # leave xcatiport permanently dead while this daemon kept running, so installing nodes could - # no longer report status or request the boot flip until the WHOLE daemon was restarted. - # - # xCAT::RespawnUtils paces the attempts, and never stops allowing them, so the monitor also - # comes back on its own once whatever held the port lets go of it. + # Nothing else re-forks the monitor; the reaper only clears $pid_MON. The pacing never + # stops allowing an attempt, so the monitor also comes back once whatever holds + # xcatiport lets go of it. if (!$pid_MON && !$quit && $sport && xCAT::RespawnUtils::due($mon_respawn, time())) { if (xCAT::RespawnUtils::should_report($mon_respawn)) { xCAT::MsgUtils->message("S", @@ -1512,16 +1498,12 @@ until ($quit) { $pid_UDP = 0; close($listener); close($udpctl); $udpctl = 0; - # This fork happens further down the program than the one at startup, so it also - # inherits what the parent has opened since: the rescanplugins channel. The monitor - # has no use for either end, and holding them is the only way a respawned monitor - # would differ from the one forked at startup. + # This fork is further down the program than the one at startup, so it also + # inherits what the parent opened since: the rescanplugins channel. close($chreadpipe); close($chwritepipe); - # A pending connection is a client socket the parent has accepted and not yet - # handed to a worker. The monitor never serves one, and it outlives the worker - # that does, so a copy left open here holds that client's socket for the life of - # the daemon. + # Client sockets the parent accepted and has not dispatched yet. The monitor + # outlives the worker that serves one. close($_) for @pendingconnections; do_installm_service; xexit(0); @@ -1558,9 +1540,8 @@ until ($quit) { } else { # if select returned with no ready fds, there might be udpctl broken. - # While the install monitor is down, wait in shorter hops: the respawn at the top of - # this loop only gets a turn when this select returns, so on an otherwise idle daemon - # a full 30s wait is added to the respawn delay before xcatiport comes back. + # The respawn at the top of this loop only gets a turn when this select returns, so + # wait in shorter hops while the monitor is down. if (not $bothwatcher->can_read((!$pid_MON && $sport) ? 5 : 30)) { # if the errno is 'bad fd', check the health of the udpctl