From 4acaa5b3d86a9d6ce843ca44e5820f80835606be Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Thu, 22 Oct 2026 10:20:01 -0300 Subject: [PATCH] fix(xcat-core): every build lock failure is reported as a concurrent build acquire_lock_dir dies with "another build of already holds " whenever XCAT::NFSLock->acquire fails, whatever the failure was. A lockdir under a directory that does not exist, or one the build cannot write, both tell the operator to look for a build that is not running, and both print the advice about --mock-uniqueext. The nested text said "Trying to unlock", which is the reverse of what acquire was doing. acquire_lock_dir now reports contention only when the retry budget ran out, and prints the error that happened otherwise. The vendored XCAT::NFSLock is resynced with xcat-dep: its refusal names which failure it was, and it reclaims a lock.borrow that has no metadata and that two attempts find unchanged, which was a permanent strand. A lock.d with no metadata stays refused, so a directory left by a build from before this change is not recovered by acquire; sweep_stranded removes that one, and only under an exclusive claim on the root. buildrpms.pl kept "use POSIX (); # EEXIST, for the directory build lock", and the lock has not used EEXIST since it became an XCAT::NFSLock. The import is still needed for WNOHANG in abort_builds. Depends on xcat-dep/#93, which carries the same change to the original copy of XCAT/NFSLock.pm. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- build-utils/lib/XCAT/BuildUtils.pm | 19 +++-- build-utils/lib/XCAT/NFSLock.pm | 118 +++++++++++++++++++++-------- buildrpms.pl | 2 +- 3 files changed, 102 insertions(+), 37 deletions(-) diff --git a/build-utils/lib/XCAT/BuildUtils.pm b/build-utils/lib/XCAT/BuildUtils.pm index 69252f9737..10172d37d1 100644 --- a/build-utils/lib/XCAT/BuildUtils.pm +++ b/build-utils/lib/XCAT/BuildUtils.pm @@ -658,8 +658,8 @@ sub lock_path_for { # and one target. A lock on the shared checkout would serialise the per-arch builds of one # checkout that run on two hosts, which share nothing. # -# $held is the text appended when a live build holds the lock. $what names what contends, so the -# two callers keep their own wording -- a checkout for builddebs.pl, a mock target for +# $held is the advice printed when another build still holds the lock. $what names what contends, +# so the two callers keep their own wording -- a checkout for builddebs.pl, a mock target for # buildrpms.pl -- while the protocol lives in one place. sub acquire_lock_dir { my ($lockdir, $what, $held) = @_; @@ -667,15 +667,24 @@ sub acquire_lock_dir { $what = $lockdir unless defined $what; $held = '' unless defined $held; - # Two attempts, because recovering a stranded borrow takes one turn and the take-over the - # next. Builds must fail fast rather than queue, so the budget stops there. + # Three attempts: reaping a stranded borrow takes two, because a borrow with no metadata is + # reclaimed only when two of them find it unchanged, and the take-over takes the third. + # Builds must fail fast rather than queue, so the budget stops there. my $lock = eval { XCAT::NFSLock->acquire($lockdir, retries => 2, label => $what, quiet => 1); }; unless ($lock) { my $why = $@ // ''; $why =~ s/\s+\z//; - die "FATAL: another build of $what already holds $lockdir\n$held$why\n"; + $why =~ s/^/ /mg; + # NFSLock says which failure it was. Only an exhausted retry budget means something else + # holds the lock; a mkdir that failed outright is a different problem, and every failure + # used to be reported as a concurrent build. + my $contended = $why =~ /\bstill held after [0-9]+ retr/; + die "FATAL: cannot take the build lock for $what\n" + . ($contended ? " another build still holds $lockdir\n" : '') + . "$why\n" + . ($contended ? $held : ''); } push @LIVE_LOCKS, $lock; # Registered weakly, so holding it here does not keep the lock alive past its caller's diff --git a/build-utils/lib/XCAT/NFSLock.pm b/build-utils/lib/XCAT/NFSLock.pm index 6bb7629456..e55d4dad30 100644 --- a/build-utils/lib/XCAT/NFSLock.pm +++ b/build-utils/lib/XCAT/NFSLock.pm @@ -4,11 +4,13 @@ package XCAT::NFSLock; # # Copies # This module is vendored three times, because no repo installs another's build tooling: -# xcat-dep lib/XCAT/NFSLock.pm (this file, the original) +# xcat-dep lib/XCAT/NFSLock.pm (the original) # xcat-core build-utils/lib/XCAT/NFSLock.pm # xcat-core-ci-cd ci/lib/XCAT/NFSLock.pm # The three must stay identical. They have diverged once already: sweep_stranded was added to -# the ci-cd copy alone, so the protocol hole it covers stayed open in the other two. +# the ci-cd copy alone, so the protocol hole it covers stayed open in the other two. Its only +# caller is still in xcat-core-ci-cd (ci/sweep-stranded-locks.pl); it stays in all three copies +# because a copy that drops it reopens that hole. # # A lock on a shared, possibly re-exported, NFS tree. flock and fcntl are not # available there. The protocol uses mkdir, rmdir, unlink and plain file @@ -27,10 +29,15 @@ package XCAT::NFSLock; # Cloned VMs and images can share it by accident unless it is regenerated. # 3. Metadata writes eventually become readable completely and consistently. # 4. A host never declares one of its own live process incarnations dead. -# 5. A process cannot die after it creates lock.d, until it publishes valid metadata. -# It MAY die while it holds lock.borrow: the borrow carries the same metadata as the lock, -# so a borrow whose taker is provably dead is recovered by the same rule. A borrow left by -# the earlier protocol carries none, and sweep_stranded removes those. +# 5. A process MAY die at any point, lock.borrow included. The borrow carries the same +# metadata as the lock, so a borrow whose taker is provably dead is recovered by the same +# rule. A borrow with NO metadata is left by the earlier protocol, or by a death between +# mkdir and the first write; no version of this protocol holds one for more than +# milliseconds, so one that two attempts find unchanged is held by nobody. +# A lock.d with no metadata is a different case and acquire never recovers it. The build +# lock in xcat-core wrote no metadata and held lock.d for the whole build, so no fact on +# disk tells that holder from a dead one. Only sweep_stranded removes those, and only when +# the caller claims the root exclusively. # 6. A crashed worker leaves recoverable state. The owning host eventually # returns and retries. Eventually one worker and its release complete. # 7. All participants follow the protocol. @@ -47,6 +54,10 @@ package XCAT::NFSLock; # match, assume a partial or inconsistent read and retry. Never infer stale # ownership from invalid metadata. # +# One case is not a partial read: a lock.borrow with no metadata FILE, unchanged over two +# attempts. Two attempts of one acquire or release are at least T seconds apart, and no +# participant holds a metadata-less borrow that long, so nobody holds it. +# # Retry # R = max retries, T = base delay, δ = jitter # R > 0, δ >= 0, T >= 3, T > 2δ @@ -67,7 +78,8 @@ package XCAT::NFSLock; # - different machine-id: retry # - same host: save the observed ownership identity # 3. mkdir lock.borrow, then write valid metadata into it -# - EEXIST: recover the borrow if its metadata proves its taker dead, then retry +# - EEXIST: recover the borrow if its metadata proves its taker dead, or if it has no +# metadata file and two attempts found it unchanged; then retry # - other failure: retry # 4. Read and validate the metadata again. # - invalid, missing, or ownership identity changed: rmdir lock.borrow, retry @@ -78,7 +90,7 @@ package XCAT::NFSLock; # 7. rmdir lock.borrow # 8. Call the worker. # 9. mkdir lock.borrow, then write valid metadata into it -# - EEXIST: recover the borrow if its metadata proves its taker dead +# - EEXIST: recover the borrow by the same rule as step 3 # - failure: sleep(T + rand(-δ, δ)), retry step 9 # 10. unlink lock.d/metadata # 11. rmdir lock.d (the actual unlock) @@ -89,8 +101,9 @@ package XCAT::NFSLock; # lock.d absent => acquirable # invalid metadata => retry only # different machine-id => never recover here -# lock.borrow exists => ownership transition or release in progress, -# unless its metadata proves its taker dead +# lock.borrow exists => ownership transition or release in progress, unless its metadata +# proves its taker dead, or it has no metadata file and two attempts +# found it unchanged # # Steps 1 to 7 are acquire, step 8 is the caller, steps 9 to 12 are release. # release does steps 10 and 11 only when the metadata names this acquisition. @@ -175,6 +188,9 @@ sub acquire { }, $class; my $here = _here(); my $seen; + # Directories this acquire has already observed, for _unchanged. The attempts of one acquire + # are one delay apart, which is what makes "unchanged" mean "untouched for T seconds". + my %observed_before; for (my $count = 0 ; ; $count++) { # Step 1. @@ -219,8 +235,8 @@ sub acquire { elsif ($! == EEXIST) { # Step 3 fails for as long as the borrow exists, and that skips steps 4 to 7, so # a taker killed here blocked the lock for every later acquire. - $self->_log('reaped-borrow', "taker pid $1 is dead") - if _reap_borrow($borrow, $here) =~ /\Apid (\d+)\z/; + my ($reaped, $why) = _reap_borrow($borrow, $here, \%observed_before); + $self->_log('reaped-borrow', $why) if $reaped; } } @@ -231,7 +247,9 @@ sub acquire { my $who = _describe($seen); my $s = $retries == 1 ? 'retry' : 'retries'; - die "Trying to unlock $abs failed after $retries $s; $label owned by $who.\n" + # Names what happened, so a caller can tell a lock another process still holds from a mkdir + # that failed outright. The earlier wording said "Trying to unlock", which is the reverse. + die "Cannot lock $abs: still held after $retries $s by $who.\n" . "If you are sure it is safe, remove the lock: rm -rf $abs\n"; } @@ -259,6 +277,7 @@ sub release { return 0 unless defined($self->{identity}); my $path = $self->{path}; my $borrow = "$path.borrow"; + my %observed_before; while (1) { # Step 9. if (mkdir($borrow)) { @@ -288,7 +307,7 @@ sub release { } else { # A taker killed while holding the borrow would otherwise block this release for ever. - _reap_borrow($borrow, _here()); + _reap_borrow($borrow, _here(), \%observed_before); } _sleep(_wait($self->{delay}, $self->{jitter})); } @@ -455,21 +474,43 @@ sub _drop_borrow { return rmdir($borrow); } -# _reap_borrow: remove a borrow whose taker is provably dead, and say what was found. +# _reap_borrow: remove a borrow no live taker holds, and say what was found. # -# The same rule the lock itself uses: a borrow is recovered only when its metadata is valid, names -# this machine, and names a process this host can prove is dead. A borrow with no metadata is one -# the previous protocol left, and staleness is never inferred from metadata that cannot be -# validated -- sweep_stranded removes those. +# Two rules. A borrow with valid metadata is recovered when that metadata names this machine and a +# process this host can prove dead -- the rule the lock itself uses. A borrow with no metadata +# FILE is recovered when two attempts find it unchanged: the earlier protocol wrote none, and no +# version of this one holds a metadata-less borrow for more than milliseconds. A borrow whose +# metadata file exists but does not validate may be a partial read, so it is left alone. +# +# $seen carries the observations across the attempts of one acquire or release. +# Returns ($removed, $why). sub _reap_borrow { - my ($borrow, $here) = @_; + my ($borrow, $here, $seen) = @_; $here //= _here(); my $taker = _read_metadata($borrow); - return 'invalid' unless defined($taker); - return 'other machine' unless $taker->{'machine-id'} eq $here->{'machine-id'}; - return 'alive' unless owner_is_dead($taker, $here); + unless (defined($taker)) { + return (0, 'metadata invalid') if -e "$borrow/metadata"; + return (0, 'no metadata, first attempt') unless _unchanged($seen, $borrow); + _drop_borrow($borrow); + return (1, 'no metadata, and two attempts found the borrow unchanged'); + } + return (0, 'other machine') unless $taker->{'machine-id'} eq $here->{'machine-id'}; + return (0, 'taker is alive') unless owner_is_dead($taker, $here); _drop_borrow($borrow); - return "pid $taker->{pid}"; + return (1, "taker pid $taker->{pid} is dead"); +} + +# _unchanged: true when $dir carries the identity and mtime of an earlier observation in $seen. +# Creating or removing a file in a directory moves its mtime, so an unchanged directory had +# nothing written in it between the two observations. +sub _unchanged { + my ($seen, $dir) = @_; + return 0 unless defined($seen); + my @stat = stat($dir) or return 0; + my $mark = join(':', @stat[0, 1, 9]); + my $before = $seen->{$dir}; + $seen->{$dir} = $mark; + return defined($before) && $before eq $mark ? 1 : 0; } sub _key { @@ -591,16 +632,22 @@ sub DESTROY { A dead owner's lock.d is removed as well. acquire would take it over, so this only saves the wait. - It never infers staleness from metadata it cannot validate, and it never removes a lock - owned by a live process or by another machine. + It never removes a lock owned by a live process or by another machine, and it never + infers staleness from a metadata file it cannot validate. + + A lock.d with NO metadata file is removed only with exclusive => 1. acquire refuses one, + because the build lock in xcat-core wrote no metadata and held lock.d for the whole + build, so no fact on disk tells that holder from a dead one. Only a caller that claims + the root exclusively can say it is abandoned. Arguments: $root: the directory to walk $here: hash ref with machine-id, boot-id and start_of, as owner_is_dead takes %opt: exclusive => 1 when the caller holds the run's exclusive claim on $root, so no - live take-over can be in progress. Only then is a borrow removed: a borrow is - otherwise a transition that is milliseconds from finishing, and removing it lets - two takers rewrite one metadata file. + build and no take-over can be in progress. Only then is a borrow removed, and + only then a lock.d with no metadata: a borrow is otherwise a transition that is + milliseconds from finishing, and removing it lets two takers rewrite one + metadata file. Returns: A list of hash refs: path, action (removed or kept), why. @@ -637,8 +684,17 @@ sub sweep_stranded { } my $owner = _read_metadata($d); unless (defined($owner)) { - push @report, { path => $d, action => 'kept', - why => 'metadata missing or invalid; staleness is never inferred' }; + if (-e "$d/metadata") { + push @report, { path => $d, action => 'kept', + why => 'metadata invalid; staleness is never inferred' }; + next; + } + unless ($opt{exclusive}) { + push @report, { path => $d, action => 'kept', + why => 'no metadata; no exclusive claim on the root' }; + next; + } + push @report, _remove($d, 'no metadata, and the caller claims the root exclusively'); next; } if ($owner->{'machine-id'} ne $here->{'machine-id'}) { diff --git a/buildrpms.pl b/buildrpms.pl index d403733be2..255f494258 100755 --- a/buildrpms.pl +++ b/buildrpms.pl @@ -61,7 +61,7 @@ use XCAT::BuildUtils qw(git_revision source_date_epoch sh sh_or_die usage buildi write_script read_line targetarch_from_target openeuler_build_target openeuler_repo_subdir mock_config_text acquire_lock_dir); -use POSIX (); # EEXIST, for the directory build lock (see main()) +use POSIX (); # WNOHANG, for the non-blocking waitpid in abort_builds() use Getopt::Long qw(GetOptions); use POSIX qw(strftime); use Parallel::ForkManager;