mirror of
https://github.com/xcat2/xcat-core.git
synced 2026-10-21 15:18:19 +00:00
test(ci): fail a phase whose cases produced no verdict
run_cases counts the Passed and Failed lines a run prints, so a case that crashed before either one was counted nowhere and the phase reported success over cases that never ran. Each phase now knows how many cases it asked for and fails when the verdicts do not add up. A failed query for the case list returned an empty list, which read as a phase with nothing to do; it is a failure now. A backend whose setup failed runs its teardown before the next one starts, rather than leaving the machine configured for a backend that is no longer under test.
This commit is contained in:
Daniel Hilst
@@ -668,8 +668,14 @@ sub write_regression_conf{
|
||||
# Fuction name: list_cases
|
||||
# Description: the names of the cases matching an xcattest label expression
|
||||
# Attributes: $expression - e.g. "ci_test-dhcp_wire"
|
||||
# Return code: the list of case names, empty if none match or the query failed
|
||||
# Return code: the list of case names; empty if none match. A query that
|
||||
# failed returns $QUERY_FAILED instead, because the two must not
|
||||
# be confused: "no case carries this label" is a phase with
|
||||
# nothing to do, and "xcattest could not be asked" is a broken
|
||||
# installation that would otherwise be reported as green.
|
||||
#--------------------------------------------------------
|
||||
our $QUERY_FAILED = "__list_cases_failed__";
|
||||
|
||||
sub list_cases{
|
||||
my $expression = shift;
|
||||
my $cmd = "sudo bash -c '. /etc/profile.d/xcat.sh && xcattest -s \"$expression\" -l'";
|
||||
@@ -677,7 +683,7 @@ sub list_cases{
|
||||
if($::RUNCMD_RC){
|
||||
print RED "[list_cases] $cmd ....[Failed]\n";
|
||||
print Dumper \@cases;
|
||||
return ();
|
||||
return ($QUERY_FAILED);
|
||||
}
|
||||
print "[list_cases] $expression:\n";
|
||||
print Dumper \@cases;
|
||||
@@ -745,6 +751,12 @@ sub run_cases{
|
||||
#--------------------------------------------------------
|
||||
sub run_dhcp_wire_test{
|
||||
my @cases = list_cases("dhcp_wire");
|
||||
if(@cases and $cases[0] eq $QUERY_FAILED){
|
||||
print RED "[run_dhcp_wire_test] the case list could not be read\n";
|
||||
$check_result_str .= "> **DHCP WIRE TEST Failed**: xcattest could not be asked which cases carry the dhcp_wire label\n";
|
||||
print $check_result_str;
|
||||
return 1;
|
||||
}
|
||||
unless(@cases){
|
||||
print "[run_dhcp_wire_test] no DHCP wire cases to run\n";
|
||||
$check_result_str .= "> **DHCP WIRE TEST Skipped**: no case carries the dhcp_wire label\n";
|
||||
@@ -771,6 +783,11 @@ sub run_dhcp_wire_test{
|
||||
}
|
||||
print "[run_dhcp_wire_test] DHCP backends installed here: @backends\n";
|
||||
|
||||
# What has to be accounted for at the end. run_cases counts a case only
|
||||
# when its output carries a verdict line, so a case that hung, was killed or
|
||||
# was truncated is counted nowhere -- and pass+fail would then agree with
|
||||
# itself and report a clean run of however many cases survived.
|
||||
my $expected = 0;
|
||||
my %counts = (pass => 0, fail => 0, failed => []);
|
||||
foreach my $backend (@backends){
|
||||
my $cmd = "sudo bash -c '. /etc/profile.d/xcat.sh && $fixture backend-setup $backend'";
|
||||
@@ -780,9 +797,15 @@ sub run_dhcp_wire_test{
|
||||
print RED "[run_dhcp_wire_test] cannot serve with $backend\n";
|
||||
push @{$counts{failed}}, "backend-setup($backend)";
|
||||
++$counts{fail};
|
||||
++$expected;
|
||||
# Torn down even though it never came up: backend-setup writes
|
||||
# site.dhcpbackend and $BSTATE before it can fail, and leaving them
|
||||
# would hand the next backend's pass a machine already reconfigured.
|
||||
runcmd("sudo bash -c '. /etc/profile.d/xcat.sh && $fixture backend-teardown $backend'");
|
||||
next;
|
||||
}
|
||||
|
||||
$expected += scalar(@cases);
|
||||
run_cases($conf_file, \@cases, "($backend)", \%counts, "run_dhcp_wire_test");
|
||||
|
||||
@output = runcmd("sudo bash -c '. /etc/profile.d/xcat.sh && $fixture backend-teardown $backend'");
|
||||
@@ -794,9 +817,16 @@ sub run_dhcp_wire_test{
|
||||
print "[run_dhcp_wire_test] WARNING: only $backend_str is installed here, so the backends were not compared\n";
|
||||
}
|
||||
|
||||
# Case runs rather than cases: each case is run once per backend.
|
||||
my $casenum = $counts{pass} + $counts{fail};
|
||||
if($counts{fail}){
|
||||
# Case runs rather than cases: each case is run once per backend. Counted
|
||||
# from what was asked for, so a run that produced no verdict is missing from
|
||||
# the arithmetic and says so.
|
||||
my $casenum = $expected;
|
||||
my $missing = $expected - ($counts{pass} + $counts{fail});
|
||||
if($missing){
|
||||
print RED "[run_dhcp_wire_test] $missing case run(s) produced no verdict\n";
|
||||
push @{$counts{failed}}, "$missing-without-a-verdict";
|
||||
}
|
||||
if($counts{fail} or $missing){
|
||||
my $log_str = join(",", @{$counts{failed}});
|
||||
$check_result_str .= "> **DHCP WIRE TEST Failed** ($backend_str): Totalrun $casenum Passed $counts{pass} Failed $counts{fail} FailedCases: $log_str. Please click ``Details`` label in ``Merge pull request`` box for detailed information\n";
|
||||
print $check_result_str;
|
||||
@@ -827,6 +857,12 @@ sub run_dhcp_wire_test{
|
||||
#--------------------------------------------------------
|
||||
sub run_prov_wire_test{
|
||||
my @cases = list_cases("prov_wire");
|
||||
if(@cases and $cases[0] eq $QUERY_FAILED){
|
||||
print RED "[run_prov_wire_test] the case list could not be read\n";
|
||||
$check_result_str .= "> **PROVISION WIRE TEST Failed**: xcattest could not be asked which cases carry the prov_wire label\n";
|
||||
print $check_result_str;
|
||||
return 1;
|
||||
}
|
||||
unless(@cases){
|
||||
print "[run_prov_wire_test] no provisioning wire cases to run\n";
|
||||
$check_result_str .= "> **PROVISION WIRE TEST Skipped**: no case carries the prov_wire label\n";
|
||||
@@ -846,8 +882,16 @@ sub run_prov_wire_test{
|
||||
my %counts = (pass => 0, fail => 0, failed => []);
|
||||
run_cases($conf_file, \@cases, "", \%counts, "run_prov_wire_test");
|
||||
|
||||
my $casenum = $counts{pass} + $counts{fail};
|
||||
if($counts{fail}){
|
||||
# From the list that was asked for, not from what came back with a verdict:
|
||||
# a case that hung or was killed is counted by neither, and pass+fail would
|
||||
# report the survivors as a complete run.
|
||||
my $casenum = scalar(@cases);
|
||||
my $missing = $casenum - ($counts{pass} + $counts{fail});
|
||||
if($missing){
|
||||
print RED "[run_prov_wire_test] $missing case(s) produced no verdict\n";
|
||||
push @{$counts{failed}}, "$missing-without-a-verdict";
|
||||
}
|
||||
if($counts{fail} or $missing){
|
||||
my $log_str = join(",", @{$counts{failed}});
|
||||
$check_result_str .= "> **PROVISION WIRE TEST Failed**: Totalcase $casenum Passed $counts{pass} Failed $counts{fail} FailedCases: $log_str. Please click ``Details`` label in ``Merge pull request`` box for detailed information\n";
|
||||
print $check_result_str;
|
||||
@@ -914,6 +958,16 @@ sub run_fast_regression_test{
|
||||
my $failnum = $counts{fail};
|
||||
my @failcase = @{$counts{failed}};
|
||||
|
||||
# This phase already counted the cases it asked for rather than the ones
|
||||
# that answered; what was missing from it is the failure when the two
|
||||
# disagree. A case that hung or was killed carries no verdict line.
|
||||
my $missing = $casenum - ($passnum + $failnum);
|
||||
if($missing){
|
||||
print RED "[run_fast_regression_test] $missing case(s) produced no verdict\n";
|
||||
push @failcase, "$missing-without-a-verdict";
|
||||
++$failnum;
|
||||
}
|
||||
|
||||
if($failnum){
|
||||
my $log_str = join (",", @failcase );
|
||||
$check_result_str .= "> **FAST REGRESSION TEST Failed**: Totalcase $casenum Passed $passnum Failed $failnum FailedCases: $log_str. Please click ``Details`` label in ``Merge pull request`` box for detailed information\n";
|
||||
|
||||
Reference in New Issue
Block a user