diff --git a/github_action_xcat_test.pl b/github_action_xcat_test.pl index 4b49f95d3a..024009982e 100644 --- a/github_action_xcat_test.pl +++ b/github_action_xcat_test.pl @@ -1027,10 +1027,9 @@ mark_time("install_xcat"); #Run the xCAT-test unit tests. They need the perl dependencies xCAT pulls in #(Net::DNS, XML::Simple) and a usable xCAT database, so they run after install. print GREEN "\n------Running xCAT-test unit tests ------\n"; -$rst = run_unit_tests(); -if($rst){ +my $unit_rst = run_unit_tests(); +if($unit_rst){ print RED "Run of xCAT-test unit tests failed\n"; - exit $rst; } mark_time("run_unit_tests"); @@ -1039,9 +1038,9 @@ print GREEN "\n------Running xCAT-test BATS tests ------\n"; $rst = run_bats_tests(); if($rst){ print RED "Run of xCAT-test BATS tests failed\n"; - exit $rst; } mark_time("run_bats_tests"); +exit 1 if $unit_rst || $rst; print GREEN "\n------Running xCAT-test source integration tests ------\n"; $rst = run_source_integration_tests(); diff --git a/xCAT-test/unit/fixtures/github_action/Commands.pm b/xCAT-test/unit/fixtures/github_action/Commands.pm index 76ba8c9fa0..c58306a185 100644 --- a/xCAT-test/unit/fixtures/github_action/Commands.pm +++ b/xCAT-test/unit/fixtures/github_action/Commands.pm @@ -2,12 +2,14 @@ package Commands; use strict; use warnings; -use JSON::PP qw(encode_json); +use JSON::PP qw(decode_json encode_json); + +my @failed_commands = @{ decode_json($ENV{XCAT_TEST_CI_FAIL_COMMANDS} || '[]') }; sub record_command { my ($command) = @_; open(my $log, '>>', $ENV{XCAT_TEST_CI_COMMANDS}) or die "open command log: $!"; - print {$log} encode_json($command), "\n"; + print {$log} encode_json([$command]), "\n"; close($log) or die "close command log: $!"; return; } @@ -18,7 +20,7 @@ BEGIN { record_command($command); my @output = ("ok\n"); my $status = 0; - if ($command eq $ENV{XCAT_TEST_CI_FAIL_COMMAND}) { + if (grep { $command eq $_ } @failed_commands) { @output = ("command failed\n"); $status = 1; } elsif ($command =~ /^ip -o link /) { diff --git a/xCAT-test/unit/github_action_noninteractive_install.t b/xCAT-test/unit/github_action_noninteractive_install.t index 387ea7279a..ff0836418d 100644 --- a/xCAT-test/unit/github_action_noninteractive_install.t +++ b/xCAT-test/unit/github_action_noninteractive_install.t @@ -8,7 +8,7 @@ use File::Path qw(make_path); use File::Slurper qw(read_binary read_text write_text); use File::Temp qw(tempdir); use FindBin; -use JSON::PP qw(decode_json); +use JSON::PP qw(decode_json encode_json); use Text::ParseWords qw(shellwords); use Test::More; use YAML::PP; @@ -29,7 +29,8 @@ chmod(0755, "$repository/mklocalrepo.sh") or die "chmod repository fixture: $!"; sub run_driver { my ($failure, $failed_case) = @_; local $ENV{XCAT_TEST_CI_COMMANDS} = "$runner/commands.jsonl"; - local $ENV{XCAT_TEST_CI_FAIL_COMMAND} = defined($failure) ? $failure : ''; + my $failures = ref($failure) eq 'ARRAY' ? $failure : defined($failure) ? [$failure] : []; + local $ENV{XCAT_TEST_CI_FAIL_COMMANDS} = encode_json($failures); local $ENV{XCAT_TEST_CI_FAIL_CASE} = $failed_case || 0; local $ENV{RUNNER_WORKSPACE} = $workspace; local $ENV{RUNNER_TEMP} = $runner_temp; @@ -50,7 +51,7 @@ sub run_driver { my $error = $@; chdir($cwd) or die "restore working directory: $!"; die $error unless $captured; - my @commands = map { decode_json($_) } + my @commands = map { decode_json($_)->[0] } split /\n/, read_binary($ENV{XCAT_TEST_CI_COMMANDS}); is($stderr, '', 'driver emits no warnings') or diag($stdout); return ($status, \@commands, $stdout); @@ -109,7 +110,6 @@ is_deeply([grep { $_ eq $preserve || $_ eq $build || $_ eq $unit || $_ eq $bats for my $stage ( ['apt update', $apt[0]], ['xcat install', $apt[1]], ['xcat-test install', $apt[3]], ['source preservation', $preserve], ['package build', $build], - ['unit tests', $unit], ['BATS tests', $bats], ) { my ($name, $failure) = @$stage; subtest "stop after failed $name" => sub { @@ -122,6 +122,37 @@ for my $stage ( }; } +for my $case ( + ['both suites pass', 0, 0], + ['Perl fails and BATS passes', 1, 0], + ['Perl passes and BATS fails', 0, 1], + ['both suites fail', 1, 1], +) { + my ($name, $perl_fails, $bats_fails) = @$case; + subtest $name => sub { + my @failures; + push @failures, $unit if $perl_fails; + push @failures, $bats if $bats_fails; + my ($suite_status, $suite_commands, $suite_output) = run_driver(\@failures); + is_deeply([grep { $_ eq $unit || $_ eq $bats } @$suite_commands], [$unit, $bats], + 'both suites run once, in order'); + is($suite_status, ($perl_fails || $bats_fails) ? 256 : 0, + 'either suite failure fails the driver'); + for my $suite (['UNIT', $perl_fails], ['BATS', $bats_fails]) { + my ($label, $fails) = @$suite; + my $result = $fails ? 'Failed' : 'Successful'; + like($suite_output, qr/\*\*\Q$label TESTS $result\E\*\*/, + "$label result remains visible"); + } + my ($bats_index) = grep { $commands->[$_] eq $bats } 0 .. $#$commands; + ok(defined($bats_index), 'BATS ran in the successful reference run') or return; + my $expected = ($perl_fails || $bats_fails) + ? [@$commands[0 .. $bats_index]] : $commands; + is_deeply($suite_commands, $expected, + 'later phases run only when both suites pass'); + }; +} + my @setup_steps = grep { /setup-local-client\.sh/ } @$commands; is(scalar @setup_steps, 1, 'local client setup runs once'); for my $stage (