xCAT/xcat-dep
2
0
Fork 0
mirror of https://github.com/xcat2/xcat-dep.git synced 2026-09-12 12:36:23 +00:00
Code Issues Projects Releases Wiki Activity

refactor(genesis): share release build helpers

Browse Source
This commit is contained in:
Vinícius Ferrão
2026-08-22 19:24:45 -03:00
parent 881605847c
commit 69a00bf43f
6 changed files with 282 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
genesis-openembedded/build
+28 -82
View File
@@ -4,17 +4,22 @@ use strict;
use warnings;
use Cwd qw(abs_path);
use Digest::SHA ();
use File::Basename qw(basename dirname);
use File::Copy qw(copy);
use File::Find qw(find);
use File::Path qw(make_path);
use File::Spec;
use File::Temp qw(tempdir);
use FindBin;
use Getopt::Long qw(GetOptions);
use POSIX qw(strftime);
use lib "$FindBin::Bin/lib";
use lib "$FindBin::Bin/../lib";
use XCAT::BuildUtils qw(
capture_command
digest_manifest
read_first_line
relative_files
run_command
write_binary
);
use XCAT::GenesisRelease qw(
architectures
deb_package_name
@@ -60,20 +65,24 @@ for my $path (qw(Version xCAT-genesis-builder/oe/build xCAT-genesis-builder/oe/e
die "xcat-core source is missing $path\n" unless -f "$xcat_source/$path";
}
die "xcat-core checkout is not clean\n"
if _capture('git', '-C', $xcat_source, 'status', '--porcelain') ne '';
if capture_command('git', '-C', $xcat_source, 'status', '--porcelain') ne '';
my $revision = _capture('git', '-C', $xcat_source, 'rev-parse', 'HEAD');
my $revision = capture_command('git', '-C', $xcat_source, 'rev-parse', 'HEAD');
die "Invalid xcat-core revision: $revision\n" unless $revision =~ /^[0-9a-f]{40}$/;
if ($xcat_ref ne '') {
die "Invalid xcat-core ref: $xcat_ref\n" unless $xcat_ref =~ /^[A-Za-z0-9][A-Za-z0-9._\/-]*$/;
my $expected = _capture('git', '-C', $xcat_source, 'rev-parse', '--verify', "$xcat_ref^{commit}");
my $expected = capture_command(
'git', '-C', $xcat_source, 'rev-parse', '--verify', "$xcat_ref^{commit}",
);
die "xcat-core HEAD $revision does not match $xcat_ref ($expected)\n"
unless $revision eq $expected;
}
my $version = _read_line("$xcat_source/Version");
my $version = read_first_line("$xcat_source/Version");
die "Invalid xCAT version: $version\n" unless $version =~ /^\d+(?:\.\d+){1,3}$/;
my $source_date_epoch = _capture('git', '-C', $xcat_source, 'show', '-s', '--format=%ct', 'HEAD');
my $source_date_epoch = capture_command(
'git', '-C', $xcat_source, 'show', '-s', '--format=%ct', 'HEAD',
);
die "Invalid xcat-core commit time\n" unless $source_date_epoch =~ /^\d+$/;
my $release = strftime('snap%Y%m%d%H%M', gmtime($source_date_epoch));
@@ -101,7 +110,7 @@ local $ENV{XCAT_GENESIS_WORK_DIR} = $oe_work;
{
local $ENV{TMPDIR} = $oe_tmp;
make_path($oe_tmp);
_run("$xcat_source/xCAT-genesis-builder/oe/build", @requested_architectures);
run_command("$xcat_source/xCAT-genesis-builder/oe/build", @requested_architectures);
}
my $effective_deploy = "$oe_tmp/deploy";
die "Invalid OpenEmbedded deploy directory: $effective_deploy\n"
@@ -110,13 +119,13 @@ die "Invalid OpenEmbedded deploy directory: $effective_deploy\n"
for my $architecture (@requested_architectures) {
my $export = "$work/exports/$architecture";
make_path(dirname($export));
_run(
run_command(
"$xcat_source/xCAT-genesis-builder/oe/export",
$architecture, $effective_deploy, $export,
);
my $packages = "$work/packages/$architecture";
_run(
run_command(
"$FindBin::Bin/package",
'--architecture', $architecture,
'--export-dir', $export,
@@ -131,7 +140,7 @@ for my $architecture (@requested_architectures) {
}
my @formats = $format eq 'all' ? qw(deb rpm) : ($format);
_write_file(
write_binary(
"$staging/release.manifest",
"format=xcat-genesis-packages\n"
. "version=1\n"
@@ -142,11 +151,15 @@ _write_file(
. "architectures=" . join(',', @requested_architectures) . "\n"
. "formats=" . join(',', @formats) . "\n",
);
_write_checksums($staging);
my @release_files = grep { $_ ne 'SHA256SUMS' } relative_files($staging);
write_binary(
"$staging/SHA256SUMS",
digest_manifest($staging, 'sha256', @release_files),
);
validate_release($staging);
my @verify_args = ('--format', $format);
push(@verify_args, '--complete') if $all;
_run("$FindBin::Bin/verify-release", @verify_args, $staging);
run_command("$FindBin::Bin/verify-release", @verify_args, $staging);
chmod(0755, $staging) or die "Cannot make release directory readable: $!\n";
rename($staging, $output_dir) or die "Cannot publish $output_dir: $!\n";
@@ -183,73 +196,6 @@ sub _collect_one {
copy($source, $destination) or die "Cannot collect $source: $!\n";
}
sub _write_checksums {
my ($directory) = @_;
my @files;
find(
{
no_chdir => 1,
wanted => sub {
return unless -f $_ && !-l $_;
my $relative = File::Spec->abs2rel($File::Find::name, $directory);
$relative =~ tr{\\}{/};
push(@files, $relative) unless $relative eq 'SHA256SUMS';
},
},
$directory,
);
my $content = '';
for my $relative (sort @files) {
open(my $fh, '<:raw', "$directory/$relative")
or die "Cannot read $directory/$relative: $!\n";
my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest;
close($fh) or die "Cannot close $directory/$relative: $!\n";
$content .= "$digest $relative\n";
}
_write_file("$directory/SHA256SUMS", $content);
}
sub _read_line {
my ($path) = @_;
open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n";
my $line = <$fh>;
close($fh) or die "Cannot close $path: $!\n";
die "Empty file: $path\n" unless defined($line);
chomp($line);
$line =~ s/\r\z//;
return $line;
}
sub _capture {
my (@command) = @_;
open(my $fh, '-|', @command) or die "Cannot run $command[0]: $!\n";
local $/;
my $output = <$fh> // '';
close($fh) or die "Command failed: $command[0]\n";
$output =~ s/\s+\z//;
return $output;
}
sub _run {
my (@command) = @_;
print '+ ', join(' ', map { _display_quote($_) } @command), "\n";
system(@command) == 0 or die "Command failed: $command[0]\n";
}
sub _display_quote {
my ($value) = @_;
return $value if $value =~ /^[A-Za-z0-9_.,+\/:=@~-]+$/;
$value =~ s/'/'\\''/g;
return "'$value'";
}
sub _write_file {
my ($path, $content) = @_;
open(my $fh, '>:raw', $path) or die "Cannot write $path: $!\n";
print {$fh} $content or die "Cannot write $path: $!\n";
close($fh) or die "Cannot close $path: $!\n";
}
sub usage {
return <<'USAGE';
Usage: build [--xcat-source DIR] [--xcat-ref REF] [--output-dir DIR]
genesis-openembedded/package
+25 -75
View File
@@ -4,15 +4,21 @@ use strict;
use warnings;
use Cwd qw(abs_path);
use Digest::MD5 ();
use File::Basename qw(basename dirname);
use File::Copy qw(copy);
use File::Path qw(make_path remove_tree);
use File::Spec;
use File::Temp qw(tempdir);
use FindBin;
use Getopt::Long qw(GetOptions);
use lib "$FindBin::Bin/lib";
use lib "$FindBin::Bin/../lib";
use XCAT::BuildUtils qw(
capture_command
digest_manifest
relative_files
require_command
run_command
write_binary
);
use XCAT::GenesisRelease qw(
deb_package_name
rpm_package_name
@@ -84,16 +90,16 @@ for my $entry (_flat_files($export_dir)) {
copy("$export_dir/$entry", "$source_root/image/$entry")
or die "Cannot copy export file $entry: $!\n";
}
_write_file("$source_root/xcat-core-revision", "$revision\n");
write_binary("$source_root/xcat-core-revision", "$revision\n");
local $ENV{SOURCE_DATE_EPOCH} = $source_date_epoch;
if ($format eq 'all' || $format eq 'rpm') {
_require_command('rpmbuild');
require_command('rpmbuild');
_build_rpm($work, $staging, $source_root, $source_name, $rpm_name);
}
if ($format eq 'all' || $format eq 'deb') {
_require_command('dpkg-deb');
require_command('dpkg-deb');
_build_deb($work, $staging, $source_root, $deb_name);
}
@@ -111,7 +117,7 @@ sub _build_rpm {
make_path("$staging/rpm", "$staging/srpm");
my $archive = "$rpm_top/SOURCES/$source_name.tar.gz";
_run(
run_command(
'tar', '--sort=name', '--owner=0', '--group=0', '--numeric-owner',
"--mtime=\@$source_date_epoch", '--use-compress-program=gzip -n',
'-cf', $archive, '-C', dirname($source_root), basename($source_root),
@@ -119,7 +125,7 @@ sub _build_rpm {
my $spec = "$FindBin::Bin/rpm/xCAT-genesis-openembedded.spec";
copy($spec, "$rpm_top/SPECS/xCAT-genesis-openembedded.spec")
or die "Cannot stage RPM spec: $!\n";
_run(
run_command(
'rpmbuild', '-ba',
'--define', "_topdir $rpm_top",
'--define', "genesis_arch $architecture",
@@ -162,7 +168,7 @@ sub _build_deb {
my $installed_kib = _tree_bytes($root);
$installed_kib = int(($installed_kib + 1023) / 1024);
_write_file(
write_binary(
"$root/DEBIAN/control",
"Package: $package_name\n"
. "Version: $version-$release\n"
@@ -177,36 +183,17 @@ sub _build_deb {
_normalize_mtime($root, $source_date_epoch);
my $deb = "$staging/deb/${package_name}_${version}-${release}_all.deb";
_run('dpkg-deb', '--root-owner-group', '-Zgzip', '-z9', '--build', $root, $deb);
run_command('dpkg-deb', '--root-owner-group', '-Zgzip', '-z9', '--build', $root, $deb);
die "DEB build did not produce $deb\n" unless -f $deb;
}
sub _write_deb_md5sums {
my ($root) = @_;
my @files;
require File::Find;
File::Find::find(
{
no_chdir => 1,
wanted => sub {
return unless -f $_ && !-l $_;
return if $File::Find::name =~ m{\Q$root/DEBIAN/\E};
my $relative = File::Spec->abs2rel($File::Find::name, $root);
$relative =~ tr{\\}{/};
push(@files, $relative);
},
},
$root,
my @files = grep { $_ !~ m{\ADEBIAN/} } relative_files($root);
write_binary(
"$root/DEBIAN/md5sums",
digest_manifest($root, 'md5', @files),
);
my $content = '';
for my $relative (sort @files) {
open(my $fh, '<:raw', "$root/$relative")
or die "Cannot read $root/$relative: $!\n";
my $digest = Digest::MD5->new->addfile($fh)->hexdigest;
close($fh) or die "Cannot close $root/$relative: $!\n";
$content .= "$digest $relative\n";
}
_write_file("$root/DEBIAN/md5sums", $content);
}
sub _normalize_mtime {
@@ -226,12 +213,9 @@ sub _normalize_mtime {
sub _flat_files {
my ($directory) = @_;
opendir(my $dh, $directory) or die "Cannot read $directory: $!\n";
my @entries = sort grep { $_ ne '.' && $_ ne '..' } readdir($dh);
closedir($dh) or die "Cannot close $directory: $!\n";
my @entries = relative_files($directory);
for my $entry (@entries) {
my $path = "$directory/$entry";
die "Expected a regular file: $path\n" unless -f $path && !-l $path;
die "Expected a flat directory: $directory\n" if $entry =~ m{/};
}
return @entries;
}
@@ -239,51 +223,17 @@ sub _flat_files {
sub _tree_bytes {
my ($directory) = @_;
my $bytes = 0;
require File::Find;
File::Find::find(
{ no_chdir => 1, wanted => sub { $bytes += -s $_ if -f $_ && !-l $_ } },
$directory,
);
$bytes += -s "$directory/$_" for relative_files($directory);
return $bytes;
}
sub _require_command {
my ($command) = @_;
for my $directory (File::Spec->path()) {
return if -x "$directory/$command";
}
die "Required command not found: $command\n";
}
sub _require_gnu_tar {
_require_command('tar');
open(my $fh, '-|', 'tar', '--version') or die "Cannot run tar: $!\n";
my $version_line = <$fh> // '';
close($fh) or die "Cannot query tar version\n";
require_command('tar');
my $version_line = capture_command('tar', '--version');
die "GNU tar is required to build Genesis source packages\n"
unless $version_line =~ /^tar \(GNU tar\)/;
}
sub _run {
my (@command) = @_;
print '+ ', join(' ', map { _display_quote($_) } @command), "\n";
system(@command) == 0 or die "Command failed: $command[0]\n";
}
sub _display_quote {
my ($value) = @_;
return $value if $value =~ /^[A-Za-z0-9_.,+\/:=@~-]+$/;
$value =~ s/'/'\\''/g;
return "'$value'";
}
sub _write_file {
my ($path, $content) = @_;
open(my $fh, '>:raw', $path) or die "Cannot write $path: $!\n";
print {$fh} $content or die "Cannot write $path: $!\n";
close($fh) or die "Cannot close $path: $!\n";
}
sub usage {
return <<'USAGE';
Usage: package --architecture ARCH --export-dir DIR --output-dir DIR
genesis-openembedded/verify-release
+9 -27
View File
@@ -4,10 +4,10 @@ use strict;
use warnings;
use Cwd qw(abs_path);
use File::Spec;
use FindBin;
use Getopt::Long qw(GetOptions);
use lib "$FindBin::Bin/lib";
use lib "$FindBin::Bin/../lib";
use XCAT::BuildUtils qw(capture_command require_command);
use XCAT::GenesisRelease qw(
deb_package_name
rpm_package_name
@@ -49,7 +49,7 @@ print "Verified Genesis package release: $directory\n";
sub _verify_rpm {
my ($directory, $manifest, $architecture) = @_;
_require_command('rpm');
require_command('rpm');
my $name = rpm_package_name($architecture);
my $source_name = "$name-$manifest->{xcat_version}-$manifest->{xcat_release}.src.rpm";
my @packages = (
@@ -59,7 +59,7 @@ sub _verify_rpm {
);
for my $package (@packages) {
my ($path, $expected_arch, $expected_source) = @{$package};
my $metadata = _capture(
my $metadata = capture_command(
'rpm', '-qp', '--qf',
"%{NAME}\t%{VERSION}\t%{RELEASE}\t%{ARCH}\t%{EPOCHNUM}\t%{SOURCERPM}"
. "\t%{BUILDHOST}\t%{BUILDTIME}\n",
@@ -72,10 +72,10 @@ sub _verify_rpm {
);
die "Unexpected RPM identity for $path: $metadata\n" unless $metadata eq $expected;
for my $relationship (qw(conflicts obsoletes)) {
my $value = _capture('rpm', '-qp', "--$relationship", $path);
my $value = capture_command('rpm', '-qp', "--$relationship", $path);
die "Unexpected RPM $relationship for $path: $value\n" if $value ne '';
}
my $provides = _capture('rpm', '-qp', '--provides', $path);
my $provides = capture_command('rpm', '-qp', '--provides', $path);
die "Legacy RPM relationship in $path: $provides\n"
if $provides =~ /xCAT-genesis-base/;
}
@@ -83,12 +83,12 @@ sub _verify_rpm {
sub _verify_deb {
my ($directory, $manifest, $architecture) = @_;
_require_command('dpkg-deb');
require_command('dpkg-deb');
my $name = deb_package_name($architecture);
my $path = "$directory/deb/${name}_$manifest->{xcat_version}-$manifest->{xcat_release}_all.deb";
my $metadata = join(
"\n",
map { _capture('dpkg-deb', '-f', $path, $_) }
map { capture_command('dpkg-deb', '-f', $path, $_) }
qw(Package Version Architecture),
);
my $expected = join(
@@ -97,29 +97,11 @@ sub _verify_deb {
);
die "Unexpected DEB identity for $path: $metadata\n" unless $metadata eq $expected;
for my $field (qw(Breaks Conflicts Provides Replaces)) {
my $value = _capture('dpkg-deb', '-f', $path, $field);
my $value = capture_command('dpkg-deb', '-f', $path, $field);
die "Unexpected DEB $field for $path: $value\n" if $value ne '';
}
}
sub _require_command {
my ($command) = @_;
for my $directory (File::Spec->path()) {
return if -x "$directory/$command";
}
die "Required command not found: $command\n";
}
sub _capture {
my (@command) = @_;
open(my $fh, '-|', @command) or die "Cannot run $command[0]: $!\n";
local $/;
my $output = <$fh> // '';
close($fh) or die "Command failed: $command[0]\n";
$output =~ s/\s+\z//;
return $output;
}
sub usage {
return "Usage: verify-release [--format all|rpm|deb] [--complete] RELEASE_DIRECTORY\n";
}
lib/XCAT/BuildUtils.pm
+160
View File
@@ -0,0 +1,160 @@
package XCAT::BuildUtils;
use strict;
use warnings;
use Digest::MD5 ();
use Digest::SHA ();
use Exporter qw(import);
use File::Find qw(find);
use File::Slurper qw(read_binary write_binary);
use File::Spec;
use IPC::Cmd qw(can_run);
our @EXPORT_OK = qw(
capture_command
command_exists
digest_file
digest_manifest
display_quote
hashes_equal
print_step
read_binary
read_first_line
read_lines
relative_files
require_command
run_command
shell_quote
write_binary
);
sub command_exists {
my ($command) = @_;
return defined(can_run($command));
}
sub require_command {
my ($command) = @_;
return can_run($command)
// die "Required command not found: $command\n";
}
sub capture_command {
my (@command) = @_;
open(my $fh, '-|', @command) or die "Cannot run $command[0]: $!\n";
local $/;
my $output = <$fh> // '';
close($fh) or die "Command failed: $command[0]\n";
$output =~ s/\s+\z//;
return $output;
}
sub run_command {
my (@command) = @_;
print '+ ', join(' ', map { display_quote($_) } @command), "\n";
my $status = system(@command);
return 1 if $status == 0;
my $exit = $status == -1
? 255
: ($status & 127) ? 128 + ($status & 127) : $status >> 8;
die "Command failed (rc=$exit): "
. join(' ', map { display_quote($_) } @command) . "\n";
}
sub display_quote {
my ($value) = @_;
return $value if $value =~ /^[A-Za-z0-9_.,+\/:=@~-]+$/;
return shell_quote($value);
}
sub shell_quote {
my ($value) = @_;
$value = '' unless defined($value);
$value =~ s/'/'"'"'/g;
return "'$value'";
}
sub print_step {
my ($message) = @_;
print "\n== $message ==\n";
}
sub read_lines {
my ($path) = @_;
my $content = read_binary($path);
return () if $content eq '';
my @lines = split(/\n/, $content, -1);
pop(@lines) if @lines && $lines[-1] eq '';
s/\r\z// for @lines;
return @lines;
}
sub read_first_line {
my ($path) = @_;
my @lines = read_lines($path);
die "Empty file: $path\n" unless @lines;
return $lines[0];
}
sub relative_files {
my ($root) = @_;
die "Invalid directory: $root\n" unless -d $root && !-l $root;
my $absolute = File::Spec->rel2abs($root);
my @files;
find(
{
no_chdir => 1,
wanted => sub {
my $path = $File::Find::name;
return if $path eq $absolute;
die "Symbolic links are not allowed: $path\n" if -l $path;
return if -d $path;
die "Non-regular entry: $path\n" unless -f $path;
my $relative = File::Spec->abs2rel($path, $absolute);
$relative =~ tr{\\}{/};
push(@files, $relative);
},
},
$absolute,
);
my @sorted = sort @files;
return @sorted;
}
sub digest_file {
my ($path, $algorithm) = @_;
$algorithm //= 'sha256';
my $digest = $algorithm eq 'sha256' ? Digest::SHA->new(256)
: $algorithm eq 'md5' ? Digest::MD5->new
: die "Unsupported digest algorithm: $algorithm\n";
open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n";
my $value = $digest->addfile($fh)->hexdigest;
close($fh) or die "Cannot close $path: $!\n";
return $value;
}
sub digest_manifest {
my ($root, $algorithm, @files) = @_;
return join(
'',
map { digest_file("$root/$_", $algorithm) . " $_\n" }
sort @files,
);
}
sub hashes_equal {
my ($left, $right) = @_;
return 0 unless keys(%{$left}) == keys(%{$right});
for my $name (keys %{$left}) {
return 0 unless exists($right->{$name})
&& $left->{$name} eq $right->{$name};
}
return 1;
}
1;
genesis-openembedded/lib/XCAT/GenesisRelease.pm → lib/XCAT/GenesisRelease.pm
+8 -47
View File
@@ -3,10 +3,8 @@ package XCAT::GenesisRelease;
use strict;
use warnings;
use Digest::SHA ();
use Exporter qw(import);
use File::Find qw(find);
use File::Spec;
use XCAT::BuildUtils qw(digest_file read_lines relative_files);
our @EXPORT_OK = qw(
architectures
@@ -50,11 +48,8 @@ sub deb_package_name {
sub _read_key_values {
my ($path, $allowed) = @_;
open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n";
my %values;
while (my $line = <$fh>) {
chomp($line);
$line =~ s/\r\z//;
for my $line (read_lines($path)) {
die "Invalid manifest entry in $path: $line\n"
unless $line =~ /^([a-z][a-z0-9_]*)=([A-Za-z0-9][A-Za-z0-9.,_+~-]*)$/;
my ($key, $value) = ($1, $2);
@@ -62,43 +57,14 @@ sub _read_key_values {
die "Duplicate manifest key in $path: $key\n" if exists($values{$key});
$values{$key} = $value;
}
close($fh) or die "Cannot close $path: $!\n";
return \%values;
}
sub _regular_files {
my ($root) = @_;
die "Invalid directory: $root\n" unless -d $root && !-l $root;
my @files;
find(
{
no_chdir => 1,
wanted => sub {
my $path = $File::Find::name;
return if $path eq $root;
die "Symbolic links are not allowed: $path\n" if -l $path;
return if -d $path;
die "Non-regular release entry: $path\n" unless -f $path;
my $relative = File::Spec->abs2rel($path, $root);
$relative =~ tr{\\}{/};
push(@files, $relative);
},
},
$root,
);
my @sorted = sort @files;
return @sorted;
}
sub _read_checksums {
my ($root) = @_;
my $path = "$root/SHA256SUMS";
open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n";
my %checksums;
while (my $line = <$fh>) {
chomp($line);
$line =~ s/\r\z//;
for my $line (read_lines($path)) {
die "Invalid checksum entry in $path: $line\n"
unless $line =~ /^([0-9a-f]{64}) ([A-Za-z0-9][A-Za-z0-9._\/+~-]*)$/;
my ($digest, $name) = ($1, $2);
@@ -108,21 +74,18 @@ sub _read_checksums {
if exists($checksums{$name});
$checksums{$name} = $digest;
}
close($fh) or die "Cannot close $path: $!\n";
return \%checksums;
}
sub _verify_checksums {
my ($root) = @_;
my @files = grep { $_ ne 'SHA256SUMS' } _regular_files($root);
my @files = grep { $_ ne 'SHA256SUMS' } relative_files($root);
my $checksums = _read_checksums($root);
my %files = map { $_ => 1 } @files;
for my $name (@files) {
die "Missing checksum for $name\n" unless exists($checksums->{$name});
open(my $fh, '<:raw', "$root/$name") or die "Cannot read $root/$name: $!\n";
my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest;
close($fh) or die "Cannot close $root/$name: $!\n";
my $digest = digest_file("$root/$name", 'sha256');
die "Checksum mismatch for $name\n" unless $digest eq $checksums->{$name};
}
for my $name (keys %{$checksums}) {
@@ -148,7 +111,7 @@ sub validate_export {
);
$required{'fw_jump.elf'} = 1 if $architecture eq 'riscv64';
my @files = _regular_files($directory);
my @files = relative_files($directory);
my %files = map { $_ => 1 } @files;
for my $name (sort keys %required) {
die "Genesis export is missing $name\n" unless $files{$name};
@@ -240,7 +203,7 @@ sub validate_release {
$expected{"deb/${deb}_$manifest->{xcat_version}-$manifest->{xcat_release}_all.deb"} = 1;
}
}
for my $file (grep { $_ ne 'SHA256SUMS' } _regular_files($directory)) {
for my $file (grep { $_ ne 'SHA256SUMS' } relative_files($directory)) {
die "Unexpected Genesis release artifact: $file\n" unless $expected{$file};
delete($expected{$file});
}
@@ -269,9 +232,7 @@ sub verify_release_file {
die "Missing verified checksum for $relative\n"
unless ref($checksums) eq 'HASH' && exists($checksums->{$relative});
die "Invalid collected release file: $path\n" unless -f $path && !-l $path;
open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n";
my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest;
close($fh) or die "Cannot close $path: $!\n";
my $digest = digest_file($path, 'sha256');
die "Collected release file checksum mismatch: $path\n"
unless $digest eq $checksums->{$relative};
return 1;
mockbuild-all.pl
+52 -99
View File
@@ -13,7 +13,16 @@ use FindBin;
use Getopt::Long qw(GetOptions);
use Parallel::ForkManager;
use POSIX qw(strftime);
use lib "$FindBin::Bin/genesis-openembedded/lib";
use lib "$FindBin::Bin/lib";
use XCAT::BuildUtils qw(
capture_command
hashes_equal
print_step
read_first_line
require_command
run_command
shell_quote
);
use XCAT::GenesisRelease qw(
validated_release_checksums
verify_release_file
@@ -96,7 +105,7 @@ $repo_root = abs_path($repo_root);
my $SOURCE_DATE_EPOCH;
$SOURCE_DATE_EPOCH = $build_timestamp if defined $build_timestamp;
if (!$SOURCE_DATE_EPOCH && -f "$repo_root/Gitepoch") {
$SOURCE_DATE_EPOCH = slurp_chomp("$repo_root/Gitepoch");
$SOURCE_DATE_EPOCH = read_first_line("$repo_root/Gitepoch");
}
unless ($SOURCE_DATE_EPOCH && $SOURCE_DATE_EPOCH =~ /^\d+$/) {
$SOURCE_DATE_EPOCH = `git -C \Q$repo_root\E log -1 --format=%ct HEAD 2>/dev/null`;
@@ -128,7 +137,7 @@ acquire_output_lock($output_base, $force_unlock);
$xcat_src = resolve_xcat_source($xcat_src, $repo_root);
my $arch = capture('uname -m');
my $arch = capture_command('uname', '-m');
my %os = read_os_release('/etc/os-release');
my $os_id = $os{ID} // '';
my $version_id = $os{VERSION_ID} // '';
@@ -153,10 +162,10 @@ if ($genesis_release ne '') {
my $verifier = "$script_dir/genesis-openembedded/verify-release";
die "Genesis release verifier not found: $verifier\n" unless -x $verifier;
my $checksums_before = validated_release_checksums($genesis_release);
run_argv($^X, $verifier, '--complete', '--format', 'rpm', $genesis_release);
run_command($^X, $verifier, '--complete', '--format', 'rpm', $genesis_release);
my $checksums_after = validated_release_checksums($genesis_release);
die "Genesis release changed during verification\n"
unless checksum_sets_match($checksums_before, $checksums_after);
unless hashes_equal($checksums_before, $checksums_after);
$genesis_release_checksums = $checksums_before;
}
@@ -190,7 +199,7 @@ $tgt_workers = scalar(@build_targets) if $tgt_workers > scalar(@build_targets);
# mock build already gets a unique --uniqueext (separate chroot), so the only limit needed is
# hardware: total concurrent builds across all targets stays <= $cap (default host nproc). The
# per-target build-step concurrency is therefore the cap divided across the active targets.
my $cap = $max_parallel > 0 ? $max_parallel : (capture('nproc') || 4);
my $cap = $max_parallel > 0 ? $max_parallel : (capture_command('nproc') || 4);
my $per_target_builds = defined($parallel_builds) ? $parallel_builds : int($cap / $tgt_workers);
$per_target_builds = 1 if $per_target_builds < 1;
print "parallel_targets: " . ($parallel_targets > 0 ? $parallel_targets : "auto($tgt_workers)") . "\n";
@@ -304,7 +313,7 @@ my @collect_roots;
if ($scrub_all_chroots) {
run_step(
step => "Scrub all chroots for target $target",
cmd => "mock -r " . sh_quote($target) . " --scrub=all",
cmd => "mock -r " . shell_quote($target) . " --scrub=all",
log => "$log_root/scrub-all-chroots.log",
);
}
@@ -321,13 +330,13 @@ if (!$skip_build) {
my $step_log = "$log_root/$name";
my $step_uniqueext = build_mock_uniqueext($run_id, ++$build_step_seq, $name);
my $cmd = join(' ',
'perl', sh_quote($script),
'--mock-cfg', sh_quote($target),
'--mock-uniqueext', sh_quote($step_uniqueext),
'--result-dir', sh_quote($step_result),
'--log-dir', sh_quote($step_log),
'perl', shell_quote($script),
'--mock-cfg', shell_quote($target),
'--mock-uniqueext', shell_quote($step_uniqueext),
'--result-dir', shell_quote($step_result),
'--log-dir', shell_quote($step_log),
# host-local, run-scoped work dir so /tmp doesn't collide between runs
'--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/$name"),
'--work-dir', shell_quote("/tmp/mockbuild-all-$run_id/$name"),
'--build-timestamp', $SOURCE_DATE_EPOCH,
($skip_install ? '--skip-install' : ()),
);
@@ -349,12 +358,12 @@ if (!$skip_build) {
# forks one mock build per perl package (~7), which -- multiplied by parallel EL targets --
# oversubscribes the host.
my $cmd = join(' ',
'perl', sh_quote($perl_builder),
'--mock-cfg', sh_quote($target),
'--mock-uniqueext', sh_quote($perl_uniqueext),
'--result-dir', sh_quote($perl_result),
'--log-dir', sh_quote($perl_log),
'--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/perl-list6"),
'perl', shell_quote($perl_builder),
'--mock-cfg', shell_quote($target),
'--mock-uniqueext', shell_quote($perl_uniqueext),
'--result-dir', shell_quote($perl_result),
'--log-dir', shell_quote($perl_log),
'--work-dir', shell_quote("/tmp/mockbuild-all-$run_id/perl-list6"),
(($max_build_workers && $max_build_workers >= 1) ? ('--jobs', $max_build_workers) : ()),
'--build-timestamp', $SOURCE_DATE_EPOCH,
($skip_install ? '--skip-install' : ()),
@@ -371,14 +380,14 @@ if (!$skip_build) {
if (!$skip_xcat) {
# Own HOME per target (buildrpms.pl uses $HOME/rpmbuild) so parallel targets don't race.
my $xcat_home = "/tmp/mockbuild-all-$run_id/xcat-home";
my $mktree = join(' ', map { sh_quote("$xcat_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS));
my $cmd = "mkdir -p $mktree && HOME=" . sh_quote($xcat_home) . ' ' . join(' ',
'perl', sh_quote("$xcat_src/buildrpms.pl"),
'--target', sh_quote($target),
my $mktree = join(' ', map { shell_quote("$xcat_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS));
my $cmd = "mkdir -p $mktree && HOME=" . shell_quote($xcat_home) . ' ' . join(' ',
'perl', shell_quote("$xcat_src/buildrpms.pl"),
'--target', shell_quote($target),
'--nproc', int($nproc),
'--force',
'--verbose',
'--xcat_dep_path', sh_quote($repo_root),
'--xcat_dep_path', shell_quote($repo_root),
);
push @build_steps, {
id => 'xcat',
@@ -402,15 +411,15 @@ if (!$skip_build) {
my $genesis_home = "/tmp/mockbuild-all-$run_id/genesis-home";
# buildrpms.pl's rpmdev-setuptree only runs during env setup, not per build, so create the
# rpmbuild tree ourselves for this per-target HOME (else $HOME/rpmbuild/SOURCES is missing).
my $mktree = join(' ', map { sh_quote("$genesis_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS));
my $cmd = "mkdir -p $mktree && HOME=" . sh_quote($genesis_home) . ' ' . join(' ',
'perl', sh_quote("$xcat_src/buildrpms.pl"),
my $mktree = join(' ', map { shell_quote("$genesis_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS));
my $cmd = "mkdir -p $mktree && HOME=" . shell_quote($genesis_home) . ' ' . join(' ',
'perl', shell_quote("$xcat_src/buildrpms.pl"),
'--package', 'xCAT-genesis-base',
'--target', sh_quote($target),
'--target', shell_quote($target),
'--nproc', int($nproc),
'--force',
'--verbose',
'--xcat_dep_path', sh_quote($repo_root),
'--xcat_dep_path', shell_quote($repo_root),
);
push @build_steps, {
id => 'genesis',
@@ -537,8 +546,8 @@ if (!$skip_tarball) {
my $cmd = join(' ',
'tar', '--sort=name', '--owner=0', '--group=0',
"--mtime=\@$SOURCE_DATE_EPOCH",
'-C', sh_quote($run_root),
'-czf', sh_quote($tarball),
'-C', shell_quote($run_root),
'-czf', shell_quote($tarball),
'repo'
);
run_step(
@@ -549,8 +558,8 @@ if (!$skip_tarball) {
my $srpm_cmd = join(' ',
'tar', '--sort=name', '--owner=0', '--group=0',
"--mtime=\@$SOURCE_DATE_EPOCH",
'-C', sh_quote($run_root),
'-czf', sh_quote($srpm_tarball),
'-C', shell_quote($run_root),
'-czf', shell_quote($srpm_tarball),
'repo-src'
);
run_step(
@@ -637,8 +646,8 @@ sub deploy_target {
sub createrepo_c_cmd {
my ($dir) = @_;
return 'createrepo_c --update --database '
. '--revision ' . sh_quote($SOURCE_DATE_EPOCH) . ' --set-timestamp-to-revision '
. sh_quote($dir);
. '--revision ' . shell_quote($SOURCE_DATE_EPOCH) . ' --set-timestamp-to-revision '
. shell_quote($dir);
}
sub sign_and_index_repo {
@@ -647,15 +656,15 @@ sub sign_and_index_repo {
if ($gpg_sign && @rpms) {
local $ENV{GNUPGHOME} = $gpg_home if $gpg_home;
run_simple(qq(rpmsign --define "%_gpg_name $gpg_key_name" --addsign )
. join(' ', map { sh_quote($_) } @rpms));
. join(' ', map { shell_quote($_) } @rpms));
}
run_simple(createrepo_c_cmd($dir));
if ($gpg_sign) {
local $ENV{GNUPGHOME} = $gpg_home if $gpg_home;
my $repomd = "$dir/repodata/repomd.xml";
unlink "$repomd.asc" if -f "$repomd.asc";
run_simple(qq(gpg -a --detach-sign --default-key "$gpg_key_name" ) . sh_quote($repomd));
run_simple(qq(gpg -a --export "$gpg_key_name" > ) . sh_quote("$repomd.key"));
run_simple(qq(gpg -a --detach-sign --default-key "$gpg_key_name" ) . shell_quote($repomd));
run_simple(qq(gpg -a --export "$gpg_key_name" > ) . shell_quote("$repomd.key"));
}
}
@@ -769,16 +778,6 @@ Notes:
USAGE
}
sub print_step {
my ($msg) = @_;
print "\n== $msg ==\n";
}
sub require_command {
my ($cmd) = @_;
run_simple("command -v " . sh_quote($cmd) . " >/dev/null 2>&1");
}
sub run_simple {
my ($cmd) = @_;
my $rc = system($cmd);
@@ -788,27 +787,6 @@ sub run_simple {
}
}
sub run_argv {
my (@command) = @_;
my $rc = system(@command);
if ($rc != 0) {
my $exit = $rc == -1 ? 255 : ($rc >> 8);
die "Command failed (rc=$exit): @command\n";
}
}
sub capture {
my ($cmd) = @_;
my $out = `$cmd`;
my $rc = $?;
if ($rc != 0) {
my $exit = $rc == -1 ? 255 : ($rc >> 8);
die "Command failed (rc=$exit): $cmd\n$out\n";
}
chomp $out;
return $out;
}
sub run_step {
my (%args) = @_;
my $step = $args{step} // 'Run command';
@@ -829,12 +807,12 @@ sub run_step {
my $full_cmd = $cmd;
if ($cwd) {
$full_cmd = "cd " . sh_quote($cwd) . " && $cmd";
$full_cmd = "cd " . shell_quote($cwd) . " && $cmd";
}
if ($log) {
my $log_dir = dirname($log);
make_path($log_dir) if !-d $log_dir;
$full_cmd .= " > " . sh_quote($log) . " 2>&1";
$full_cmd .= " > " . shell_quote($log) . " 2>&1";
}
my $rc = system($full_cmd);
@@ -1006,15 +984,6 @@ sub assert_genesis_release_copied {
}
}
sub checksum_sets_match {
my ($left, $right) = @_;
return 0 unless keys(%{$left}) == keys(%{$right});
for my $name (keys %{$left}) {
return 0 unless exists($right->{$name}) && $left->{$name} eq $right->{$name};
}
return 1;
}
sub collect_rpms {
my (%args) = @_;
my $roots = $args{roots} // [];
@@ -1125,14 +1094,14 @@ sub resolve_mock_cfg {
rocky => 'rocky',
);
my $candidate = "${os_id}+epel-${rel}-${arch}";
my $rc = system("mock -r " . sh_quote($candidate) . " --print-root-path >/dev/null 2>&1");
my $rc = system("mock -r " . shell_quote($candidate) . " --print-root-path >/dev/null 2>&1");
if ($rc == 0) {
return $candidate;
}
if (exists $short_forms{$os_id}) {
my $short = $short_forms{$os_id};
$candidate = "${short}+epel-${rel}-${arch}";
$rc = system("mock -r " . sh_quote($candidate) . " --print-root-path >/dev/null 2>&1");
$rc = system("mock -r " . shell_quote($candidate) . " --print-root-path >/dev/null 2>&1");
if ($rc == 0) {
print "Mock config resolved (short form): $candidate\n";
return $candidate;
@@ -1193,7 +1162,7 @@ sub acquire_output_lock {
if (mkdir $lock) {
$HELD_LOCK = $lock;
$LOCK_OWNER_PID = $$;
my $host = capture('uname -n') || 'unknown';
my $host = capture_command('uname', '-n') || 'unknown';
if (open my $fh, '>', "$lock/owner") {
print {$fh} "host=$host\npid=$$\nepoch=" . time() . "\n";
close $fh;
@@ -1250,19 +1219,3 @@ sub uniq {
my %seen;
return grep { defined($_) && !$seen{$_}++ } @_;
}
sub slurp_chomp {
my ($path) = @_;
open my $fh, '<', $path or die "Cannot read $path: $!\n";
my $line = <$fh>;
close $fh;
chomp $line if defined $line;
return $line // '';
}
sub sh_quote {
my ($s) = @_;
$s = '' if !defined $s;
$s =~ s/'/'"'"'/g;
return "'$s'";
}