From f909033b10b7de17473bad5acc014d66bdb8efce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:07:02 -0300 Subject: [PATCH 01/82] feat(genesis): package OpenEmbedded exports --- .gitignore | 1 + genesis-openembedded/build | 238 ++++++++++++++ .../lib/XCAT/GenesisRelease.pm | 248 +++++++++++++++ genesis-openembedded/package | 297 ++++++++++++++++++ .../rpm/xCAT-genesis-base.spec | 49 +++ genesis-openembedded/verify-release | 96 ++++++ 6 files changed, 929 insertions(+) create mode 100755 genesis-openembedded/build create mode 100644 genesis-openembedded/lib/XCAT/GenesisRelease.pm create mode 100755 genesis-openembedded/package create mode 100644 genesis-openembedded/rpm/xCAT-genesis-base.spec create mode 100755 genesis-openembedded/verify-release diff --git a/.gitignore b/.gitignore index 0622450..ed2cb46 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ Gitepoch Gitinfo Release +build-output/ diff --git a/genesis-openembedded/build b/genesis-openembedded/build new file mode 100755 index 0000000..da39fe0 --- /dev/null +++ b/genesis-openembedded/build @@ -0,0 +1,238 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use Cwd qw(abs_path); +use Digest::SHA (); +use File::Basename qw(basename dirname); +use File::Copy qw(copy); +use File::Find qw(find); +use File::Path qw(make_path); +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use Getopt::Long qw(GetOptions); +use POSIX qw(strftime); +use lib "$FindBin::Bin/lib"; +use XCAT::GenesisRelease qw( + architectures + deb_package_name + rpm_package_name + validate_architecture + validate_release +); + +my $repo_root = abs_path("$FindBin::Bin/.."); +my $xcat_source = "$repo_root/../xcat-core"; +my $xcat_ref = ''; +my $output_dir = ''; +my $format = 'all'; +my $all = 0; +my @requested_architectures; + +GetOptions( + 'xcat-source=s' => \$xcat_source, + 'xcat-ref=s' => \$xcat_ref, + 'output-dir=s' => \$output_dir, + 'format=s' => \$format, + 'architecture=s@' => \@requested_architectures, + 'all!' => \$all, +) or die usage(); + +die "Unsupported package format: $format\n" + unless $format eq 'all' || $format eq 'rpm' || $format eq 'deb'; +die "Use either --all or --architecture\n" if $all && @requested_architectures; +@requested_architectures = architectures() if $all; +@requested_architectures = ('x86_64') unless @requested_architectures; + +my %seen; +@requested_architectures = grep { !$seen{$_}++ } @requested_architectures; +validate_architecture($_) for @requested_architectures; + +$xcat_source = abs_path($xcat_source) or die "Cannot resolve xcat-core source\n"; +for my $path (qw(Version xCAT-genesis-builder/oe/build xCAT-genesis-builder/oe/export)) { + die "xcat-core source is missing $path\n" unless -f "$xcat_source/$path"; +} +die "xcat-core has tracked changes\n" + if _capture('git', '-C', $xcat_source, 'status', '--porcelain', '--untracked-files=no') ne ''; + +my $revision = _capture('git', '-C', $xcat_source, 'rev-parse', 'HEAD'); +die "Invalid xcat-core revision: $revision\n" unless $revision =~ /^[0-9a-f]{40}$/; +if ($xcat_ref ne '') { + die "Invalid xcat-core ref: $xcat_ref\n" unless $xcat_ref =~ /^[A-Za-z0-9][A-Za-z0-9._\/-]*$/; + my $expected = _capture('git', '-C', $xcat_source, 'rev-parse', '--verify', "$xcat_ref^{commit}"); + die "xcat-core HEAD $revision does not match $xcat_ref ($expected)\n" + unless $revision eq $expected; +} + +my $version = _read_line("$xcat_source/Version"); +die "Invalid xCAT version: $version\n" unless $version =~ /^\d+(?:\.\d+){1,3}$/; +my $source_date_epoch = _capture('git', '-C', $xcat_source, 'show', '-s', '--format=%ct', 'HEAD'); +die "Invalid xcat-core commit time\n" unless $source_date_epoch =~ /^\d+$/; +my $release = strftime('snap%Y%m%d%H%M', gmtime($source_date_epoch)); + +if ($output_dir eq '') { + $output_dir = "$repo_root/build-output/genesis-openembedded/$version-$release"; +} +my $output_parent = dirname($output_dir); +make_path($output_parent) unless -d $output_parent; +$output_parent = abs_path($output_parent) or die "Cannot resolve output parent\n"; +$output_dir = "$output_parent/" . basename($output_dir); +die "Output already exists: $output_dir\n" if -e $output_dir || -l $output_dir; + +my $work = tempdir('xcat-genesis-release.XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $staging = tempdir('.xcat-genesis-release.XXXXXX', DIR => $output_parent, CLEANUP => 1); +my $oe_work = "$work/openembedded"; +local $ENV{XCAT_GENESIS_WORK_DIR} = $oe_work; +_run("$xcat_source/xCAT-genesis-builder/oe/build", @requested_architectures); +my $effective_deploy = "$oe_work/build/tmp/deploy"; +die "Invalid OpenEmbedded deploy directory: $effective_deploy\n" + unless -d $effective_deploy && !-l $effective_deploy; + +for my $architecture (@requested_architectures) { + my $export = "$work/exports/$architecture"; + make_path(dirname($export)); + _run( + "$xcat_source/xCAT-genesis-builder/oe/export", + $architecture, $effective_deploy, $export, + ); + + my $packages = "$work/packages/$architecture"; + _run( + "$FindBin::Bin/package", + '--architecture', $architecture, + '--export-dir', $export, + '--output-dir', $packages, + '--version', $version, + '--release', $release, + '--revision', $revision, + '--source-date-epoch', $source_date_epoch, + '--format', $format, + ); + _collect_packages($packages, $staging, $architecture); +} + +my @formats = $format eq 'all' ? qw(deb rpm) : ($format); +_write_file( + "$staging/release.manifest", + "format=xcat-genesis-packages\n" + . "version=1\n" + . "xcat_version=$version\n" + . "xcat_release=$release\n" + . "xcat_revision=$revision\n" + . "source_date_epoch=$source_date_epoch\n" + . "architectures=" . join(',', @requested_architectures) . "\n" + . "formats=" . join(',', @formats) . "\n", +); +_write_checksums($staging); +validate_release($staging); +_run("$FindBin::Bin/verify-release", '--format', $format, $staging); + +rename($staging, $output_dir) or die "Cannot publish $output_dir: $!\n"; +print "Built Genesis package release in $output_dir\n"; + +sub _collect_packages { + my ($source, $destination, $architecture) = @_; + if ($format eq 'all' || $format eq 'rpm') { + my $name = rpm_package_name($architecture); + _collect_one( + "$source/rpm/$name-$version-$release.noarch.rpm", + "$destination/rpm", + ); + _collect_one( + "$source/srpm/$name-$version-$release.src.rpm", + "$destination/srpm", + ); + } + if ($format eq 'all' || $format eq 'deb') { + my $name = deb_package_name($architecture); + _collect_one( + "$source/deb/${name}_${version}-${release}_all.deb", + "$destination/deb", + ); + } +} + +sub _collect_one { + my ($source, $directory) = @_; + die "Missing package artifact: $source\n" unless -f $source && !-l $source; + make_path($directory) unless -d $directory; + my $destination = "$directory/" . basename($source); + die "Duplicate package artifact: $destination\n" if -e $destination || -l $destination; + copy($source, $destination) or die "Cannot collect $source: $!\n"; +} + +sub _write_checksums { + my ($directory) = @_; + my @files; + find( + { + no_chdir => 1, + wanted => sub { + return unless -f $_ && !-l $_; + my $relative = File::Spec->abs2rel($File::Find::name, $directory); + $relative =~ tr{\\}{/}; + push(@files, $relative) unless $relative eq 'SHA256SUMS'; + }, + }, + $directory, + ); + my $content = ''; + for my $relative (sort @files) { + open(my $fh, '<:raw', "$directory/$relative") + or die "Cannot read $directory/$relative: $!\n"; + my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest; + close($fh) or die "Cannot close $directory/$relative: $!\n"; + $content .= "$digest $relative\n"; + } + _write_file("$directory/SHA256SUMS", $content); +} + +sub _read_line { + my ($path) = @_; + open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; + my $line = <$fh>; + close($fh) or die "Cannot close $path: $!\n"; + die "Empty file: $path\n" unless defined($line); + chomp($line); + $line =~ s/\r\z//; + return $line; +} + +sub _capture { + my (@command) = @_; + open(my $fh, '-|', @command) or die "Cannot run $command[0]: $!\n"; + local $/; + my $output = <$fh> // ''; + close($fh) or die "Command failed: $command[0]\n"; + $output =~ s/\s+\z//; + return $output; +} + +sub _run { + my (@command) = @_; + print '+ ', join(' ', map { _display_quote($_) } @command), "\n"; + system(@command) == 0 or die "Command failed: $command[0]\n"; +} + +sub _display_quote { + my ($value) = @_; + return $value if $value =~ /^[A-Za-z0-9_.,+\/:=@~-]+$/; + $value =~ s/'/'\\''/g; + return "'$value'"; +} + +sub _write_file { + my ($path, $content) = @_; + open(my $fh, '>:raw', $path) or die "Cannot write $path: $!\n"; + print {$fh} $content or die "Cannot write $path: $!\n"; + close($fh) or die "Cannot close $path: $!\n"; +} + +sub usage { + return <<'USAGE'; +Usage: build [--xcat-source DIR] [--xcat-ref REF] [--output-dir DIR] + [--format all|rpm|deb] [--architecture ARCH ... | --all] +USAGE +} diff --git a/genesis-openembedded/lib/XCAT/GenesisRelease.pm b/genesis-openembedded/lib/XCAT/GenesisRelease.pm new file mode 100644 index 0000000..6bf70fc --- /dev/null +++ b/genesis-openembedded/lib/XCAT/GenesisRelease.pm @@ -0,0 +1,248 @@ +package XCAT::GenesisRelease; + +use strict; +use warnings; + +use Digest::SHA (); +use Exporter qw(import); +use File::Find qw(find); +use File::Spec; + +our @EXPORT_OK = qw( + architectures + deb_package_name + read_release_manifest + rpm_package_name + validate_architecture + validate_export + validate_release +); + +my @ARCHITECTURES = qw(x86 x86_64 ppc64 ppc64le armv7hf aarch64 riscv64); +my %ARCHITECTURE = map { $_ => 1 } @ARCHITECTURES; + +sub architectures { + return @ARCHITECTURES; +} + +sub validate_architecture { + my ($architecture) = @_; + die "Unsupported Genesis architecture: $architecture\n" + unless defined($architecture) && $ARCHITECTURE{$architecture}; + return $architecture; +} + +sub rpm_package_name { + my ($architecture) = @_; + validate_architecture($architecture); + return "xCAT-genesis-base-$architecture"; +} + +sub deb_package_name { + my ($architecture) = @_; + validate_architecture($architecture); + $architecture =~ tr/_/-/; + return "xcat-genesis-base-$architecture"; +} + +sub _read_key_values { + my ($path, $allowed) = @_; + open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; + my %values; + while (my $line = <$fh>) { + chomp($line); + $line =~ s/\r\z//; + die "Invalid manifest entry in $path: $line\n" + unless $line =~ /^([a-z][a-z0-9_]*)=([A-Za-z0-9][A-Za-z0-9.,_+~-]*)$/; + my ($key, $value) = ($1, $2); + die "Unknown manifest key in $path: $key\n" unless $allowed->{$key}; + die "Duplicate manifest key in $path: $key\n" if exists($values{$key}); + $values{$key} = $value; + } + close($fh) or die "Cannot close $path: $!\n"; + return \%values; +} + +sub _regular_files { + my ($root) = @_; + die "Invalid directory: $root\n" unless -d $root && !-l $root; + + my @files; + find( + { + no_chdir => 1, + wanted => sub { + my $path = $File::Find::name; + return if $path eq $root; + die "Symbolic links are not allowed: $path\n" if -l $path; + return if -d $path; + die "Non-regular release entry: $path\n" unless -f $path; + my $relative = File::Spec->abs2rel($path, $root); + $relative =~ tr{\\}{/}; + push(@files, $relative); + }, + }, + $root, + ); + return sort @files; +} + +sub _read_checksums { + my ($root) = @_; + my $path = "$root/SHA256SUMS"; + open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; + my %checksums; + while (my $line = <$fh>) { + chomp($line); + $line =~ s/\r\z//; + die "Invalid checksum entry in $path: $line\n" + unless $line =~ /^([0-9a-f]{64}) ([A-Za-z0-9][A-Za-z0-9._\/-]*)$/; + my ($digest, $name) = ($1, $2); + die "Unsafe checksum path in $path: $name\n" + if $name =~ m{(?:\A|/)\.\.(?:/|\z)} || $name =~ m{//}; + die "Duplicate checksum entry in $path: $name\n" + if exists($checksums{$name}); + $checksums{$name} = $digest; + } + close($fh) or die "Cannot close $path: $!\n"; + return \%checksums; +} + +sub _verify_checksums { + my ($root) = @_; + my @files = grep { $_ ne 'SHA256SUMS' } _regular_files($root); + my $checksums = _read_checksums($root); + + my %files = map { $_ => 1 } @files; + for my $name (@files) { + die "Missing checksum for $name\n" unless exists($checksums->{$name}); + open(my $fh, '<:raw', "$root/$name") or die "Cannot read $root/$name: $!\n"; + my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest; + close($fh) or die "Cannot close $root/$name: $!\n"; + die "Checksum mismatch for $name\n" unless $digest eq $checksums->{$name}; + } + for my $name (keys %{$checksums}) { + die "Checksum names a missing file: $name\n" unless $files{$name}; + } + return 1; +} + +sub validate_export { + my ($directory, $architecture) = @_; + validate_architecture($architecture); + die "Invalid Genesis export: $directory\n" unless -d $directory && !-l $directory; + + my %required = map { $_ => 1 } qw( + SHA256SUMS + image.manifest + image.spdx.json + image.vex.json + initramfs.cpio.gz + kernel + license.manifest + xcat-genesis.manifest + ); + $required{'fw_jump.elf'} = 1 if $architecture eq 'riscv64'; + + my @files = _regular_files($directory); + my %files = map { $_ => 1 } @files; + for my $name (sort keys %required) { + die "Genesis export is missing $name\n" unless $files{$name}; + } + for my $name (@files) { + die "Unexpected Genesis export file: $name\n" unless $required{$name}; + } + + my $manifest = _read_key_values( + "$directory/xcat-genesis.manifest", + { map { $_ => 1 } qw(format version architecture) }, + ); + die "Unsupported Genesis export format\n" + unless ($manifest->{format} // '') eq 'xcat-genesis'; + die "Unsupported Genesis export version\n" + unless ($manifest->{version} // '') eq '1'; + die "Genesis export architecture mismatch\n" + unless ($manifest->{architecture} // '') eq $architecture; + + _verify_checksums($directory); + return 1; +} + +sub read_release_manifest { + my ($directory) = @_; + my $values = _read_key_values( + "$directory/release.manifest", + { + map { $_ => 1 } qw( + format version xcat_version xcat_release xcat_revision + source_date_epoch architectures formats + ) + }, + ); + for my $key (qw(format version xcat_version xcat_release xcat_revision source_date_epoch architectures formats)) { + die "Release manifest is missing $key\n" unless exists($values->{$key}); + } + return $values; +} + +sub validate_release { + my ($directory) = @_; + die "Invalid Genesis package release: $directory\n" + unless -d $directory && !-l $directory; + + my $manifest = read_release_manifest($directory); + die "Unsupported Genesis package release format\n" + unless $manifest->{format} eq 'xcat-genesis-packages'; + die "Unsupported Genesis package release version\n" + unless $manifest->{version} eq '1'; + die "Invalid xCAT version in release manifest\n" + unless $manifest->{xcat_version} =~ /^\d+(?:\.\d+){1,3}$/; + die "Invalid xCAT release in release manifest\n" + unless $manifest->{xcat_release} =~ /^[A-Za-z0-9][A-Za-z0-9.+~]*$/; + die "Invalid xCAT revision in release manifest\n" + unless $manifest->{xcat_revision} =~ /^[0-9a-f]{40}$/; + die "Invalid source epoch in release manifest\n" + unless $manifest->{source_date_epoch} =~ /^\d+$/; + + my @architectures = split(/,/, $manifest->{architectures}); + my %seen_arch; + for my $architecture (@architectures) { + validate_architecture($architecture); + die "Duplicate release architecture: $architecture\n" if $seen_arch{$architecture}++; + } + die "Release manifest has no architectures\n" unless @architectures; + + my @formats = split(/,/, $manifest->{formats}); + my %seen_format; + for my $format (@formats) { + die "Unsupported package format: $format\n" unless $format eq 'rpm' || $format eq 'deb'; + die "Duplicate package format: $format\n" if $seen_format{$format}++; + } + die "Release manifest has no package formats\n" unless @formats; + + _verify_checksums($directory); + + my %expected = ( + 'release.manifest' => 1, + ); + for my $architecture (@architectures) { + my $rpm = rpm_package_name($architecture); + my $deb = deb_package_name($architecture); + if ($seen_format{rpm}) { + $expected{"rpm/$rpm-$manifest->{xcat_version}-$manifest->{xcat_release}.noarch.rpm"} = 1; + $expected{"srpm/$rpm-$manifest->{xcat_version}-$manifest->{xcat_release}.src.rpm"} = 1; + } + if ($seen_format{deb}) { + $expected{"deb/${deb}_$manifest->{xcat_version}-$manifest->{xcat_release}_all.deb"} = 1; + } + } + for my $file (grep { $_ ne 'SHA256SUMS' } _regular_files($directory)) { + die "Unexpected Genesis release artifact: $file\n" unless $expected{$file}; + delete($expected{$file}); + } + die "Genesis release is missing: " . join(', ', sort keys %expected) . "\n" + if %expected; + return $manifest; +} + +1; diff --git a/genesis-openembedded/package b/genesis-openembedded/package new file mode 100755 index 0000000..a12fb0c --- /dev/null +++ b/genesis-openembedded/package @@ -0,0 +1,297 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use Cwd qw(abs_path); +use Digest::MD5 (); +use File::Basename qw(basename dirname); +use File::Copy qw(copy); +use File::Path qw(make_path remove_tree); +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use Getopt::Long qw(GetOptions); +use lib "$FindBin::Bin/lib"; +use XCAT::GenesisRelease qw( + deb_package_name + rpm_package_name + validate_architecture + validate_export +); + +my ($architecture, $export_dir, $output_dir, $version, $release, $revision); +my $source_date_epoch; +my $format = 'all'; +my $rpm_top; + +END { + remove_tree($rpm_top) if defined($rpm_top) && -d $rpm_top && !-l $rpm_top; +} + +GetOptions( + 'architecture=s' => \$architecture, + 'export-dir=s' => \$export_dir, + 'output-dir=s' => \$output_dir, + 'version=s' => \$version, + 'release=s' => \$release, + 'revision=s' => \$revision, + 'source-date-epoch=i' => \$source_date_epoch, + 'format=s' => \$format, +) or die usage(); + +for my $required ( + [ architecture => $architecture ], + [ 'export-dir' => $export_dir ], + [ 'output-dir' => $output_dir ], + [ version => $version ], + [ release => $release ], + [ revision => $revision ], + [ 'source-date-epoch' => $source_date_epoch ], + ) +{ + die "Missing --$required->[0]\n" unless defined($required->[1]) && $required->[1] ne ''; +} + +validate_architecture($architecture); +die "Invalid version: $version\n" unless $version =~ /^\d+(?:\.\d+){1,3}$/; +die "Invalid release: $release\n" unless $release =~ /^[A-Za-z0-9][A-Za-z0-9.+~]*$/; +die "Invalid xcat-core revision: $revision\n" unless $revision =~ /^[0-9a-f]{40}$/; +die "Invalid source date epoch\n" unless $source_date_epoch =~ /^\d+$/; +die "Unsupported package format: $format\n" + unless $format eq 'all' || $format eq 'rpm' || $format eq 'deb'; + +$export_dir = abs_path($export_dir) or die "Cannot resolve export directory\n"; +validate_export($export_dir, $architecture); + +my $output_parent = dirname($output_dir); +make_path($output_parent) unless -d $output_parent; +$output_parent = abs_path($output_parent) or die "Cannot resolve output parent\n"; +$output_dir = "$output_parent/" . basename($output_dir); +die "Output already exists: $output_dir\n" if -e $output_dir || -l $output_dir; + +my $work = tempdir('xcat-genesis-package.XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $staging = tempdir('.xcat-genesis-output.XXXXXX', DIR => $output_parent, CLEANUP => 1); +my $rpm_name = rpm_package_name($architecture); +my $deb_name = deb_package_name($architecture); +my $source_name = "$rpm_name-$version"; +my $source_root = "$work/$source_name"; +make_path("$source_root/image"); + +for my $entry (_flat_files($export_dir)) { + copy("$export_dir/$entry", "$source_root/image/$entry") + or die "Cannot copy export file $entry: $!\n"; +} +_write_file("$source_root/xcat-core-revision", "$revision\n"); + +local $ENV{SOURCE_DATE_EPOCH} = $source_date_epoch; + +if ($format eq 'all' || $format eq 'rpm') { + _require_command('rpmbuild'); + _build_rpm($work, $staging, $source_root, $source_name, $rpm_name); +} +if ($format eq 'all' || $format eq 'deb') { + _require_command('dpkg-deb'); + _build_deb($work, $staging, $source_root, $deb_name); +} + +rename($staging, $output_dir) or die "Cannot publish $output_dir: $!\n"; +print "Packaged Genesis $architecture in $output_dir\n"; + +sub _build_rpm { + my ($work, $staging, $source_root, $source_name, $package_name) = @_; + my $candidate = "/var/tmp/xcat-genesis-rpmbuild-$revision-$architecture-$version-$release"; + die "RPM work directory already exists: $candidate\n" if -e $candidate || -l $candidate; + mkdir($candidate, 0700) or die "Cannot create RPM work directory $candidate: $!\n"; + $rpm_top = $candidate; + make_path(map { "$rpm_top/$_" } qw(BUILD BUILDROOT RPMS SOURCES SPECS SRPMS)); + make_path("$staging/rpm", "$staging/srpm"); + + my $archive = "$rpm_top/SOURCES/$source_name.tar.gz"; + _run( + 'tar', '--sort=name', '--owner=0', '--group=0', + "--mtime=\@$source_date_epoch", '--use-compress-program=gzip -n', + '-cf', $archive, '-C', dirname($source_root), basename($source_root), + ); + my $spec = "$FindBin::Bin/rpm/xCAT-genesis-base.spec"; + copy($spec, "$rpm_top/SPECS/xCAT-genesis-base.spec") + or die "Cannot stage RPM spec: $!\n"; + _run( + 'rpmbuild', '-ba', + '--define', "_topdir $rpm_top", + '--define', "genesis_arch $architecture", + '--define', "version $version", + '--define', "release $release", + '--define', "_buildtime $source_date_epoch", + '--define', '_buildhost xcat-release', + '--define', 'use_source_date_epoch_as_buildtime 1', + '--define', 'clamp_mtime_to_source_date_epoch 1', + '--define', '_binary_payload w9.gzdio', + '--define', '_source_payload w9.gzdio', + "$rpm_top/SPECS/xCAT-genesis-base.spec", + ); + + my $binary = "$rpm_top/RPMS/noarch/$package_name-$version-$release.noarch.rpm"; + my $source = "$rpm_top/SRPMS/$package_name-$version-$release.src.rpm"; + die "RPM build did not produce $binary\n" unless -f $binary; + die "RPM build did not produce $source\n" unless -f $source; + copy($binary, "$staging/rpm/" . basename($binary)) or die "Cannot collect RPM: $!\n"; + copy($source, "$staging/srpm/" . basename($source)) or die "Cannot collect SRPM: $!\n"; + remove_tree($rpm_top); + undef($rpm_top); +} + +sub _build_deb { + my ($work, $staging, $source_root, $package_name) = @_; + my $root = "$work/deb-root"; + my $image = "$root/opt/xcat/share/xcat/netboot/genesis/$architecture"; + my $doc = "$root/usr/share/doc/$package_name"; + make_path("$root/DEBIAN", $image, $doc, "$staging/deb"); + + for my $entry (_flat_files("$source_root/image")) { + copy("$source_root/image/$entry", "$image/$entry") + or die "Cannot stage DEB image file $entry: $!\n"; + chmod(0644, "$image/$entry") or die "Cannot set mode on $entry: $!\n"; + } + copy("$source_root/xcat-core-revision", "$doc/xcat-core-revision") + or die "Cannot stage DEB revision: $!\n"; + chmod(0644, "$doc/xcat-core-revision") or die "Cannot set revision mode: $!\n"; + + my $installed_kib = _tree_bytes($root); + $installed_kib = int(($installed_kib + 1023) / 1024); + _write_file( + "$root/DEBIAN/control", + "Package: $package_name\n" + . "Version: $version-$release\n" + . "Section: admin\nPriority: optional\nArchitecture: all\n" + . "Installed-Size: $installed_kib\n" + . "Maintainer: xCAT \n" + . "Description: xCAT Genesis netboot image\n" + . " The OpenEmbedded Genesis image used for node discovery, inventory,\n" + . " and service actions.\n", + ); + _write_file( + "$root/DEBIAN/postinst", + "#!/bin/sh\nset -e\n" + . "if [ \"\${1:-}\" = configure ]; then\n" + . " mkdir -p /etc/xcat\n" + . " touch /etc/xcat/genesis-base-updated\n" + . " if [ -x /opt/xcat/sbin/mknb ] && [ -r /proc/cmdline ] && \\\n" + . " [ \"\$(stat -c '%i %d' / 2>/dev/null)\" = \"\$(stat -c '%i %d' /proc/1/root/. 2>/dev/null)\" ]; then\n" + . " if /opt/xcat/sbin/mknb '$architecture'; then\n" + . " rm -f /etc/xcat/genesis-base-updated\n" + . " else\n" + . " echo 'mknb $architecture was deferred; run it after xcatd is ready' >&2\n" + . " fi\n" + . " fi\nfi\nexit 0\n", + ); + chmod(0755, "$root/DEBIAN/postinst") or die "Cannot make postinst executable: $!\n"; + _write_deb_md5sums($root); + _normalize_mtime($root, $source_date_epoch); + + my $deb = "$staging/deb/${package_name}_${version}-${release}_all.deb"; + _run('dpkg-deb', '--root-owner-group', '-Zgzip', '-z9', '--build', $root, $deb); + die "DEB build did not produce $deb\n" unless -f $deb; +} + +sub _write_deb_md5sums { + my ($root) = @_; + my @files; + require File::Find; + File::Find::find( + { + no_chdir => 1, + wanted => sub { + return unless -f $_ && !-l $_; + return if $File::Find::name =~ m{\Q$root/DEBIAN/\E}; + my $relative = File::Spec->abs2rel($File::Find::name, $root); + $relative =~ tr{\\}{/}; + push(@files, $relative); + }, + }, + $root, + ); + my $content = ''; + for my $relative (sort @files) { + open(my $fh, '<:raw', "$root/$relative") + or die "Cannot read $root/$relative: $!\n"; + my $digest = Digest::MD5->new->addfile($fh)->hexdigest; + close($fh) or die "Cannot close $root/$relative: $!\n"; + $content .= "$digest $relative\n"; + } + _write_file("$root/DEBIAN/md5sums", $content); +} + +sub _normalize_mtime { + my ($root, $epoch) = @_; + require File::Find; + File::Find::find( + { + no_chdir => 1, + wanted => sub { + utime($epoch, $epoch, $File::Find::name) + or die "Cannot set timestamp on $File::Find::name: $!\n"; + }, + }, + $root, + ); +} + +sub _flat_files { + my ($directory) = @_; + opendir(my $dh, $directory) or die "Cannot read $directory: $!\n"; + my @entries = sort grep { $_ ne '.' && $_ ne '..' } readdir($dh); + closedir($dh) or die "Cannot close $directory: $!\n"; + for my $entry (@entries) { + my $path = "$directory/$entry"; + die "Expected a regular file: $path\n" unless -f $path && !-l $path; + } + return @entries; +} + +sub _tree_bytes { + my ($directory) = @_; + my $bytes = 0; + require File::Find; + File::Find::find( + { no_chdir => 1, wanted => sub { $bytes += -s $_ if -f $_ && !-l $_ } }, + $directory, + ); + return $bytes; +} + +sub _require_command { + my ($command) = @_; + for my $directory (File::Spec->path()) { + return if -x "$directory/$command"; + } + die "Required command not found: $command\n"; +} + +sub _run { + my (@command) = @_; + print '+ ', join(' ', map { _display_quote($_) } @command), "\n"; + system(@command) == 0 or die "Command failed: $command[0]\n"; +} + +sub _display_quote { + my ($value) = @_; + return $value if $value =~ /^[A-Za-z0-9_.,+\/:=@~-]+$/; + $value =~ s/'/'\\''/g; + return "'$value'"; +} + +sub _write_file { + my ($path, $content) = @_; + open(my $fh, '>:raw', $path) or die "Cannot write $path: $!\n"; + print {$fh} $content or die "Cannot write $path: $!\n"; + close($fh) or die "Cannot close $path: $!\n"; +} + +sub usage { + return <<'USAGE'; +Usage: package --architecture ARCH --export-dir DIR --output-dir DIR + --version VERSION --release RELEASE --revision COMMIT + --source-date-epoch EPOCH [--format all|rpm|deb] +USAGE +} diff --git a/genesis-openembedded/rpm/xCAT-genesis-base.spec b/genesis-openembedded/rpm/xCAT-genesis-base.spec new file mode 100644 index 0000000..12db225 --- /dev/null +++ b/genesis-openembedded/rpm/xCAT-genesis-base.spec @@ -0,0 +1,49 @@ +%global genesis_arch %{?genesis_arch}%{!?genesis_arch:x86_64} + +Name: xCAT-genesis-base-%{genesis_arch} +Version: %{?version}%{!?version:0} +Release: %{?release}%{!?release:1} +Epoch: 2 +Summary: xCAT Genesis netboot image +License: Various +URL: https://xcat.org/ +Source0: %{name}-%{version}.tar.gz +BuildArch: noarch +AutoReqProv: no + +%description +The OpenEmbedded Genesis image used by xCAT for discovery, inventory, and +service actions before a node boots its installed operating system. + +%prep +%setup -q + +%build + +%install +rm -rf %{buildroot} +install -d -m 0755 %{buildroot}/opt/xcat/share/xcat/netboot/genesis/%{genesis_arch} +install -m 0644 image/* %{buildroot}/opt/xcat/share/xcat/netboot/genesis/%{genesis_arch}/ +install -d -m 0755 %{buildroot}%{_docdir}/%{name} +install -m 0644 xcat-core-revision %{buildroot}%{_docdir}/%{name}/ + +%post +mkdir -p /etc/xcat +touch /etc/xcat/genesis-base-updated +if [ -x /opt/xcat/sbin/mknb ] && [ -r /proc/cmdline ] && \ + [ "$(stat -c '%%i %%d' / 2>/dev/null)" = "$(stat -c '%%i %%d' /proc/1/root/. 2>/dev/null)" ]; then + if /opt/xcat/sbin/mknb %{genesis_arch}; then + rm -f /etc/xcat/genesis-base-updated + else + echo "mknb %{genesis_arch} was deferred; run it after xcatd is ready" >&2 + fi +fi + +%files +%dir /opt/xcat +%dir /opt/xcat/share +%dir /opt/xcat/share/xcat +%dir /opt/xcat/share/xcat/netboot +%dir /opt/xcat/share/xcat/netboot/genesis +/opt/xcat/share/xcat/netboot/genesis/%{genesis_arch} +%doc %{_docdir}/%{name}/xcat-core-revision diff --git a/genesis-openembedded/verify-release b/genesis-openembedded/verify-release new file mode 100755 index 0000000..eeda243 --- /dev/null +++ b/genesis-openembedded/verify-release @@ -0,0 +1,96 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use Cwd qw(abs_path); +use File::Spec; +use FindBin; +use Getopt::Long qw(GetOptions); +use lib "$FindBin::Bin/lib"; +use XCAT::GenesisRelease qw( + deb_package_name + rpm_package_name + validate_release +); + +my $format = 'all'; +GetOptions('format=s' => \$format) or die usage(); +die usage() unless @ARGV == 1; +die "Unsupported verification format: $format\n" + unless $format eq 'all' || $format eq 'rpm' || $format eq 'deb'; + +my $directory = abs_path($ARGV[0]) or die "Cannot resolve release directory\n"; +my $manifest = validate_release($directory); +my %available = map { $_ => 1 } split(/,/, $manifest->{formats}); +my @formats = $format eq 'all' ? sort keys %available : ($format); +for my $required (@formats) { + die "Release does not contain $required packages\n" unless $available{$required}; +} + +my @architectures = split(/,/, $manifest->{architectures}); +for my $architecture (@architectures) { + if (grep { $_ eq 'rpm' } @formats) { + _verify_rpm($directory, $manifest, $architecture); + } + if (grep { $_ eq 'deb' } @formats) { + _verify_deb($directory, $manifest, $architecture); + } +} + +print "Verified Genesis package release: $directory\n"; + +sub _verify_rpm { + my ($directory, $manifest, $architecture) = @_; + _require_command('rpm'); + my $name = rpm_package_name($architecture); + my @packages = ( + [ "$directory/rpm/$name-$manifest->{xcat_version}-$manifest->{xcat_release}.noarch.rpm", 'noarch' ], + [ "$directory/srpm/$name-$manifest->{xcat_version}-$manifest->{xcat_release}.src.rpm", 'src' ], + ); + for my $package (@packages) { + my ($path, $expected_arch) = @{$package}; + my $metadata = _capture( + 'rpm', '-qp', '--qf', "%{NAME}\t%{VERSION}\t%{RELEASE}\t%{ARCH}\t%{EPOCHNUM}\n", $path, + ); + my $expected = join( + "\t", $name, $manifest->{xcat_version}, $manifest->{xcat_release}, $expected_arch, '2', + ); + die "Unexpected RPM identity for $path: $metadata\n" unless $metadata eq $expected; + } +} + +sub _verify_deb { + my ($directory, $manifest, $architecture) = @_; + _require_command('dpkg-deb'); + my $name = deb_package_name($architecture); + my $path = "$directory/deb/${name}_$manifest->{xcat_version}-$manifest->{xcat_release}_all.deb"; + my $metadata = join( + "\n", + map { _capture('dpkg-deb', '-f', $path, $_) } qw(Package Version Architecture), + ); + my $expected = join("\n", $name, "$manifest->{xcat_version}-$manifest->{xcat_release}", 'all'); + die "Unexpected DEB identity for $path: $metadata\n" unless $metadata eq $expected; +} + +sub _require_command { + my ($command) = @_; + for my $directory (File::Spec->path()) { + return if -x "$directory/$command"; + } + die "Required command not found: $command\n"; +} + +sub _capture { + my (@command) = @_; + open(my $fh, '-|', @command) or die "Cannot run $command[0]: $!\n"; + local $/; + my $output = <$fh> // ''; + close($fh) or die "Command failed: $command[0]\n"; + $output =~ s/\s+\z//; + return $output; +} + +sub usage { + return "Usage: verify-release [--format all|rpm|deb] RELEASE_DIRECTORY\n"; +} From 063ffc7724cebea8df04c27a6a5c48035c430edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:10:51 -0300 Subject: [PATCH 02/82] fix(genesis): verify source package identity --- genesis-openembedded/verify-release | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/genesis-openembedded/verify-release b/genesis-openembedded/verify-release index eeda243..5772f50 100755 --- a/genesis-openembedded/verify-release +++ b/genesis-openembedded/verify-release @@ -44,17 +44,22 @@ sub _verify_rpm { my ($directory, $manifest, $architecture) = @_; _require_command('rpm'); my $name = rpm_package_name($architecture); + my $source_name = "$name-$manifest->{xcat_version}-$manifest->{xcat_release}.src.rpm"; my @packages = ( - [ "$directory/rpm/$name-$manifest->{xcat_version}-$manifest->{xcat_release}.noarch.rpm", 'noarch' ], - [ "$directory/srpm/$name-$manifest->{xcat_version}-$manifest->{xcat_release}.src.rpm", 'src' ], + [ "$directory/rpm/$name-$manifest->{xcat_version}-$manifest->{xcat_release}.noarch.rpm", + 'noarch', $source_name ], + [ "$directory/srpm/$source_name", 'noarch', '(none)' ], ); for my $package (@packages) { - my ($path, $expected_arch) = @{$package}; + my ($path, $expected_arch, $expected_source) = @{$package}; my $metadata = _capture( - 'rpm', '-qp', '--qf', "%{NAME}\t%{VERSION}\t%{RELEASE}\t%{ARCH}\t%{EPOCHNUM}\n", $path, + 'rpm', '-qp', '--qf', + "%{NAME}\t%{VERSION}\t%{RELEASE}\t%{ARCH}\t%{EPOCHNUM}\t%{SOURCERPM}\n", + $path, ); my $expected = join( - "\t", $name, $manifest->{xcat_version}, $manifest->{xcat_release}, $expected_arch, '2', + "\t", $name, $manifest->{xcat_version}, $manifest->{xcat_release}, + $expected_arch, '2', $expected_source, ); die "Unexpected RPM identity for $path: $metadata\n" unless $metadata eq $expected; } From badcaedef0ad5bee675dd2687bb5d16849b05bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:11:01 -0300 Subject: [PATCH 03/82] test(genesis): cover OpenEmbedded packages --- t/genesis_openembedded_release.t | 269 +++++++++++++++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 t/genesis_openembedded_release.t diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t new file mode 100644 index 0000000..b69f4cc --- /dev/null +++ b/t/genesis_openembedded_release.t @@ -0,0 +1,269 @@ +use strict; +use warnings; + +use Cwd qw(abs_path); +use Digest::SHA (); +use File::Basename qw(dirname); +use File::Copy qw(copy); +use File::Find qw(find); +use File::Path qw(make_path); +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use Test::More; + +use lib "$FindBin::Bin/../genesis-openembedded/lib"; +use XCAT::GenesisRelease qw( + architectures + deb_package_name + rpm_package_name + validate_architecture + validate_export + validate_release +); + +my $repo_root = abs_path("$FindBin::Bin/.."); +my $packager = "$repo_root/genesis-openembedded/package"; +my $verifier = "$repo_root/genesis-openembedded/verify-release"; +my $revision = 'a' x 40; +my $version = '2.19.0'; +my $release = 'snap202608210726'; +my $epoch = 1787293573; + +is_deeply( + [ architectures() ], + [ qw(x86 x86_64 ppc64 ppc64le armv7hf aarch64 riscv64) ], + 'supported architectures keep their exact xCAT names', +); +is(rpm_package_name('ppc64le'), 'xCAT-genesis-base-ppc64le', + 'RPM package keeps ppc64le distinct'); +is(deb_package_name('x86_64'), 'xcat-genesis-base-x86-64', + 'DEB package uses a legal spelling of x86_64'); +dies_like(sub { validate_architecture('ppc') }, qr/Unsupported Genesis architecture/, + 'legacy ppc alias is rejected'); + +my $tmp = tempdir(CLEANUP => 1); +my $export = make_export("$tmp/export", 'x86_64'); +ok(validate_export($export, 'x86_64'), 'valid export passes'); +dies_like(sub { validate_export($export, 'ppc64le') }, qr/architecture mismatch/, + 'wrong architecture fails'); + +my $missing = make_export("$tmp/missing", 'x86_64'); +unlink("$missing/image.vex.json") or die $!; +write_checksums($missing); +dies_like(sub { validate_export($missing, 'x86_64') }, qr/missing image\.vex\.json/, + 'missing release evidence fails'); + +my $corrupt = make_export("$tmp/corrupt", 'x86_64'); +write_file("$corrupt/kernel", 'changed'); +dies_like(sub { validate_export($corrupt, 'x86_64') }, qr/Checksum mismatch for kernel/, + 'corrupt payload fails'); + +my $unexpected = make_export("$tmp/unexpected", 'x86_64'); +write_file("$unexpected/extra", 'not part of the export'); +write_checksums($unexpected); +dies_like(sub { validate_export($unexpected, 'x86_64') }, qr/Unexpected Genesis export file/, + 'unlisted export files fail'); + +my $linked = make_export("$tmp/linked", 'x86_64'); +unlink("$linked/kernel") or die $!; +symlink('initramfs.cpio.gz', "$linked/kernel") or die $!; +dies_like(sub { validate_export($linked, 'x86_64') }, qr/Symbolic links are not allowed/, + 'export symlinks fail'); + +my $riscv = make_export("$tmp/riscv", 'riscv64'); +ok(-f "$riscv/fw_jump.elf", 'RISC-V export carries firmware'); +ok(validate_export($riscv, 'riscv64'), 'RISC-V export passes'); + +my $release_dir = "$tmp/release"; +make_path("$release_dir/rpm", "$release_dir/srpm", "$release_dir/deb"); +for my $architecture (qw(x86_64 ppc64le)) { + my $rpm = rpm_package_name($architecture); + my $deb = deb_package_name($architecture); + write_file("$release_dir/rpm/$rpm-$version-$release.noarch.rpm", "rpm $architecture"); + write_file("$release_dir/srpm/$rpm-$version-$release.src.rpm", "srpm $architecture"); + write_file("$release_dir/deb/${deb}_${version}-${release}_all.deb", "deb $architecture"); +} +write_release_manifest($release_dir, 'x86_64,ppc64le', 'deb,rpm'); +write_checksums($release_dir); +my $manifest = validate_release($release_dir); +is($manifest->{xcat_revision}, $revision, 'release records xcat-core revision'); + +my $bad_release = "$tmp/bad-release"; +copy_tree($release_dir, $bad_release); +write_file("$bad_release/rpm/stale.rpm", 'stale'); +write_checksums($bad_release); +dies_like(sub { validate_release($bad_release) }, qr/Unexpected Genesis release artifact/, + 'stale package fails'); + +my $missing_release = "$tmp/missing-release"; +copy_tree($release_dir, $missing_release); +unlink("$missing_release/rpm/xCAT-genesis-base-ppc64le-$version-$release.noarch.rpm") or die $!; +write_checksums($missing_release); +dies_like(sub { validate_release($missing_release) }, qr/Genesis release is missing/, + 'incomplete architecture set fails'); + +SKIP: { + skip 'rpmbuild and rpm are not installed', 8 + unless command_exists('rpmbuild') && command_exists('rpm'); + exercise_packager('rpm'); +} + +SKIP: { + skip 'dpkg-deb is not installed', 8 unless command_exists('dpkg-deb'); + exercise_packager('deb'); +} + +done_testing(); + +sub exercise_packager { + my ($format) = @_; + my $first = "$tmp/$format-first"; + my $second = "$tmp/$format-second"; + my @command = ( + $packager, + '--architecture', 'x86_64', + '--export-dir', $export, + '--version', $version, + '--release', $release, + '--revision', $revision, + '--source-date-epoch', $epoch, + '--format', $format, + ); + + is(system(@command, '--output-dir', $first), 0, "$format package builds"); + is(system(@command, '--output-dir', $second), 0, "$format package rebuilds"); + + my ($relative, $source_relative); + if ($format eq 'rpm') { + $relative = "rpm/xCAT-genesis-base-x86_64-$version-$release.noarch.rpm"; + $source_relative = "srpm/xCAT-genesis-base-x86_64-$version-$release.src.rpm"; + } else { + $relative = "deb/xcat-genesis-base-x86-64_${version}-${release}_all.deb"; + } + ok(-f "$first/$relative", "$format binary exists"); + is(file_sha("$first/$relative"), file_sha("$second/$relative"), + "$format binary is reproducible"); + if ($format eq 'rpm') { + ok(-f "$first/$source_relative", 'source RPM exists'); + is(file_sha("$first/$source_relative"), file_sha("$second/$source_relative"), + 'source RPM is reproducible'); + } else { + pass('DEB has no separate source package'); + pass('DEB source is carried by the RPM source archive'); + } + + my $release_root = "$tmp/$format-release"; + make_path($release_root); + copy_tree($first, $release_root); + write_release_manifest($release_root, 'x86_64', $format); + write_checksums($release_root); + ok(validate_release($release_root), "$format release layout passes"); + is(system($verifier, '--format', $format, $release_root), 0, + "$format package metadata passes"); +} + +sub make_export { + my ($directory, $architecture) = @_; + make_path($directory); + my %content = ( + 'kernel' => 'kernel', + 'initramfs.cpio.gz' => 'initramfs', + 'image.manifest' => 'packages', + 'image.spdx.json' => '{}', + 'image.vex.json' => '{}', + 'license.manifest' => 'licenses', + 'xcat-genesis.manifest' => "format=xcat-genesis\nversion=1\narchitecture=$architecture\n", + ); + $content{'fw_jump.elf'} = 'firmware' if $architecture eq 'riscv64'; + write_file("$directory/$_", $content{$_}) for sort keys %content; + write_checksums($directory); + return $directory; +} + +sub write_release_manifest { + my ($directory, $architectures, $formats) = @_; + write_file( + "$directory/release.manifest", + "format=xcat-genesis-packages\n" + . "version=1\n" + . "xcat_version=$version\n" + . "xcat_release=$release\n" + . "xcat_revision=$revision\n" + . "source_date_epoch=$epoch\n" + . "architectures=$architectures\n" + . "formats=$formats\n", + ); +} + +sub write_checksums { + my ($directory) = @_; + unlink("$directory/SHA256SUMS") if -e "$directory/SHA256SUMS"; + my @files; + find( + { + no_chdir => 1, + wanted => sub { + return unless -f $_ && !-l $_; + my $relative = File::Spec->abs2rel($File::Find::name, $directory); + $relative =~ tr{\\}{/}; + push(@files, $relative); + }, + }, + $directory, + ); + my $content = join('', map { file_sha("$directory/$_") . " $_\n" } sort @files); + write_file("$directory/SHA256SUMS", $content); +} + +sub file_sha { + my ($path) = @_; + open(my $fh, '<:raw', $path) or die $!; + my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest; + close($fh) or die $!; + return $digest; +} + +sub copy_tree { + my ($source, $destination) = @_; + make_path($destination); + find( + { + no_chdir => 1, + wanted => sub { + return if $File::Find::name eq $source; + my $relative = File::Spec->abs2rel($File::Find::name, $source); + my $target = "$destination/$relative"; + if (-d $File::Find::name) { + make_path($target); + } elsif (-f $File::Find::name) { + make_path(dirname($target)); + copy($File::Find::name, $target) or die $!; + } + }, + }, + $source, + ); +} + +sub write_file { + my ($path, $content) = @_; + open(my $fh, '>:raw', $path) or die $!; + print {$fh} $content or die $!; + close($fh) or die $!; +} + +sub command_exists { + my ($command) = @_; + for my $directory (File::Spec->path()) { + return 1 if -x "$directory/$command"; + } + return 0; +} + +sub dies_like { + my ($code, $pattern, $name) = @_; + my $error = ''; + eval { $code->(); 1 } or $error = $@; + like($error, $pattern, $name); +} From df438781116fac0b1ae22d41ef310003efdec26a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:21:33 -0300 Subject: [PATCH 04/82] feat(build): consume Genesis package releases --- build-apt-repo.sh | 30 +++++++++++++++++++- mockbuild-all.pl | 71 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/build-apt-repo.sh b/build-apt-repo.sh index 7a2b8ca..b3636b2 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -12,6 +12,7 @@ APT_DIR="" GPG_KEY_ID="xcat@megware.com" SKIP_SIGN=0 DRY_RUN=0 +GENESIS_RELEASE="" declare -A CODENAME_MAP=( [ubuntu22.04]=jammy @@ -44,6 +45,7 @@ Options: --apt-dir PATH APT output directory (default: /repos/apt) --gpg-key-id ID GPG key ID for signing (default: xcat@megware.com) --skip-sign Skip GPG signing (for testing) + --genesis-release PATH Add a verified OpenEmbedded Genesis DEB release to each selected suite --dry-run Print planned actions without executing -h, --help Show this help @@ -70,6 +72,7 @@ while [[ $# -gt 0 ]]; do --apt-dir) APT_DIR="$2"; shift 2 ;; --gpg-key-id) GPG_KEY_ID="$2"; shift 2 ;; --skip-sign) SKIP_SIGN=1; shift ;; + --genesis-release) GENESIS_RELEASE="$2"; shift 2 ;; --dry-run) DRY_RUN=1; shift ;; -h|--help) usage; exit 0 ;; -*) die "Unknown option: $1" ;; @@ -80,6 +83,15 @@ done REPO_ROOT="$(cd "$REPO_ROOT" && pwd)" APT_DIR="${APT_DIR:-$REPO_ROOT/repos/apt}" +if [[ -n "$GENESIS_RELEASE" ]]; then + [[ -d "$GENESIS_RELEASE" ]] || die "Genesis release directory not found: $GENESIS_RELEASE" + GENESIS_RELEASE="$(cd "$GENESIS_RELEASE" && pwd)" + verifier="$SCRIPT_DIR/genesis-openembedded/verify-release" + [[ -x "$verifier" ]] || die "Genesis release verifier not found: $verifier" + "$verifier" --format deb "$GENESIS_RELEASE" + echo "Genesis release: $GENESIS_RELEASE" +fi + # Default to all known versions when no DIST arg was given; otherwise validate # each requested version against CODENAME_MAP. if [[ ${#SELECTED_VERS[@]} -eq 0 ]]; then @@ -144,6 +156,17 @@ done step "Populating pool" +copy_deb() { + local source="$1" + local destination="$2/$(basename "$source")" + if [[ -e "$destination" ]]; then + cmp -s "$source" "$destination" \ + || die "Package collision with different content: $destination" + return + fi + ln "$source" "$destination" 2>/dev/null || cp "$source" "$destination" +} + for ver in "${SELECTED_VERS[@]}"; do codename="${CODENAME_MAP[$ver]}" src="$APT_DIR/$ver" @@ -151,8 +174,13 @@ for ver in "${SELECTED_VERS[@]}"; do echo "$ver -> pool/main/$codename/" if [[ $DRY_RUN -eq 0 ]]; then for deb in "$src"/*.deb; do - ln "$deb" "$dst/" 2>/dev/null || cp "$deb" "$dst/" + copy_deb "$deb" "$dst" done + if [[ -n "$GENESIS_RELEASE" ]]; then + for deb in "$GENESIS_RELEASE"/deb/*.deb; do + copy_deb "$deb" "$dst" + done + fi fi done diff --git a/mockbuild-all.pl b/mockbuild-all.pl index f1be0b0..9642612 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -36,6 +36,7 @@ my $skip_xcat = 0; my $skip_genesis = 0; my $skip_createrepo = 0; my $skip_tarball = 0; +my $genesis_release = ''; my $scrub_all_chroots = 0; my $dry_run = 0; my @extra_collect_dirs; @@ -72,6 +73,7 @@ GetOptions( 'skip-genesis!' => \$skip_genesis, 'skip-createrepo!' => \$skip_createrepo, 'skip-tarball!' => \$skip_tarball, + 'genesis-release=s' => \$genesis_release, 'scrub-all-chroots!' => \$scrub_all_chroots, 'collect-dir=s@' => \@extra_collect_dirs, 'dry-run!' => \$dry_run, @@ -135,6 +137,18 @@ require_command('mock') if $scrub_all_chroots; require_command('rpmsign') if $gpg_sign; require_command('gpg') if $gpg_sign; +if ($genesis_release ne '') { + require_command('cmp'); + $genesis_release = abs_path($genesis_release) + or die "Cannot resolve --genesis-release directory\n"; + die "Genesis release directory not found: $genesis_release\n" + unless -d $genesis_release; + my $verifier = "$repo_root/genesis-openembedded/verify-release"; + die "Genesis release verifier not found: $verifier\n" unless -x $verifier; + run_argv($^X, $verifier, '--format', 'rpm', $genesis_release); + $skip_genesis = 1; +} + # An explicit --target builds just that target; otherwise build the current host # arch across rh8/rh9/rh10 into a deployable per-EL xcat-dep repo. This script builds # ONLY the host arch (uname -m) -- the other arch is produced on its own build host. @@ -154,6 +168,7 @@ print "lock: $output_base/.lock (held)\n"; print "gpg_sign: $gpg_sign\n"; print "gpg_key_name: $gpg_key_name\n" if $gpg_sign; print "gpg_home: " . ($gpg_home ne '' ? $gpg_home : '(default keyring)') . "\n" if $gpg_sign; +print "genesis_release: " . ($genesis_release || '(legacy builder)') . "\n"; # Build (and deploy) EL targets concurrently. Each target is fully isolated -- distinct run_id # (target-folded), mock --uniqueext, /tmp work dir, xcat_src/dist/, and deploy dir @@ -262,6 +277,7 @@ print "skip_xcat_dep: $skip_xcat_dep\n"; print "skip_perl: $skip_perl\n"; print "skip_xcat: $skip_xcat\n"; print "skip_genesis: $skip_genesis\n"; +print "genesis_release: " . ($genesis_release || '(none)') . "\n"; print "skip_install: $skip_install\n"; print "skip_createrepo: $skip_createrepo\n"; print "skip_tarball: $skip_tarball\n"; @@ -273,6 +289,7 @@ print "srpm_repo_dir: $srpm_repo_dir\n"; print "srpm_tarball: $srpm_tarball\n"; my @collect_roots; +push @collect_roots, "$genesis_release/rpm" if $genesis_release; if ($scrub_all_chroots) { run_step( @@ -439,6 +456,13 @@ push @collect_roots, @extra_collect_dirs; my @srpm_collect_roots = (!$skip_xcat) ? uniq(@collect_roots, $xcat_srpms_dir) : uniq(@collect_roots); +push @srpm_collect_roots, "$genesis_release/srpm" if $genesis_release; +@srpm_collect_roots = uniq(@srpm_collect_roots); + +if ($genesis_release && !$dry_run) { + remove_genesis_packages($repo_dir, 0); + remove_genesis_packages($srpm_repo_dir, 1); +} print_step('Collect RPM artifacts'); print "collection roots:\n"; @@ -475,6 +499,9 @@ my ($copied_srpms, $skipped_non_src, $missing_srpm_roots) = collect_srpms( dry_run => $dry_run, ); +assert_genesis_release_copied($repo_dir, $srpm_repo_dir) + if $genesis_release && !$dry_run; + if (!$dry_run && $copied_srpms == 0) { print "WARN: No source RPMs were collected. SRPM repo and tarball may be empty.\n"; } @@ -567,6 +594,7 @@ sub deploy_target { print_step("Deploy $tgt -> $dest"); return if $dry_run; make_path($dest); + remove_genesis_packages($dest, 0) if $genesis_release; for my $rpm (glob("$src/*.rpm")) { next if $rpm =~ /\.src\.rpm$/; copy($rpm, "$dest/" . basename($rpm)) @@ -698,6 +726,8 @@ Options: --skip-xcat Skip xCAT buildrpms.pl step --skip-createrepo Skip createrepo --skip-tarball Skip binary/SRPM tarball creation + --genesis-release PATH Consume a verified OpenEmbedded Genesis RPM release and skip the + legacy per-EL Genesis build --scrub-all-chroots Run mock -r --scrub=all before build/collect --collect-dir PATH Additional directory to scan recursively for RPMs (repeatable) --dry-run Print planned commands without executing @@ -733,6 +763,15 @@ sub run_simple { } } +sub run_argv { + my (@command) = @_; + my $rc = system(@command); + if ($rc != 0) { + my $exit = $rc == -1 ? 255 : ($rc >> 8); + die "Command failed (rc=$exit): @command\n"; + } +} + sub capture { my ($cmd) = @_; my $out = `$cmd`; @@ -879,6 +918,38 @@ sub assert_required_deps { print "[deps] required set present in $dir: @req\n"; } +sub remove_genesis_packages { + my ($directory, $source) = @_; + return unless -d $directory; + for my $path (glob("$directory/xCAT-genesis-base-*.rpm")) { + next if $source && $path !~ /\.src\.rpm\z/; + next if !$source && $path =~ /\.src\.rpm\z/; + unlink($path) or die "Cannot remove stale Genesis package $path: $!\n"; + } +} + +sub assert_genesis_release_copied { + my ($binary_directory, $source_directory) = @_; + for my $entry ( + [ "$genesis_release/rpm", $binary_directory ], + [ "$genesis_release/srpm", $source_directory ], + ) { + my ($source_root, $destination_root) = @{$entry}; + for my $source (glob("$source_root/*.rpm")) { + my $destination = "$destination_root/" . basename($source); + die "Genesis package was not collected: $destination\n" unless -f $destination; + die "Collected Genesis package differs from release: $destination\n" + unless files_match($source, $destination); + } + } +} + +sub files_match { + my ($left, $right) = @_; + return 0 unless -f $left && -f $right; + return system('cmp', '-s', $left, $right) == 0; +} + sub collect_rpms { my (%args) = @_; my $roots = $args{roots} // []; From acc972d524c09c5e067e522adfb90fe37e399076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:25:49 -0300 Subject: [PATCH 05/82] test(build): cover Genesis release consumers --- t/genesis_openembedded_consumer.t | 248 ++++++++++++++++++++++++++++++ t/genesis_openembedded_release.t | 130 +++------------- t/lib/XCAT/GenesisReleaseTest.pm | 133 ++++++++++++++++ 3 files changed, 399 insertions(+), 112 deletions(-) create mode 100644 t/genesis_openembedded_consumer.t create mode 100644 t/lib/XCAT/GenesisReleaseTest.pm diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t new file mode 100644 index 0000000..d04febf --- /dev/null +++ b/t/genesis_openembedded_consumer.t @@ -0,0 +1,248 @@ +use strict; +use warnings; + +use Cwd qw(abs_path); +use File::Basename qw(basename); +use File::Copy qw(copy); +use File::Path qw(make_path); +use File::Temp qw(tempdir); +use FindBin; +use Test::More; + +use lib "$FindBin::Bin/lib"; +use XCAT::GenesisReleaseTest qw( + command_exists + file_sha + make_export + write_checksums + write_file + write_release_manifest +); + +my $repo_root = abs_path("$FindBin::Bin/.."); +my $packager = "$repo_root/genesis-openembedded/package"; +my $rpm_consumer = "$repo_root/mockbuild-all.pl"; +my $deb_consumer = "$repo_root/build-apt-repo.sh"; +my $revision = 'b' x 40; +my $version = '2.19.0'; +my $release = 'snap202608210726'; +my $epoch = 1787293573; +my $tmp = tempdir(CLEANUP => 1); + +SKIP: { + skip 'RPM repository tools require a root Linux builder', 8 + unless $^O eq 'linux' + && $> == 0 + && command_exists('rpmbuild') + && command_exists('rpm') + && command_exists('createrepo_c') + && command_exists('cmp'); + test_rpm_consumer(); +} + +SKIP: { + skip 'APT repository tools are not installed', 8 + unless $^O eq 'linux' + && command_exists('bash') + && command_exists('dpkg-deb') + && command_exists('apt-ftparchive') + && command_exists('gpg'); + test_deb_consumer(); +} + +done_testing(); + +sub test_rpm_consumer { + my $release_root = make_package_release("$tmp/rpm", 'rpm'); + my $package = "xCAT-genesis-base-x86_64-$version-$release.noarch.rpm"; + my $source_package = "xCAT-genesis-base-x86_64-$version-$release.src.rpm"; + my $output = "$tmp/rpm-output"; + my $target = 'test+epel-10-' . capture('uname', '-m'); + my $run = "$target-consumer"; + my $run_repo = "$output/mockbuild-all/$run/repo/" . capture('uname', '-m'); + my $source_repo = "$output/mockbuild-all/$run/repo-src"; + my $deploy_repo = "$output/xcat-dep/rh10/" . capture('uname', '-m'); + make_path($run_repo, $source_repo, $deploy_repo); + write_file("$run_repo/xCAT-genesis-base-stale.noarch.rpm", 'stale'); + write_file("$source_repo/xCAT-genesis-base-stale.src.rpm", 'stale'); + write_file("$deploy_repo/xCAT-genesis-base-stale.noarch.rpm", 'stale'); + + my $dependencies = "$tmp/rpm-dependencies"; + make_path($dependencies); + for my $name (qw( + ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi + perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB + )) { + copy("$release_root/rpm/$package", "$dependencies/$name-1.noarch.rpm") + or die $!; + } + + my $stub = "$tmp/perl-stub/Parallel/ForkManager.pm"; + make_path("$tmp/perl-stub/Parallel"); + write_file( + $stub, + "package Parallel::ForkManager;\n" + . "sub new { bless {}, shift }\n" + . "sub run_on_finish { \$_[0]->{callback} = \$_[1] }\n" + . "sub start { 0 }\n" + . "sub finish { my (\$self, \$exit) = \@_; " + . "\$self->{callback}->(\$\$, \$exit, undef, 0, 0) if \$self->{callback}; 0 }\n" + . "sub wait_all_children { 0 }\n1;\n", + ); + + my @perl_lib = ("$tmp/perl-stub"); + push(@perl_lib, $ENV{PERL5LIB}) + if defined($ENV{PERL5LIB}) && $ENV{PERL5LIB} ne ''; + local $ENV{PERL5LIB} = join(':', @perl_lib); + my $log = "$tmp/rpm-consumer.log"; + my $status = run_capture( + $log, + $^X, $rpm_consumer, + '--repo-root', $repo_root, + '--output', $output, + '--target', $target, + '--run-id', 'consumer', + '--build-timestamp', $epoch, + '--skip-build', '--skip-xcat', '--skip-xcat-dep', '--skip-perl', + '--skip-createrepo', '--skip-tarball', + '--genesis-release', $release_root, + '--collect-dir', $dependencies, + ); + + is($status, 0, 'RPM repository accepts a verified Genesis release'); + is(file_sha("$deploy_repo/$package"), file_sha("$release_root/rpm/$package"), + 'deployed RPM matches the release'); + ok(!-e "$run_repo/xCAT-genesis-base-stale.noarch.rpm", + 'stale run RPM is removed'); + ok(!-e "$source_repo/xCAT-genesis-base-stale.src.rpm", + 'stale source RPM is removed'); + ok(!-e "$deploy_repo/xCAT-genesis-base-stale.noarch.rpm", + 'stale deployed RPM is removed'); + is(file_sha("$source_repo/$source_package"), + file_sha("$release_root/srpm/$source_package"), + 'source RPM matches the release'); + ok(-f "$deploy_repo/repodata/repomd.xml", 'RPM repository metadata is generated'); + like(read_file("$output/mockbuild-all/$run/summary.txt"), qr/^copied_rpms=8$/m, + 'release RPM is counted with required dependencies'); +} + +sub test_deb_consumer { + my $release_root = make_package_release("$tmp/deb", 'deb'); + my $package = "xcat-genesis-base-x86-64_${version}-${release}_all.deb"; + my $apt_root = "$tmp/apt"; + my $input = "$apt_root/ubuntu24.04"; + my $dummy = "$tmp/dummy-deb"; + make_path("$dummy/DEBIAN", $input, "$apt_root/pool/main/noble"); + write_file( + "$dummy/DEBIAN/control", + "Package: xcat-dep-test\nVersion: 1\nArchitecture: all\n" + . "Maintainer: xCAT \n" + . "Description: repository test package\n", + ); + die "Cannot build test DEB\n" + if run_capture( + "$tmp/dummy-deb.log", 'dpkg-deb', '--root-owner-group', '--build', + $dummy, "$input/xcat-dep-test_1_all.deb", + ); + write_file("$apt_root/pool/main/noble/xcat-genesis-base-stale.deb", 'stale'); + + local $ENV{SOURCE_DATE_EPOCH} = $epoch; + my $log = "$tmp/deb-consumer.log"; + my $status = run_capture( + $log, + 'bash', $deb_consumer, + '--repo-root', $repo_root, + '--apt-dir', $apt_root, + '--skip-sign', + '--genesis-release', $release_root, + 'ubuntu24.04', + ); + my $pool_package = "$apt_root/pool/main/noble/$package"; + my $amd64 = "$apt_root/dists/noble/main/binary-amd64/Packages"; + my $ppc64el = "$apt_root/dists/noble/main/binary-ppc64el/Packages"; + + is($status, 0, 'APT repository accepts a verified Genesis release'); + is(file_sha($pool_package), file_sha("$release_root/deb/$package"), + 'pooled DEB matches the release'); + ok(!-e "$apt_root/pool/main/noble/xcat-genesis-base-stale.deb", + 'stale pooled DEB is removed'); + like(read_file($amd64), qr/^Package: xcat-genesis-base-x86-64$/m, + 'all-architecture DEB is indexed for amd64'); + like(read_file($ppc64el), qr/^Package: xcat-genesis-base-x86-64$/m, + 'all-architecture DEB is indexed for ppc64el'); + ok(-f "$apt_root/dists/noble/Release", 'APT Release metadata is generated'); + + my $collision = "$tmp/apt-collision"; + make_path("$collision/ubuntu24.04"); + write_file("$collision/ubuntu24.04/$package", 'different'); + my $collision_log = "$tmp/deb-collision.log"; + my $collision_status = run_capture( + $collision_log, + 'bash', $deb_consumer, + '--repo-root', $repo_root, + '--apt-dir', $collision, + '--skip-sign', + '--genesis-release', $release_root, + 'ubuntu24.04', + ); + isnt($collision_status, 0, 'APT repository rejects a package collision'); + like(read_file($collision_log), qr/Package collision with different content/, + 'collision failure identifies the package'); +} + +sub make_package_release { + my ($root, $format) = @_; + my $export = make_export("$root/export", 'x86_64'); + my $release_root = "$root/release"; + die "Cannot package test release\n" + if run_capture( + "$root/package.log", + $packager, + '--architecture', 'x86_64', + '--export-dir', $export, + '--output-dir', $release_root, + '--version', $version, + '--release', $release, + '--revision', $revision, + '--source-date-epoch', $epoch, + '--format', $format, + ); + write_release_manifest( + $release_root, $version, $release, $revision, $epoch, 'x86_64', $format, + ); + write_checksums($release_root); + return $release_root; +} + +sub run_capture { + my ($log, @command) = @_; + my $pid = fork(); + die "Cannot fork: $!\n" unless defined $pid; + if ($pid == 0) { + open(STDOUT, '>:raw', $log) or die $!; + open(STDERR, '>&', STDOUT) or die $!; + exec(@command) or die "Cannot run $command[0]: $!\n"; + } + waitpid($pid, 0); + return 255 if $? == -1; + return 128 + ($? & 127) if $? & 127; + return $? >> 8; +} + +sub capture { + my (@command) = @_; + open(my $fh, '-|', @command) or die $!; + my $output = <$fh>; + close($fh) or die $!; + chomp($output); + return $output; +} + +sub read_file { + my ($path) = @_; + open(my $fh, '<:raw', $path) or die $!; + local $/; + my $content = <$fh> // ''; + close($fh) or die $!; + return $content; +} diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index b69f4cc..43266cf 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -2,17 +2,13 @@ use strict; use warnings; use Cwd qw(abs_path); -use Digest::SHA (); -use File::Basename qw(dirname); -use File::Copy qw(copy); -use File::Find qw(find); use File::Path qw(make_path); -use File::Spec; use File::Temp qw(tempdir); use FindBin; use Test::More; use lib "$FindBin::Bin/../genesis-openembedded/lib"; +use lib "$FindBin::Bin/lib"; use XCAT::GenesisRelease qw( architectures deb_package_name @@ -21,6 +17,16 @@ use XCAT::GenesisRelease qw( validate_export validate_release ); +use XCAT::GenesisReleaseTest qw( + command_exists + copy_tree + dies_like + file_sha + make_export + write_checksums + write_file + write_release_manifest +); my $repo_root = abs_path("$FindBin::Bin/.."); my $packager = "$repo_root/genesis-openembedded/package"; @@ -84,7 +90,10 @@ for my $architecture (qw(x86_64 ppc64le)) { write_file("$release_dir/srpm/$rpm-$version-$release.src.rpm", "srpm $architecture"); write_file("$release_dir/deb/${deb}_${version}-${release}_all.deb", "deb $architecture"); } -write_release_manifest($release_dir, 'x86_64,ppc64le', 'deb,rpm'); +write_release_manifest( + $release_dir, $version, $release, $revision, $epoch, + 'x86_64,ppc64le', 'deb,rpm', +); write_checksums($release_dir); my $manifest = validate_release($release_dir); is($manifest->{xcat_revision}, $revision, 'release records xcat-core revision'); @@ -156,114 +165,11 @@ sub exercise_packager { my $release_root = "$tmp/$format-release"; make_path($release_root); copy_tree($first, $release_root); - write_release_manifest($release_root, 'x86_64', $format); + write_release_manifest( + $release_root, $version, $release, $revision, $epoch, 'x86_64', $format, + ); write_checksums($release_root); ok(validate_release($release_root), "$format release layout passes"); is(system($verifier, '--format', $format, $release_root), 0, "$format package metadata passes"); } - -sub make_export { - my ($directory, $architecture) = @_; - make_path($directory); - my %content = ( - 'kernel' => 'kernel', - 'initramfs.cpio.gz' => 'initramfs', - 'image.manifest' => 'packages', - 'image.spdx.json' => '{}', - 'image.vex.json' => '{}', - 'license.manifest' => 'licenses', - 'xcat-genesis.manifest' => "format=xcat-genesis\nversion=1\narchitecture=$architecture\n", - ); - $content{'fw_jump.elf'} = 'firmware' if $architecture eq 'riscv64'; - write_file("$directory/$_", $content{$_}) for sort keys %content; - write_checksums($directory); - return $directory; -} - -sub write_release_manifest { - my ($directory, $architectures, $formats) = @_; - write_file( - "$directory/release.manifest", - "format=xcat-genesis-packages\n" - . "version=1\n" - . "xcat_version=$version\n" - . "xcat_release=$release\n" - . "xcat_revision=$revision\n" - . "source_date_epoch=$epoch\n" - . "architectures=$architectures\n" - . "formats=$formats\n", - ); -} - -sub write_checksums { - my ($directory) = @_; - unlink("$directory/SHA256SUMS") if -e "$directory/SHA256SUMS"; - my @files; - find( - { - no_chdir => 1, - wanted => sub { - return unless -f $_ && !-l $_; - my $relative = File::Spec->abs2rel($File::Find::name, $directory); - $relative =~ tr{\\}{/}; - push(@files, $relative); - }, - }, - $directory, - ); - my $content = join('', map { file_sha("$directory/$_") . " $_\n" } sort @files); - write_file("$directory/SHA256SUMS", $content); -} - -sub file_sha { - my ($path) = @_; - open(my $fh, '<:raw', $path) or die $!; - my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest; - close($fh) or die $!; - return $digest; -} - -sub copy_tree { - my ($source, $destination) = @_; - make_path($destination); - find( - { - no_chdir => 1, - wanted => sub { - return if $File::Find::name eq $source; - my $relative = File::Spec->abs2rel($File::Find::name, $source); - my $target = "$destination/$relative"; - if (-d $File::Find::name) { - make_path($target); - } elsif (-f $File::Find::name) { - make_path(dirname($target)); - copy($File::Find::name, $target) or die $!; - } - }, - }, - $source, - ); -} - -sub write_file { - my ($path, $content) = @_; - open(my $fh, '>:raw', $path) or die $!; - print {$fh} $content or die $!; - close($fh) or die $!; -} - -sub command_exists { - my ($command) = @_; - for my $directory (File::Spec->path()) { - return 1 if -x "$directory/$command"; - } - return 0; -} - -sub dies_like { - my ($code, $pattern, $name) = @_; - my $error = ''; - eval { $code->(); 1 } or $error = $@; - like($error, $pattern, $name); -} diff --git a/t/lib/XCAT/GenesisReleaseTest.pm b/t/lib/XCAT/GenesisReleaseTest.pm new file mode 100644 index 0000000..0ab3359 --- /dev/null +++ b/t/lib/XCAT/GenesisReleaseTest.pm @@ -0,0 +1,133 @@ +package XCAT::GenesisReleaseTest; + +use strict; +use warnings; + +use Digest::SHA (); +use Exporter qw(import); +use File::Basename qw(dirname); +use File::Copy qw(copy); +use File::Find qw(find); +use File::Path qw(make_path); +use File::Spec; +use Test::More (); + +our @EXPORT_OK = qw( + command_exists + copy_tree + dies_like + file_sha + make_export + write_checksums + write_file + write_release_manifest +); + +sub make_export { + my ($directory, $architecture) = @_; + make_path($directory); + my %content = ( + 'kernel' => 'kernel', + 'initramfs.cpio.gz' => 'initramfs', + 'image.manifest' => 'packages', + 'image.spdx.json' => '{}', + 'image.vex.json' => '{}', + 'license.manifest' => 'licenses', + 'xcat-genesis.manifest' => + "format=xcat-genesis\nversion=1\narchitecture=$architecture\n", + ); + $content{'fw_jump.elf'} = 'firmware' if $architecture eq 'riscv64'; + write_file("$directory/$_", $content{$_}) for sort keys %content; + write_checksums($directory); + return $directory; +} + +sub write_release_manifest { + my ($directory, $xcat_version, $xcat_release, $revision, $epoch, + $architectures, $formats) = @_; + write_file( + "$directory/release.manifest", + "format=xcat-genesis-packages\n" + . "version=1\n" + . "xcat_version=$xcat_version\n" + . "xcat_release=$xcat_release\n" + . "xcat_revision=$revision\n" + . "source_date_epoch=$epoch\n" + . "architectures=$architectures\n" + . "formats=$formats\n", + ); +} + +sub write_checksums { + my ($directory) = @_; + unlink("$directory/SHA256SUMS") if -e "$directory/SHA256SUMS"; + my @files; + find( + { + no_chdir => 1, + wanted => sub { + return unless -f $_ && !-l $_; + my $relative = File::Spec->abs2rel($File::Find::name, $directory); + $relative =~ tr{\\}{/}; + push(@files, $relative); + }, + }, + $directory, + ); + my $content = join('', map { file_sha("$directory/$_") . " $_\n" } sort @files); + write_file("$directory/SHA256SUMS", $content); +} + +sub file_sha { + my ($path) = @_; + open(my $fh, '<:raw', $path) or die $!; + my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest; + close($fh) or die $!; + return $digest; +} + +sub copy_tree { + my ($source, $destination) = @_; + make_path($destination); + find( + { + no_chdir => 1, + wanted => sub { + return if $File::Find::name eq $source; + my $relative = File::Spec->abs2rel($File::Find::name, $source); + my $target = "$destination/$relative"; + if (-d $File::Find::name) { + make_path($target); + } elsif (-f $File::Find::name) { + make_path(dirname($target)); + copy($File::Find::name, $target) or die $!; + } + }, + }, + $source, + ); +} + +sub write_file { + my ($path, $content) = @_; + open(my $fh, '>:raw', $path) or die $!; + print {$fh} $content or die $!; + close($fh) or die $!; +} + +sub command_exists { + my ($command) = @_; + for my $directory (File::Spec->path()) { + return 1 if -x "$directory/$command"; + } + return 0; +} + +sub dies_like { + my ($code, $pattern, $name) = @_; + my $error = ''; + eval { $code->(); 1 } or $error = $@; + Test::More::like($error, $pattern, $name); +} + +1; From d0cee4f00549a1b9193dff5071a0d126a6725372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:27:05 -0300 Subject: [PATCH 06/82] docs(genesis): document release packaging --- BUILD.md | 26 +++++++++++++++ genesis-openembedded/README.md | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 genesis-openembedded/README.md diff --git a/BUILD.md b/BUILD.md index 43fd88f..263b847 100644 --- a/BUILD.md +++ b/BUILD.md @@ -76,6 +76,11 @@ Use these flags to skip specific operations: - Skips `createrepo --update`. - `--skip-tarball` - Skips tarball creation for both binary and SRPM repos. +- `--skip-genesis` + - Skips the existing per-EL Genesis image build. +- `--genesis-release ` + - Adds a verified OpenEmbedded Genesis RPM release and skips the existing + per-EL Genesis image build for that run. - `--scrub-all-chroots` - Runs `mock -r --scrub=all` before build and collection. - `--collect-dir ` @@ -147,6 +152,27 @@ Notes: - Add `--skip-install` to skip those checks. - `` is optional; when omitted it is timestamp-based. +# Add an OpenEmbedded Genesis Release + +Build the Genesis package release first, following +[`genesis-openembedded/README.md`](genesis-openembedded/README.md). Then pass the +result to the regular repository build: + +```bash +cd +perl ./mockbuild-all.pl \ + --repo-root \ + --xcat-source \ + --genesis-release /path/to/xcat-genesis-release +``` + +The release is checked before any package is collected. Its RPMs and SRPMs are +copied into each generated EL repository. Old Genesis packages are removed from +those output directories so a previous run cannot satisfy the release by +accident. + +Omit `--genesis-release` to keep using the existing Genesis builder. + # Build Unified Repository Without xCAT (`--skip-xcat`) Use this mode to build dependency packages only and skip invoking `/root/xcat-dep/xcat-source-code/buildrpms.pl`. diff --git a/genesis-openembedded/README.md b/genesis-openembedded/README.md new file mode 100644 index 0000000..a10ab6b --- /dev/null +++ b/genesis-openembedded/README.md @@ -0,0 +1,59 @@ +# OpenEmbedded Genesis packages + +xCAT builds the Genesis image in `xcat-core`. This directory turns those build +outputs into packages and feeds them into the xcat-dep repositories. + +Run the release builder on a host that can build the OpenEmbedded layer. The +xcat-core checkout must be clean. `--xcat-ref` checks that the checkout points +to the intended commit; it does not change the checkout for you. + +```bash +./genesis-openembedded/build \ + --xcat-source /path/to/xcat-core \ + --xcat-ref \ + --all \ + --output-dir /path/to/xcat-genesis-release +``` + +The default format is `all`, which produces RPM, SRPM, and DEB packages. Use +`--format rpm` or `--format deb` when only one package family is needed. The +supported image architectures are `x86`, `x86_64`, `ppc64`, `ppc64le`, +`armv7hf`, `aarch64`, and `riscv64`. + +Each package installs one exact-architecture image under +`/opt/xcat/share/xcat/netboot/genesis//`. The packages themselves +are `noarch` or `all` because they are installed on the management node, not run +on the target node. + +The release directory contains: + +- `rpm/`, `srpm/`, and `deb/` package directories +- `release.manifest`, including the xcat-core commit +- `SHA256SUMS` + +Validate the directory before publishing it: + +```bash +./genesis-openembedded/verify-release /path/to/xcat-genesis-release +``` + +Pass that same directory to the repository builders: + +```bash +perl ./mockbuild-all.pl \ + --genesis-release /path/to/xcat-genesis-release \ + [other build options] + +./build-apt-repo.sh \ + --genesis-release /path/to/xcat-genesis-release \ + [DIST ...] +``` + +The RPM builder skips its old per-EL Genesis build when a release is supplied. +The APT builder adds the DEB packages to every selected suite. Both consumers +verify package identities and checksums before collecting packages. + +Without `--genesis-release`, both builders keep their existing behavior. This +allows the package publishing change to land before xcat-core starts depending +on the new architecture-specific package names. Generated images and packages +belong in release storage, not in Git. From ed1d72bc7780816abdd53cac6cc186ab3b72523a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:33:14 -0300 Subject: [PATCH 07/82] fix(genesis): make file listing scalar-safe --- genesis-openembedded/lib/XCAT/GenesisRelease.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/genesis-openembedded/lib/XCAT/GenesisRelease.pm b/genesis-openembedded/lib/XCAT/GenesisRelease.pm index 6bf70fc..07f4e0a 100644 --- a/genesis-openembedded/lib/XCAT/GenesisRelease.pm +++ b/genesis-openembedded/lib/XCAT/GenesisRelease.pm @@ -84,7 +84,8 @@ sub _regular_files { }, $root, ); - return sort @files; + my @sorted = sort @files; + return @sorted; } sub _read_checksums { From f8adc37a899c3d304b188687e456d42a3005b121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:33:15 -0300 Subject: [PATCH 08/82] fix(build): preserve basename failure status --- build-apt-repo.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-apt-repo.sh b/build-apt-repo.sh index b3636b2..a6e6546 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -158,7 +158,8 @@ step "Populating pool" copy_deb() { local source="$1" - local destination="$2/$(basename "$source")" + local destination + destination="$2/$(basename "$source")" if [[ -e "$destination" ]]; then cmp -s "$source" "$destination" \ || die "Package collision with different content: $destination" From 57f0db1ed31c01e947f6c8dce0d45ba1a07fd7fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:35:39 -0300 Subject: [PATCH 09/82] docs(genesis): record package transition gate --- genesis-openembedded/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/genesis-openembedded/README.md b/genesis-openembedded/README.md index a10ab6b..549ef77 100644 --- a/genesis-openembedded/README.md +++ b/genesis-openembedded/README.md @@ -53,7 +53,9 @@ The RPM builder skips its old per-EL Genesis build when a release is supplied. The APT builder adds the DEB packages to every selected suite. Both consumers verify package identities and checksums before collecting packages. -Without `--genesis-release`, both builders keep their existing behavior. This -allows the package publishing change to land before xcat-core starts depending -on the new architecture-specific package names. Generated images and packages -belong in release storage, not in Git. +Without `--genesis-release`, both builders keep their existing behavior. The +option remains opt-in until xcat-core uses the exact package names on every +platform. The current RPM naming already matches `x86_64`, but older mappings +still use `ppc64` for `ppc64le`, and the DEB packages use `amd64`. Those mappings +belong in a separate xcat-core change. Generated images and packages belong in +release storage, not in Git. From 87c769304a574851d7432e4683d0ee240e64f1e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:38:56 -0300 Subject: [PATCH 10/82] fix(genesis): reject untracked release sources --- genesis-openembedded/build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/genesis-openembedded/build b/genesis-openembedded/build index da39fe0..663827b 100755 --- a/genesis-openembedded/build +++ b/genesis-openembedded/build @@ -54,8 +54,8 @@ $xcat_source = abs_path($xcat_source) or die "Cannot resolve xcat-core source\n" for my $path (qw(Version xCAT-genesis-builder/oe/build xCAT-genesis-builder/oe/export)) { die "xcat-core source is missing $path\n" unless -f "$xcat_source/$path"; } -die "xcat-core has tracked changes\n" - if _capture('git', '-C', $xcat_source, 'status', '--porcelain', '--untracked-files=no') ne ''; +die "xcat-core checkout is not clean\n" + if _capture('git', '-C', $xcat_source, 'status', '--porcelain') ne ''; my $revision = _capture('git', '-C', $xcat_source, 'rev-parse', 'HEAD'); die "Invalid xcat-core revision: $revision\n" unless $revision =~ /^[0-9a-f]{40}$/; From a388bdacb50f2d0fc97a64cc862a90b03a4427c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:38:56 -0300 Subject: [PATCH 11/82] docs(build): list Genesis skip option --- mockbuild-all.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 9642612..6d589ed 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -724,6 +724,7 @@ Options: --skip-xcat-dep Skip xcat-dep mockbuild.pl package steps --skip-perl Skip perl package build step --skip-xcat Skip xCAT buildrpms.pl step + --skip-genesis Skip the existing per-EL Genesis image build --skip-createrepo Skip createrepo --skip-tarball Skip binary/SRPM tarball creation --genesis-release PATH Consume a verified OpenEmbedded Genesis RPM release and skip the From 98ee9edbfe9f2fc5f42040b13fbf3cb16522dbf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:38:57 -0300 Subject: [PATCH 12/82] test(genesis): cover dirty release sources --- t/genesis_openembedded_consumer.t | 26 ++---------------------- t/genesis_openembedded_release.t | 33 +++++++++++++++++++++++++++++++ t/lib/XCAT/GenesisReleaseTest.pm | 26 ++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 24 deletions(-) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index d04febf..7c336e4 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -14,6 +14,8 @@ use XCAT::GenesisReleaseTest qw( command_exists file_sha make_export + read_file + run_capture write_checksums write_file write_release_manifest @@ -214,21 +216,6 @@ sub make_package_release { return $release_root; } -sub run_capture { - my ($log, @command) = @_; - my $pid = fork(); - die "Cannot fork: $!\n" unless defined $pid; - if ($pid == 0) { - open(STDOUT, '>:raw', $log) or die $!; - open(STDERR, '>&', STDOUT) or die $!; - exec(@command) or die "Cannot run $command[0]: $!\n"; - } - waitpid($pid, 0); - return 255 if $? == -1; - return 128 + ($? & 127) if $? & 127; - return $? >> 8; -} - sub capture { my (@command) = @_; open(my $fh, '-|', @command) or die $!; @@ -237,12 +224,3 @@ sub capture { chomp($output); return $output; } - -sub read_file { - my ($path) = @_; - open(my $fh, '<:raw', $path) or die $!; - local $/; - my $content = <$fh> // ''; - close($fh) or die $!; - return $content; -} diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 43266cf..16de0d2 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -23,6 +23,8 @@ use XCAT::GenesisReleaseTest qw( dies_like file_sha make_export + read_file + run_capture write_checksums write_file write_release_manifest @@ -30,6 +32,7 @@ use XCAT::GenesisReleaseTest qw( my $repo_root = abs_path("$FindBin::Bin/.."); my $packager = "$repo_root/genesis-openembedded/package"; +my $builder = "$repo_root/genesis-openembedded/build"; my $verifier = "$repo_root/genesis-openembedded/verify-release"; my $revision = 'a' x 40; my $version = '2.19.0'; @@ -112,6 +115,36 @@ write_checksums($missing_release); dies_like(sub { validate_release($missing_release) }, qr/Genesis release is missing/, 'incomplete architecture set fails'); +SKIP: { + skip 'git is not installed', 2 unless command_exists('git'); + my $source = "$tmp/dirty-xcat-core"; + make_path("$source/xCAT-genesis-builder/oe"); + write_file("$source/Version", "$version\n"); + write_file("$source/xCAT-genesis-builder/oe/build", "#!/bin/sh\nexit 99\n"); + write_file("$source/xCAT-genesis-builder/oe/export", "#!/bin/sh\nexit 99\n"); + for my $command ( + [ 'git', '-C', $source, 'init', '-q' ], + [ 'git', '-C', $source, 'add', '.' ], + [ 'git', '-C', $source, '-c', 'user.name=xCAT test', + '-c', 'user.email=xcat-test@example.invalid', 'commit', '-qm', 'fixture' ], + ) { + die "Cannot prepare test repository\n" + if run_capture("$tmp/git-fixture.log", @{$command}); + } + write_file("$source/untracked", "not part of the commit\n"); + my $log = "$tmp/dirty-source.log"; + isnt( + run_capture( + $log, $builder, '--xcat-source', $source, + '--output-dir', "$tmp/dirty-output", + ), + 0, + 'release builder rejects untracked source files', + ); + like(read_file($log), qr/xcat-core checkout is not clean/, + 'dirty checkout failure is explicit'); +} + SKIP: { skip 'rpmbuild and rpm are not installed', 8 unless command_exists('rpmbuild') && command_exists('rpm'); diff --git a/t/lib/XCAT/GenesisReleaseTest.pm b/t/lib/XCAT/GenesisReleaseTest.pm index 0ab3359..a4f83ee 100644 --- a/t/lib/XCAT/GenesisReleaseTest.pm +++ b/t/lib/XCAT/GenesisReleaseTest.pm @@ -18,6 +18,8 @@ our @EXPORT_OK = qw( dies_like file_sha make_export + read_file + run_capture write_checksums write_file write_release_manifest @@ -123,6 +125,30 @@ sub command_exists { return 0; } +sub run_capture { + my ($log, @command) = @_; + my $pid = fork(); + die "Cannot fork: $!\n" unless defined $pid; + if ($pid == 0) { + open(STDOUT, '>:raw', $log) or die $!; + open(STDERR, '>&', STDOUT) or die $!; + exec(@command) or die "Cannot run $command[0]: $!\n"; + } + waitpid($pid, 0); + return 255 if $? == -1; + return 128 + ($? & 127) if $? & 127; + return $? >> 8; +} + +sub read_file { + my ($path) = @_; + open(my $fh, '<:raw', $path) or die $!; + local $/; + my $content = <$fh> // ''; + close($fh) or die $!; + return $content; +} + sub dies_like { my ($code, $pattern, $name) = @_; my $error = ''; From 6e499cd77b30c62d6e6b3eb9188a4162387aede7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:46:01 -0300 Subject: [PATCH 13/82] test(genesis): remove placeholder assertions --- t/genesis_openembedded_release.t | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 16de0d2..43fa56d 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -152,7 +152,7 @@ SKIP: { } SKIP: { - skip 'dpkg-deb is not installed', 8 unless command_exists('dpkg-deb'); + skip 'dpkg-deb is not installed', 6 unless command_exists('dpkg-deb'); exercise_packager('deb'); } @@ -190,9 +190,6 @@ sub exercise_packager { ok(-f "$first/$source_relative", 'source RPM exists'); is(file_sha("$first/$source_relative"), file_sha("$second/$source_relative"), 'source RPM is reproducible'); - } else { - pass('DEB has no separate source package'); - pass('DEB source is carried by the RPM source archive'); } my $release_root = "$tmp/$format-release"; From e3ff2eb1cf18b2c7dc67c548ed12820d377481ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:44:42 -0300 Subject: [PATCH 14/82] fix(genesis): isolate OpenEmbedded tmpdir --- genesis-openembedded/build | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/genesis-openembedded/build b/genesis-openembedded/build index 663827b..342fdea 100755 --- a/genesis-openembedded/build +++ b/genesis-openembedded/build @@ -84,9 +84,14 @@ die "Output already exists: $output_dir\n" if -e $output_dir || -l $output_dir; my $work = tempdir('xcat-genesis-release.XXXXXX', TMPDIR => 1, CLEANUP => 1); my $staging = tempdir('.xcat-genesis-release.XXXXXX', DIR => $output_parent, CLEANUP => 1); my $oe_work = "$work/openembedded"; +my $oe_tmp = "$oe_work/build/tmp"; local $ENV{XCAT_GENESIS_WORK_DIR} = $oe_work; -_run("$xcat_source/xCAT-genesis-builder/oe/build", @requested_architectures); -my $effective_deploy = "$oe_work/build/tmp/deploy"; +{ + local $ENV{TMPDIR} = $oe_tmp; + make_path($oe_tmp); + _run("$xcat_source/xCAT-genesis-builder/oe/build", @requested_architectures); +} +my $effective_deploy = "$oe_tmp/deploy"; die "Invalid OpenEmbedded deploy directory: $effective_deploy\n" unless -d $effective_deploy && !-l $effective_deploy; From faa72c14aa607acbb50c3ff4ce0979d882b03dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:45:31 -0300 Subject: [PATCH 15/82] test(genesis): cover isolated build tmpdir --- t/genesis_openembedded_release.t | 76 ++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 43fa56d..4fdcdb6 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -156,6 +156,12 @@ SKIP: { exercise_packager('deb'); } +SKIP: { + skip 'git and dpkg-deb are not installed', 4 + unless command_exists('git') && command_exists('dpkg-deb'); + exercise_builder_tmpdir(); +} + done_testing(); sub exercise_packager { @@ -203,3 +209,73 @@ sub exercise_packager { is(system($verifier, '--format', $format, $release_root), 0, "$format package metadata passes"); } + +sub exercise_builder_tmpdir { + my $source = "$tmp/tmpdir-xcat-core"; + my $oe = "$source/xCAT-genesis-builder/oe"; + make_path($oe); + write_file("$source/Version", "$version\n"); + write_file( + "$oe/build", + <<'BUILD', +#!/bin/sh +set -eu +expected=$XCAT_GENESIS_WORK_DIR/build/tmp +[ "${TMPDIR:-}" = "$expected" ] || exit 41 +mkdir -p "$TMPDIR/deploy" +BUILD + ); + write_file( + "$oe/export", + <<'EXPORT', +#!/bin/sh +set -eu +architecture=$1 +deploy=$2 +output=$3 +[ "$deploy" = "$XCAT_GENESIS_WORK_DIR/build/tmp/deploy" ] || exit 42 +mkdir -p "$output" +printf '%s\n' kernel >"$output/kernel" +printf '%s\n' initramfs >"$output/initramfs.cpio.gz" +printf '%s\n' packages >"$output/image.manifest" +printf '%s\n' '{}' >"$output/image.spdx.json" +printf '%s\n' '{}' >"$output/image.vex.json" +printf '%s\n' licenses >"$output/license.manifest" +printf 'format=xcat-genesis\nversion=1\narchitecture=%s\n' "$architecture" \ + >"$output/xcat-genesis.manifest" +( + cd "$output" + sha256sum -- * >SHA256SUMS +) +EXPORT + ); + chmod(0755, "$oe/build", "$oe/export") or die $!; + for my $command ( + [ 'git', '-C', $source, 'init', '-q' ], + [ 'git', '-C', $source, 'add', '.' ], + [ 'git', '-C', $source, '-c', 'user.name=xCAT test', + '-c', 'user.email=xcat-test@example.invalid', 'commit', '-qm', 'fixture' ], + ) { + die "Cannot prepare test repository\n" + if run_capture("$tmp/tmpdir-git.log", @{$command}); + } + + my $ambient_tmp = "$tmp/ambient-tmp"; + my $output = "$tmp/tmpdir-release"; + my $log = "$tmp/tmpdir-builder.log"; + make_path($ambient_tmp); + my $status; + { + local $ENV{TMPDIR} = $ambient_tmp; + $status = run_capture( + $log, $builder, '--xcat-source', $source, + '--output-dir', $output, '--format', 'deb', + ); + } + is($status, 0, 'release builder isolates the OpenEmbedded tmpdir'); + unlike(read_file($log), qr/Invalid OpenEmbedded deploy directory/, + 'release builder finds the configured deploy directory'); + my $built = validate_release($output); + is($built->{architectures}, 'x86_64', 'isolated build keeps the target architecture'); + is($built->{formats}, 'deb', 'isolated build keeps the requested format'); +} From 4d0ee7d5f135e836cf54f72b76eb28abda898784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 00:27:12 -0300 Subject: [PATCH 16/82] fix(genesis): tolerate inaccessible build cwd --- genesis-openembedded/package | 1 + 1 file changed, 1 insertion(+) diff --git a/genesis-openembedded/package b/genesis-openembedded/package index a12fb0c..13be1d2 100755 --- a/genesis-openembedded/package +++ b/genesis-openembedded/package @@ -69,6 +69,7 @@ make_path($output_parent) unless -d $output_parent; $output_parent = abs_path($output_parent) or die "Cannot resolve output parent\n"; $output_dir = "$output_parent/" . basename($output_dir); die "Output already exists: $output_dir\n" if -e $output_dir || -l $output_dir; +chdir($output_parent) or die "Cannot use output parent $output_parent: $!\n"; my $work = tempdir('xcat-genesis-package.XXXXXX', TMPDIR => 1, CLEANUP => 1); my $staging = tempdir('.xcat-genesis-output.XXXXXX', DIR => $output_parent, CLEANUP => 1); From c42e6b92eeeffa57263f91887fa4a4294b23d9ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 00:27:12 -0300 Subject: [PATCH 17/82] test(genesis): cover inaccessible build cwd --- t/genesis_openembedded_release.t | 43 ++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 4fdcdb6..37a95de 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -151,6 +151,13 @@ SKIP: { exercise_packager('rpm'); } +SKIP: { + skip 'rpmbuild and rpm are not installed', 2 + unless command_exists('rpmbuild') && command_exists('rpm'); + skip 'root can traverse an unsearchable working directory', 2 if $> == 0; + exercise_packager_from_unsearchable_cwd(); +} + SKIP: { skip 'dpkg-deb is not installed', 6 unless command_exists('dpkg-deb'); exercise_packager('deb'); @@ -210,6 +217,42 @@ sub exercise_packager { "$format package metadata passes"); } +sub exercise_packager_from_unsearchable_cwd { + my $cwd = "$tmp/unsearchable-cwd"; + my $output = "$tmp/cwd-independent-rpm"; + my $log = "$tmp/cwd-independent-rpm.log"; + make_path($cwd); + + my $pid = fork(); + die "Cannot fork: $!\n" unless defined($pid); + if ($pid == 0) { + chdir($cwd) or die "Cannot enter test directory: $!\n"; + chmod(0000, $cwd) or die "Cannot restrict test directory: $!\n"; + open(STDOUT, '>:raw', $log) or die $!; + open(STDERR, '>&', STDOUT) or die $!; + exec( + $packager, + '--architecture', 'x86_64', + '--export-dir', $export, + '--output-dir', $output, + '--version', $version, + '--release', $release, + '--revision', $revision, + '--source-date-epoch', $epoch, + '--format', 'rpm', + ) or die "Cannot run packager: $!\n"; + } + waitpid($pid, 0); + my $status = $? >> 8; + chmod(0700, $cwd) or die "Cannot restore test directory: $!\n"; + + is($status, 0, 'RPM package ignores an inaccessible inherited working directory'); + ok( + -f "$output/rpm/xCAT-genesis-base-x86_64-$version-$release.noarch.rpm", + 'RPM package is published from an inaccessible inherited working directory', + ); +} + sub exercise_builder_tmpdir { my $source = "$tmp/tmpdir-xcat-core"; my $oe = "$source/xCAT-genesis-builder/oe"; From d4e0d6c39023137fa3fcce3a07fff22e8c994277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:20:07 -0300 Subject: [PATCH 18/82] fix(genesis): normalize package build modes --- genesis-openembedded/package | 1 + 1 file changed, 1 insertion(+) diff --git a/genesis-openembedded/package b/genesis-openembedded/package index 13be1d2..4e0717b 100755 --- a/genesis-openembedded/package +++ b/genesis-openembedded/package @@ -60,6 +60,7 @@ die "Invalid xcat-core revision: $revision\n" unless $revision =~ /^[0-9a-f]{40} die "Invalid source date epoch\n" unless $source_date_epoch =~ /^\d+$/; die "Unsupported package format: $format\n" unless $format eq 'all' || $format eq 'rpm' || $format eq 'deb'; +umask(0022); $export_dir = abs_path($export_dir) or die "Cannot resolve export directory\n"; validate_export($export_dir, $architecture); From 49048459e1aa2a7c94c33e79c7496ef95e3e3083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:20:32 -0300 Subject: [PATCH 19/82] test(genesis): cover package build umasks --- t/genesis_openembedded_release.t | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 37a95de..5c38a70 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -186,8 +186,13 @@ sub exercise_packager { '--format', $format, ); - is(system(@command, '--output-dir', $first), 0, "$format package builds"); - is(system(@command, '--output-dir', $second), 0, "$format package rebuilds"); + my $original_umask = umask(0022); + is(system(@command, '--output-dir', $first), 0, + "$format package builds with umask 0022"); + umask(0002); + is(system(@command, '--output-dir', $second), 0, + "$format package rebuilds with umask 0002"); + umask($original_umask); my ($relative, $source_relative); if ($format eq 'rpm') { @@ -198,11 +203,11 @@ sub exercise_packager { } ok(-f "$first/$relative", "$format binary exists"); is(file_sha("$first/$relative"), file_sha("$second/$relative"), - "$format binary is reproducible"); + "$format binary is reproducible across umasks"); if ($format eq 'rpm') { ok(-f "$first/$source_relative", 'source RPM exists'); is(file_sha("$first/$source_relative"), file_sha("$second/$source_relative"), - 'source RPM is reproducible'); + 'source RPM is reproducible across umasks'); } my $release_root = "$tmp/$format-release"; From c0f3c3e0734b5b4eb4ecb5e40c6219034e50606f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:21:24 -0300 Subject: [PATCH 20/82] test(genesis): build complete release fixtures --- t/genesis_openembedded_consumer.t | 62 +++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index 7c336e4..2edff51 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -9,7 +9,13 @@ use File::Temp qw(tempdir); use FindBin; use Test::More; +use lib "$FindBin::Bin/../genesis-openembedded/lib"; use lib "$FindBin::Bin/lib"; +use XCAT::GenesisRelease qw( + architectures + deb_package_name + rpm_package_name +); use XCAT::GenesisReleaseTest qw( command_exists file_sha @@ -124,7 +130,7 @@ sub test_rpm_consumer { file_sha("$release_root/srpm/$source_package"), 'source RPM matches the release'); ok(-f "$deploy_repo/repodata/repomd.xml", 'RPM repository metadata is generated'); - like(read_file("$output/mockbuild-all/$run/summary.txt"), qr/^copied_rpms=8$/m, + like(read_file("$output/mockbuild-all/$run/summary.txt"), qr/^copied_rpms=14$/m, 'release RPM is counted with required dependencies'); } @@ -194,23 +200,47 @@ sub test_deb_consumer { sub make_package_release { my ($root, $format) = @_; - my $export = make_export("$root/export", 'x86_64'); my $release_root = "$root/release"; - die "Cannot package test release\n" - if run_capture( - "$root/package.log", - $packager, - '--architecture', 'x86_64', - '--export-dir', $export, - '--output-dir', $release_root, - '--version', $version, - '--release', $release, - '--revision', $revision, - '--source-date-epoch', $epoch, - '--format', $format, - ); + make_path($release_root); + for my $architecture (architectures()) { + my $export = make_export("$root/exports/$architecture", $architecture); + my $packages = "$root/packages/$architecture"; + die "Cannot package test release for $architecture\n" + if run_capture( + "$root/package-$architecture.log", + $packager, + '--architecture', $architecture, + '--export-dir', $export, + '--output-dir', $packages, + '--version', $version, + '--release', $release, + '--revision', $revision, + '--source-date-epoch', $epoch, + '--format', $format, + ); + if ($format eq 'rpm') { + my $name = rpm_package_name($architecture); + make_path("$release_root/rpm", "$release_root/srpm"); + copy( + "$packages/rpm/$name-$version-$release.noarch.rpm", + "$release_root/rpm/$name-$version-$release.noarch.rpm", + ) or die $!; + copy( + "$packages/srpm/$name-$version-$release.src.rpm", + "$release_root/srpm/$name-$version-$release.src.rpm", + ) or die $!; + } else { + my $name = deb_package_name($architecture); + make_path("$release_root/deb"); + copy( + "$packages/deb/${name}_${version}-${release}_all.deb", + "$release_root/deb/${name}_${version}-${release}_all.deb", + ) or die $!; + } + } write_release_manifest( - $release_root, $version, $release, $revision, $epoch, 'x86_64', $format, + $release_root, $version, $release, $revision, $epoch, + join(',', architectures()), $format, ); write_checksums($release_root); return $release_root; From fc11b981ab12f3f4d13c4ce2b986357a5a54cab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:22:06 -0300 Subject: [PATCH 21/82] fix(genesis): require complete repository releases --- build-apt-repo.sh | 2 +- genesis-openembedded/lib/XCAT/GenesisRelease.pm | 10 ++++++++++ genesis-openembedded/verify-release | 15 +++++++++++---- mockbuild-all.pl | 2 +- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/build-apt-repo.sh b/build-apt-repo.sh index a6e6546..9707b0b 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -88,7 +88,7 @@ if [[ -n "$GENESIS_RELEASE" ]]; then GENESIS_RELEASE="$(cd "$GENESIS_RELEASE" && pwd)" verifier="$SCRIPT_DIR/genesis-openembedded/verify-release" [[ -x "$verifier" ]] || die "Genesis release verifier not found: $verifier" - "$verifier" --format deb "$GENESIS_RELEASE" + "$verifier" --complete --format deb "$GENESIS_RELEASE" echo "Genesis release: $GENESIS_RELEASE" fi diff --git a/genesis-openembedded/lib/XCAT/GenesisRelease.pm b/genesis-openembedded/lib/XCAT/GenesisRelease.pm index 07f4e0a..892f948 100644 --- a/genesis-openembedded/lib/XCAT/GenesisRelease.pm +++ b/genesis-openembedded/lib/XCAT/GenesisRelease.pm @@ -14,6 +14,7 @@ our @EXPORT_OK = qw( read_release_manifest rpm_package_name validate_architecture + validate_complete_release validate_export validate_release ); @@ -246,4 +247,13 @@ sub validate_release { return $manifest; } +sub validate_complete_release { + my ($directory) = @_; + my $manifest = validate_release($directory); + my %present = map { $_ => 1 } split(/,/, $manifest->{architectures}); + my @missing = grep { !$present{$_} } @ARCHITECTURES; + die "Genesis release is missing supported architectures: @missing\n" if @missing; + return $manifest; +} + 1; diff --git a/genesis-openembedded/verify-release b/genesis-openembedded/verify-release index 5772f50..f3c26e8 100755 --- a/genesis-openembedded/verify-release +++ b/genesis-openembedded/verify-release @@ -11,19 +11,26 @@ use lib "$FindBin::Bin/lib"; use XCAT::GenesisRelease qw( deb_package_name rpm_package_name + validate_complete_release validate_release ); my $format = 'all'; -GetOptions('format=s' => \$format) or die usage(); +my $complete = 0; +GetOptions( + 'format=s' => \$format, + 'complete!' => \$complete, +) or die usage(); die usage() unless @ARGV == 1; die "Unsupported verification format: $format\n" unless $format eq 'all' || $format eq 'rpm' || $format eq 'deb'; my $directory = abs_path($ARGV[0]) or die "Cannot resolve release directory\n"; -my $manifest = validate_release($directory); +my $manifest = $complete + ? validate_complete_release($directory) + : validate_release($directory); my %available = map { $_ => 1 } split(/,/, $manifest->{formats}); -my @formats = $format eq 'all' ? sort keys %available : ($format); +my @formats = $format eq 'all' ? qw(rpm deb) : ($format); for my $required (@formats) { die "Release does not contain $required packages\n" unless $available{$required}; } @@ -97,5 +104,5 @@ sub _capture { } sub usage { - return "Usage: verify-release [--format all|rpm|deb] RELEASE_DIRECTORY\n"; + return "Usage: verify-release [--format all|rpm|deb] [--complete] RELEASE_DIRECTORY\n"; } diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 6d589ed..9f95981 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -145,7 +145,7 @@ if ($genesis_release ne '') { unless -d $genesis_release; my $verifier = "$repo_root/genesis-openembedded/verify-release"; die "Genesis release verifier not found: $verifier\n" unless -x $verifier; - run_argv($^X, $verifier, '--format', 'rpm', $genesis_release); + run_argv($^X, $verifier, '--complete', '--format', 'rpm', $genesis_release); $skip_genesis = 1; } From df481887ca6abf495ab69413f7cbdbb3fa377109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:22:33 -0300 Subject: [PATCH 22/82] test(genesis): reject partial repository releases --- t/genesis_openembedded_release.t | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 5c38a70..3d531f5 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -14,6 +14,7 @@ use XCAT::GenesisRelease qw( deb_package_name rpm_package_name validate_architecture + validate_complete_release validate_export validate_release ); @@ -100,6 +101,44 @@ write_release_manifest( write_checksums($release_dir); my $manifest = validate_release($release_dir); is($manifest->{xcat_revision}, $revision, 'release records xcat-core revision'); +dies_like( + sub { validate_complete_release($release_dir) }, + qr/Genesis release is missing supported architectures/, + 'partial release cannot be published', +); + +my $complete_release = "$tmp/complete-release"; +make_path("$complete_release/rpm", "$complete_release/srpm", "$complete_release/deb"); +for my $architecture (architectures()) { + my $rpm = rpm_package_name($architecture); + my $deb = deb_package_name($architecture); + write_file("$complete_release/rpm/$rpm-$version-$release.noarch.rpm", "rpm $architecture"); + write_file("$complete_release/srpm/$rpm-$version-$release.src.rpm", "srpm $architecture"); + write_file("$complete_release/deb/${deb}_${version}-${release}_all.deb", "deb $architecture"); +} +write_release_manifest( + $complete_release, $version, $release, $revision, $epoch, + join(',', architectures()), 'deb,rpm', +); +write_checksums($complete_release); +ok(validate_complete_release($complete_release), 'complete release can be published'); + +my $deb_only_release = "$tmp/deb-only-release"; +make_path("$deb_only_release/deb"); +for my $architecture (architectures()) { + my $deb = deb_package_name($architecture); + write_file("$deb_only_release/deb/${deb}_${version}-${release}_all.deb", "deb $architecture"); +} +write_release_manifest( + $deb_only_release, $version, $release, $revision, $epoch, + join(',', architectures()), 'deb', +); +write_checksums($deb_only_release); +my $verify_all_log = "$tmp/verify-all.log"; +isnt(run_capture($verify_all_log, $verifier, $deb_only_release), 0, + 'all-format verification rejects a single-format release'); +like(read_file($verify_all_log), qr/Release does not contain rpm packages/, + 'all-format failure names the missing format'); my $bad_release = "$tmp/bad-release"; copy_tree($release_dir, $bad_release); From a8b30e0ecd0350ca6f033b6aeaaa9ba806350e8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:23:00 -0300 Subject: [PATCH 23/82] fix(build): handle repository paths with spaces --- mockbuild-all.pl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 9f95981..867c8c1 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -7,6 +7,7 @@ use Cwd qw(abs_path cwd); use File::Basename qw(dirname basename); use File::Copy qw(copy); use File::Find qw(find); +use File::Glob qw(bsd_glob); use File::Path qw(make_path); use Getopt::Long qw(GetOptions); use Parallel::ForkManager; @@ -481,7 +482,7 @@ if (!$dry_run && $copied == 0) { # Ensure the OS-dependent xCAT-genesis-base rpm (built by the genesis step above) # lands in the dep repo even when the full xCAT core is built elsewhere (--skip-xcat). if (!$skip_genesis && !$dry_run) { - for my $g (glob("$xcat_rpms_dir/xCAT-genesis-base-*.rpm")) { + for my $g (bsd_glob("$xcat_rpms_dir/xCAT-genesis-base-*.rpm")) { next if $g =~ /\.src\.rpm$/; copy($g, "$repo_dir/" . basename($g)) or die "Failed to copy genesis-base $g -> $repo_dir: $!\n"; @@ -595,7 +596,7 @@ sub deploy_target { return if $dry_run; make_path($dest); remove_genesis_packages($dest, 0) if $genesis_release; - for my $rpm (glob("$src/*.rpm")) { + for my $rpm (bsd_glob("$src/*.rpm")) { next if $rpm =~ /\.src\.rpm$/; copy($rpm, "$dest/" . basename($rpm)) or die "Failed to copy $rpm -> $dest: $!\n"; @@ -603,7 +604,7 @@ sub deploy_target { assert_required_deps($dest); sign_and_index_repo($dest); write_dep_repo_metadata($dest, $rel); - my $n = scalar(grep { !/\.src\.rpm$/ } glob("$dest/*.rpm")); + my $n = scalar(grep { !/\.src\.rpm$/ } bsd_glob("$dest/*.rpm")); print "Deployed rh$rel/$arch: $n rpms\n"; } @@ -619,7 +620,7 @@ sub createrepo_c_cmd { sub sign_and_index_repo { my ($dir) = @_; - my @rpms = grep { !/\.src\.rpm$/ } glob("$dir/*.rpm"); + my @rpms = grep { !/\.src\.rpm$/ } bsd_glob("$dir/*.rpm"); if ($gpg_sign && @rpms) { local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; run_simple(qq(rpmsign --define "%_gpg_name $gpg_key_name" --addsign ) @@ -899,7 +900,7 @@ sub run_build_steps_parallel { # have_rpm: is there a non-src rpm named -... under $dir? sub have_rpm { my ($dir, $name) = @_; - my @m = grep { !/\.src\.rpm$/ } glob("$dir/${name}-*.rpm"); + my @m = grep { !/\.src\.rpm$/ } bsd_glob("$dir/${name}-*.rpm"); return scalar(@m) > 0; } @@ -922,7 +923,7 @@ sub assert_required_deps { sub remove_genesis_packages { my ($directory, $source) = @_; return unless -d $directory; - for my $path (glob("$directory/xCAT-genesis-base-*.rpm")) { + for my $path (bsd_glob("$directory/xCAT-genesis-base-*.rpm")) { next if $source && $path !~ /\.src\.rpm\z/; next if !$source && $path =~ /\.src\.rpm\z/; unlink($path) or die "Cannot remove stale Genesis package $path: $!\n"; @@ -936,7 +937,7 @@ sub assert_genesis_release_copied { [ "$genesis_release/srpm", $source_directory ], ) { my ($source_root, $destination_root) = @{$entry}; - for my $source (glob("$source_root/*.rpm")) { + for my $source (bsd_glob("$source_root/*.rpm")) { my $destination = "$destination_root/" . basename($source); die "Genesis package was not collected: $destination\n" unless -f $destination; die "Collected Genesis package differs from release: $destination\n" From 23d7cbf5dc9da75164a8f63fc0a6eee295969b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:23:10 -0300 Subject: [PATCH 24/82] test(build): cover repository paths with spaces --- t/genesis_openembedded_consumer.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index 2edff51..f65cec6 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -64,7 +64,7 @@ sub test_rpm_consumer { my $release_root = make_package_release("$tmp/rpm", 'rpm'); my $package = "xCAT-genesis-base-x86_64-$version-$release.noarch.rpm"; my $source_package = "xCAT-genesis-base-x86_64-$version-$release.src.rpm"; - my $output = "$tmp/rpm-output"; + my $output = "$tmp/rpm output"; my $target = 'test+epel-10-' . capture('uname', '-m'); my $run = "$target-consumer"; my $run_repo = "$output/mockbuild-all/$run/repo/" . capture('uname', '-m'); From 779f5f7b21b9bef186b5f1822266cfc8cee7cd71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:25:39 -0300 Subject: [PATCH 25/82] fix(genesis): pin verified release checksums --- build-apt-repo.sh | 53 +++++++++++++++++-- .../lib/XCAT/GenesisRelease.pm | 23 +++++++- mockbuild-all.pl | 39 +++++++++----- 3 files changed, 97 insertions(+), 18 deletions(-) diff --git a/build-apt-repo.sh b/build-apt-repo.sh index 9707b0b..ee93f04 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -13,6 +13,15 @@ GPG_KEY_ID="xcat@megware.com" SKIP_SIGN=0 DRY_RUN=0 GENESIS_RELEASE="" +GENESIS_CHECKSUMS="" +GENESIS_VERIFIER="" + +cleanup() { + if [[ -n "$GENESIS_CHECKSUMS" ]]; then + rm -f -- "$GENESIS_CHECKSUMS" + fi +} +trap cleanup EXIT declare -A CODENAME_MAP=( [ubuntu22.04]=jammy @@ -86,10 +95,8 @@ APT_DIR="${APT_DIR:-$REPO_ROOT/repos/apt}" if [[ -n "$GENESIS_RELEASE" ]]; then [[ -d "$GENESIS_RELEASE" ]] || die "Genesis release directory not found: $GENESIS_RELEASE" GENESIS_RELEASE="$(cd "$GENESIS_RELEASE" && pwd)" - verifier="$SCRIPT_DIR/genesis-openembedded/verify-release" - [[ -x "$verifier" ]] || die "Genesis release verifier not found: $verifier" - "$verifier" --complete --format deb "$GENESIS_RELEASE" - echo "Genesis release: $GENESIS_RELEASE" + GENESIS_VERIFIER="$SCRIPT_DIR/genesis-openembedded/verify-release" + [[ -x "$GENESIS_VERIFIER" ]] || die "Genesis release verifier not found: $GENESIS_VERIFIER" fi # Default to all known versions when no DIST arg was given; otherwise validate @@ -110,6 +117,21 @@ command -v apt-ftparchive >/dev/null 2>&1 \ || die "apt-ftparchive not found. Install: sudo apt-get install apt-utils" command -v gpg >/dev/null 2>&1 \ || die "gpg not found. Install: sudo apt-get install gnupg" +command -v cmp >/dev/null 2>&1 \ + || die "cmp not found. Install: sudo apt-get install diffutils" +command -v dpkg-deb >/dev/null 2>&1 \ + || die "dpkg-deb not found. Install: sudo apt-get install dpkg" +command -v sha256sum >/dev/null 2>&1 \ + || die "sha256sum not found. Install: sudo apt-get install coreutils" + +if [[ -n "$GENESIS_RELEASE" ]]; then + GENESIS_CHECKSUMS=$(mktemp "${TMPDIR:-/tmp}/xcat-genesis-checksums.XXXXXX") + cp -- "$GENESIS_RELEASE/SHA256SUMS" "$GENESIS_CHECKSUMS" + "$GENESIS_VERIFIER" --complete --format deb "$GENESIS_RELEASE" + cmp -s "$GENESIS_CHECKSUMS" "$GENESIS_RELEASE/SHA256SUMS" \ + || die "Genesis release changed during verification" + echo "Genesis release: $GENESIS_RELEASE" +fi if [[ $SKIP_SIGN -eq 0 ]]; then if ! gpg --list-secret-keys "$GPG_KEY_ID" >/dev/null 2>&1; then @@ -168,6 +190,27 @@ copy_deb() { ln "$source" "$destination" 2>/dev/null || cp "$source" "$destination" } +copy_genesis_deb() { + local source="$1" + local directory="$2" + local name destination relative expected actual + name=$(basename "$source") + destination="$directory/$name" + relative="deb/$name" + if [[ -e "$destination" ]]; then + cmp -s "$source" "$destination" \ + || die "Package collision with different content: $destination" + else + cp -- "$source" "$destination" + fi + expected=$(awk -v path="$relative" '$2 == path { print $1 }' "$GENESIS_CHECKSUMS") + [[ -n "$expected" ]] || die "Missing verified checksum for $relative" + actual=$(sha256sum "$destination") + actual=${actual%% *} + [[ "$actual" = "$expected" ]] \ + || die "Collected Genesis package checksum mismatch: $destination" +} + for ver in "${SELECTED_VERS[@]}"; do codename="${CODENAME_MAP[$ver]}" src="$APT_DIR/$ver" @@ -179,7 +222,7 @@ for ver in "${SELECTED_VERS[@]}"; do done if [[ -n "$GENESIS_RELEASE" ]]; then for deb in "$GENESIS_RELEASE"/deb/*.deb; do - copy_deb "$deb" "$dst" + copy_genesis_deb "$deb" "$dst" done fi fi diff --git a/genesis-openembedded/lib/XCAT/GenesisRelease.pm b/genesis-openembedded/lib/XCAT/GenesisRelease.pm index 892f948..6c87015 100644 --- a/genesis-openembedded/lib/XCAT/GenesisRelease.pm +++ b/genesis-openembedded/lib/XCAT/GenesisRelease.pm @@ -13,10 +13,12 @@ our @EXPORT_OK = qw( deb_package_name read_release_manifest rpm_package_name + validated_release_checksums validate_architecture validate_complete_release validate_export validate_release + verify_release_file ); my @ARCHITECTURES = qw(x86 x86_64 ppc64 ppc64le armv7hf aarch64 riscv64); @@ -126,7 +128,7 @@ sub _verify_checksums { for my $name (keys %{$checksums}) { die "Checksum names a missing file: $name\n" unless $files{$name}; } - return 1; + return $checksums; } sub validate_export { @@ -256,4 +258,23 @@ sub validate_complete_release { return $manifest; } +sub validated_release_checksums { + my ($directory) = @_; + validate_release($directory); + return _read_checksums($directory); +} + +sub verify_release_file { + my ($checksums, $relative, $path) = @_; + die "Missing verified checksum for $relative\n" + unless ref($checksums) eq 'HASH' && exists($checksums->{$relative}); + die "Invalid collected release file: $path\n" unless -f $path && !-l $path; + open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; + my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest; + close($fh) or die "Cannot close $path: $!\n"; + die "Collected release file checksum mismatch: $path\n" + unless $digest eq $checksums->{$relative}; + return 1; +} + 1; diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 867c8c1..dc723be 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -9,9 +9,15 @@ use File::Copy qw(copy); use File::Find qw(find); use File::Glob qw(bsd_glob); use File::Path qw(make_path); +use FindBin; use Getopt::Long qw(GetOptions); use Parallel::ForkManager; use POSIX qw(strftime); +use lib "$FindBin::Bin/genesis-openembedded/lib"; +use XCAT::GenesisRelease qw( + validated_release_checksums + verify_release_file +); my $script_dir = abs_path(dirname(__FILE__)); my $repo_root = abs_path($script_dir); @@ -38,6 +44,7 @@ my $skip_genesis = 0; my $skip_createrepo = 0; my $skip_tarball = 0; my $genesis_release = ''; +my $genesis_release_checksums; my $scrub_all_chroots = 0; my $dry_run = 0; my @extra_collect_dirs; @@ -139,14 +146,18 @@ require_command('rpmsign') if $gpg_sign; require_command('gpg') if $gpg_sign; if ($genesis_release ne '') { - require_command('cmp'); $genesis_release = abs_path($genesis_release) or die "Cannot resolve --genesis-release directory\n"; die "Genesis release directory not found: $genesis_release\n" unless -d $genesis_release; my $verifier = "$repo_root/genesis-openembedded/verify-release"; die "Genesis release verifier not found: $verifier\n" unless -x $verifier; + my $checksums_before = validated_release_checksums($genesis_release); run_argv($^X, $verifier, '--complete', '--format', 'rpm', $genesis_release); + my $checksums_after = validated_release_checksums($genesis_release); + die "Genesis release changed during verification\n" + unless checksum_sets_match($checksums_before, $checksums_after); + $genesis_release_checksums = $checksums_before; $skip_genesis = 1; } @@ -933,23 +944,27 @@ sub remove_genesis_packages { sub assert_genesis_release_copied { my ($binary_directory, $source_directory) = @_; for my $entry ( - [ "$genesis_release/rpm", $binary_directory ], - [ "$genesis_release/srpm", $source_directory ], + [ 'rpm', $binary_directory ], + [ 'srpm', $source_directory ], ) { - my ($source_root, $destination_root) = @{$entry}; - for my $source (bsd_glob("$source_root/*.rpm")) { - my $destination = "$destination_root/" . basename($source); - die "Genesis package was not collected: $destination\n" unless -f $destination; - die "Collected Genesis package differs from release: $destination\n" - unless files_match($source, $destination); + my ($prefix, $destination_root) = @{$entry}; + for my $relative ( + sort grep { /^\Q$prefix\E\/[^\/]+\.rpm\z/ } keys %{$genesis_release_checksums} + ) + { + my $destination = "$destination_root/" . basename($relative); + verify_release_file($genesis_release_checksums, $relative, $destination); } } } -sub files_match { +sub checksum_sets_match { my ($left, $right) = @_; - return 0 unless -f $left && -f $right; - return system('cmp', '-s', $left, $right) == 0; + return 0 unless keys(%{$left}) == keys(%{$right}); + for my $name (keys %{$left}) { + return 0 unless exists($right->{$name}) && $left->{$name} eq $right->{$name}; + } + return 1; } sub collect_rpms { From 7763a03efdffd76a65218a71cc0dbefbebea2f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:25:58 -0300 Subject: [PATCH 26/82] test(genesis): reject changed release packages --- t/genesis_openembedded_release.t | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 3d531f5..44aeb3c 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -2,6 +2,7 @@ use strict; use warnings; use Cwd qw(abs_path); +use File::Copy qw(copy); use File::Path qw(make_path); use File::Temp qw(tempdir); use FindBin; @@ -13,10 +14,12 @@ use XCAT::GenesisRelease qw( architectures deb_package_name rpm_package_name + validated_release_checksums validate_architecture validate_complete_release validate_export validate_release + verify_release_file ); use XCAT::GenesisReleaseTest qw( command_exists @@ -101,6 +104,18 @@ write_release_manifest( write_checksums($release_dir); my $manifest = validate_release($release_dir); is($manifest->{xcat_revision}, $revision, 'release records xcat-core revision'); +my $verified_checksums = validated_release_checksums($release_dir); +my $verified_relative = "rpm/xCAT-genesis-base-x86_64-$version-$release.noarch.rpm"; +my $verified_copy = "$tmp/verified-copy.rpm"; +copy("$release_dir/$verified_relative", $verified_copy) or die $!; +ok(verify_release_file($verified_checksums, $verified_relative, $verified_copy), + 'collected package matches the verified release'); +write_file($verified_copy, 'changed after verification'); +dies_like( + sub { verify_release_file($verified_checksums, $verified_relative, $verified_copy) }, + qr/Collected release file checksum mismatch/, + 'release changes after verification are rejected', +); dies_like( sub { validate_complete_release($release_dir) }, qr/Genesis release is missing supported architectures/, From c71b04344e8a3103393d148a569c369a6fe72c76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:27:14 -0300 Subject: [PATCH 27/82] fix(genesis): require reproducible archive tooling --- genesis-openembedded/package | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/genesis-openembedded/package b/genesis-openembedded/package index 4e0717b..2f22736 100755 --- a/genesis-openembedded/package +++ b/genesis-openembedded/package @@ -102,6 +102,7 @@ print "Packaged Genesis $architecture in $output_dir\n"; sub _build_rpm { my ($work, $staging, $source_root, $source_name, $package_name) = @_; + _require_gnu_tar(); my $candidate = "/var/tmp/xcat-genesis-rpmbuild-$revision-$architecture-$version-$release"; die "RPM work directory already exists: $candidate\n" if -e $candidate || -l $candidate; mkdir($candidate, 0700) or die "Cannot create RPM work directory $candidate: $!\n"; @@ -270,6 +271,15 @@ sub _require_command { die "Required command not found: $command\n"; } +sub _require_gnu_tar { + _require_command('tar'); + open(my $fh, '-|', 'tar', '--version') or die "Cannot run tar: $!\n"; + my $version_line = <$fh> // ''; + close($fh) or die "Cannot query tar version\n"; + die "GNU tar is required to build Genesis source packages\n" + unless $version_line =~ /^tar \(GNU tar\)/; +} + sub _run { my (@command) = @_; print '+ ', join(' ', map { _display_quote($_) } @command), "\n"; From 7ac007d545ab4e7fbd9bdd3a90e4db01cf986506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:27:26 -0300 Subject: [PATCH 28/82] fix(genesis): accept valid release filenames --- genesis-openembedded/lib/XCAT/GenesisRelease.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/genesis-openembedded/lib/XCAT/GenesisRelease.pm b/genesis-openembedded/lib/XCAT/GenesisRelease.pm index 6c87015..fcb012f 100644 --- a/genesis-openembedded/lib/XCAT/GenesisRelease.pm +++ b/genesis-openembedded/lib/XCAT/GenesisRelease.pm @@ -100,7 +100,7 @@ sub _read_checksums { chomp($line); $line =~ s/\r\z//; die "Invalid checksum entry in $path: $line\n" - unless $line =~ /^([0-9a-f]{64}) ([A-Za-z0-9][A-Za-z0-9._\/-]*)$/; + unless $line =~ /^([0-9a-f]{64}) ([A-Za-z0-9][A-Za-z0-9._\/+~-]*)$/; my ($digest, $name) = ($1, $2); die "Unsafe checksum path in $path: $name\n" if $name =~ m{(?:\A|/)\.\.(?:/|\z)} || $name =~ m{//}; From b7197aab0f632526b6e411fcc9bf7212af7b2bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:27:41 -0300 Subject: [PATCH 29/82] test(genesis): cover release qualifiers --- t/genesis_openembedded_release.t | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 44aeb3c..b4cdf32 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -104,6 +104,20 @@ write_release_manifest( write_checksums($release_dir); my $manifest = validate_release($release_dir); is($manifest->{xcat_revision}, $revision, 'release records xcat-core revision'); +my $qualified_release = "$tmp/qualified-release"; +my $qualified_release_name = '1+deb~1'; +my $qualified_deb = deb_package_name('x86_64'); +make_path("$qualified_release/deb"); +write_file( + "$qualified_release/deb/${qualified_deb}_${version}-${qualified_release_name}_all.deb", + 'deb x86_64', +); +write_release_manifest( + $qualified_release, $version, $qualified_release_name, $revision, $epoch, + 'x86_64', 'deb', +); +write_checksums($qualified_release); +ok(validate_release($qualified_release), 'package filenames accept valid release qualifiers'); my $verified_checksums = validated_release_checksums($release_dir); my $verified_relative = "rpm/xCAT-genesis-base-x86_64-$version-$release.noarch.rpm"; my $verified_copy = "$tmp/verified-copy.rpm"; From 6597def23947bb89d5939d0973eee2636d716d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:28:11 -0300 Subject: [PATCH 30/82] fix(genesis): harden package install hooks --- genesis-openembedded/package | 4 +++- genesis-openembedded/rpm/xCAT-genesis-base.spec | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/genesis-openembedded/package b/genesis-openembedded/package index 2f22736..f0cebd5 100755 --- a/genesis-openembedded/package +++ b/genesis-openembedded/package @@ -179,8 +179,10 @@ sub _build_deb { . "if [ \"\${1:-}\" = configure ]; then\n" . " mkdir -p /etc/xcat\n" . " touch /etc/xcat/genesis-base-updated\n" + . " root_stat=\$(stat -c '%i %d' / 2>/dev/null || :)\n" + . " init_stat=\$(stat -c '%i %d' /proc/1/root/. 2>/dev/null || :)\n" . " if [ -x /opt/xcat/sbin/mknb ] && [ -r /proc/cmdline ] && \\\n" - . " [ \"\$(stat -c '%i %d' / 2>/dev/null)\" = \"\$(stat -c '%i %d' /proc/1/root/. 2>/dev/null)\" ]; then\n" + . " [ -n \"\$root_stat\" ] && [ \"\$root_stat\" = \"\$init_stat\" ]; then\n" . " if /opt/xcat/sbin/mknb '$architecture'; then\n" . " rm -f /etc/xcat/genesis-base-updated\n" . " else\n" diff --git a/genesis-openembedded/rpm/xCAT-genesis-base.spec b/genesis-openembedded/rpm/xCAT-genesis-base.spec index 12db225..e2db891 100644 --- a/genesis-openembedded/rpm/xCAT-genesis-base.spec +++ b/genesis-openembedded/rpm/xCAT-genesis-base.spec @@ -30,8 +30,10 @@ install -m 0644 xcat-core-revision %{buildroot}%{_docdir}/%{name}/ %post mkdir -p /etc/xcat touch /etc/xcat/genesis-base-updated +root_stat=$(stat -c '%%i %%d' / 2>/dev/null || :) +init_stat=$(stat -c '%%i %%d' /proc/1/root/. 2>/dev/null || :) if [ -x /opt/xcat/sbin/mknb ] && [ -r /proc/cmdline ] && \ - [ "$(stat -c '%%i %%d' / 2>/dev/null)" = "$(stat -c '%%i %%d' /proc/1/root/. 2>/dev/null)" ]; then + [ -n "$root_stat" ] && [ "$root_stat" = "$init_stat" ]; then if /opt/xcat/sbin/mknb %{genesis_arch}; then rm -f /etc/xcat/genesis-base-updated else From d201f5c0f559bfc62160138cd08f8feb12a8f79d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:28:38 -0300 Subject: [PATCH 31/82] test(genesis): inspect package install hooks --- t/genesis_openembedded_release.t | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index b4cdf32..63bb048 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -214,7 +214,7 @@ SKIP: { } SKIP: { - skip 'rpmbuild and rpm are not installed', 8 + skip 'rpmbuild and rpm are not installed', 10 unless command_exists('rpmbuild') && command_exists('rpm'); exercise_packager('rpm'); } @@ -227,7 +227,7 @@ SKIP: { } SKIP: { - skip 'dpkg-deb is not installed', 6 unless command_exists('dpkg-deb'); + skip 'dpkg-deb is not installed', 8 unless command_exists('dpkg-deb'); exercise_packager('deb'); } @@ -288,6 +288,20 @@ sub exercise_packager { ok(validate_release($release_root), "$format release layout passes"); is(system($verifier, '--format', $format, $release_root), 0, "$format package metadata passes"); + my $hook_log = "$tmp/$format-install-hook.log"; + if ($format eq 'rpm') { + is(run_capture($hook_log, 'rpm', '-qp', '--scripts', "$first/$relative"), 0, + 'RPM install hook can be read'); + like(read_file($hook_log), qr/root_stat=.*\[ -n \"\$root_stat\" \]/s, + 'RPM install hook rejects an empty container identity'); + } else { + my $control = "$tmp/deb-control"; + make_path($control); + is(run_capture($hook_log, 'dpkg-deb', '-e', "$first/$relative", $control), 0, + 'DEB install hook can be extracted'); + like(read_file("$control/postinst"), qr/root_stat=.*\[ -n \"\$root_stat\" \]/s, + 'DEB install hook rejects an empty container identity'); + } } sub exercise_packager_from_unsearchable_cwd { From be73af9799ac75219b62a43fdc53fbbc1d209688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:29:22 -0300 Subject: [PATCH 32/82] fix(genesis): replace legacy DEB packages --- genesis-openembedded/lib/XCAT/GenesisRelease.pm | 11 +++++++++++ genesis-openembedded/package | 7 +++++++ genesis-openembedded/verify-release | 10 ++++++++-- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/genesis-openembedded/lib/XCAT/GenesisRelease.pm b/genesis-openembedded/lib/XCAT/GenesisRelease.pm index fcb012f..89d0e10 100644 --- a/genesis-openembedded/lib/XCAT/GenesisRelease.pm +++ b/genesis-openembedded/lib/XCAT/GenesisRelease.pm @@ -12,6 +12,7 @@ our @EXPORT_OK = qw( architectures deb_package_name read_release_manifest + replaced_deb_package_names rpm_package_name validated_release_checksums validate_architecture @@ -23,6 +24,10 @@ our @EXPORT_OK = qw( my @ARCHITECTURES = qw(x86 x86_64 ppc64 ppc64le armv7hf aarch64 riscv64); my %ARCHITECTURE = map { $_ => 1 } @ARCHITECTURES; +my %REPLACED_DEB_PACKAGES = ( + x86_64 => [ 'xcat-genesis-base-amd64' ], + ppc64le => [ 'xcat-genesis-base-ppc64el' ], +); sub architectures { return @ARCHITECTURES; @@ -48,6 +53,12 @@ sub deb_package_name { return "xcat-genesis-base-$architecture"; } +sub replaced_deb_package_names { + my ($architecture) = @_; + validate_architecture($architecture); + return @{ $REPLACED_DEB_PACKAGES{$architecture} // [] }; +} + sub _read_key_values { my ($path, $allowed) = @_; open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; diff --git a/genesis-openembedded/package b/genesis-openembedded/package index f0cebd5..a346cfe 100755 --- a/genesis-openembedded/package +++ b/genesis-openembedded/package @@ -15,6 +15,7 @@ use Getopt::Long qw(GetOptions); use lib "$FindBin::Bin/lib"; use XCAT::GenesisRelease qw( deb_package_name + replaced_deb_package_names rpm_package_name validate_architecture validate_export @@ -162,11 +163,17 @@ sub _build_deb { my $installed_kib = _tree_bytes($root); $installed_kib = int(($installed_kib + 1023) / 1024); + my @replaced = replaced_deb_package_names($architecture); + my $transition = @replaced + ? "Conflicts: " . join(', ', @replaced) . "\n" + . "Replaces: " . join(', ', @replaced) . "\n" + : ''; _write_file( "$root/DEBIAN/control", "Package: $package_name\n" . "Version: $version-$release\n" . "Section: admin\nPriority: optional\nArchitecture: all\n" + . $transition . "Installed-Size: $installed_kib\n" . "Maintainer: xCAT \n" . "Description: xCAT Genesis netboot image\n" diff --git a/genesis-openembedded/verify-release b/genesis-openembedded/verify-release index f3c26e8..ca87905 100755 --- a/genesis-openembedded/verify-release +++ b/genesis-openembedded/verify-release @@ -10,6 +10,7 @@ use Getopt::Long qw(GetOptions); use lib "$FindBin::Bin/lib"; use XCAT::GenesisRelease qw( deb_package_name + replaced_deb_package_names rpm_package_name validate_complete_release validate_release @@ -79,9 +80,14 @@ sub _verify_deb { my $path = "$directory/deb/${name}_$manifest->{xcat_version}-$manifest->{xcat_release}_all.deb"; my $metadata = join( "\n", - map { _capture('dpkg-deb', '-f', $path, $_) } qw(Package Version Architecture), + map { _capture('dpkg-deb', '-f', $path, $_) } + qw(Package Version Architecture Conflicts Replaces), + ); + my $replaced = join(', ', replaced_deb_package_names($architecture)); + my $expected = join( + "\n", $name, "$manifest->{xcat_version}-$manifest->{xcat_release}", + 'all', $replaced, $replaced, ); - my $expected = join("\n", $name, "$manifest->{xcat_version}-$manifest->{xcat_release}", 'all'); die "Unexpected DEB identity for $path: $metadata\n" unless $metadata eq $expected; } From a54f7bc3ea73582699d75dbcb4843ff1abc5ea2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:29:38 -0300 Subject: [PATCH 33/82] test(genesis): cover DEB package transitions --- t/genesis_openembedded_release.t | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 63bb048..b0a571f 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -13,6 +13,7 @@ use lib "$FindBin::Bin/lib"; use XCAT::GenesisRelease qw( architectures deb_package_name + replaced_deb_package_names rpm_package_name validated_release_checksums validate_architecture @@ -52,6 +53,21 @@ is(rpm_package_name('ppc64le'), 'xCAT-genesis-base-ppc64le', 'RPM package keeps ppc64le distinct'); is(deb_package_name('x86_64'), 'xcat-genesis-base-x86-64', 'DEB package uses a legal spelling of x86_64'); +is_deeply( + [ replaced_deb_package_names('x86_64') ], + [ 'xcat-genesis-base-amd64' ], + 'x86_64 DEB replaces the old host-spelled package', +); +is_deeply( + [ replaced_deb_package_names('ppc64le') ], + [ 'xcat-genesis-base-ppc64el' ], + 'ppc64le DEB replaces the old host-spelled package', +); +is_deeply( + [ replaced_deb_package_names('ppc64') ], + [], + 'ppc64 and ppc64le packages can be installed together', +); dies_like(sub { validate_architecture('ppc') }, qr/Unsupported Genesis architecture/, 'legacy ppc alias is rejected'); From 79cf9add2ad53cebd98157e4e0e24f0c19d16ff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:33:01 -0300 Subject: [PATCH 34/82] fix(build): preserve source epoch failures --- build-apt-repo.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-apt-repo.sh b/build-apt-repo.sh index ee93f04..02e9cf3 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -4,7 +4,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" if [ -z "${SOURCE_DATE_EPOCH:-}" ] && [ -f "$SCRIPT_DIR/Gitepoch" ]; then - export SOURCE_DATE_EPOCH=$(cat "$SCRIPT_DIR/Gitepoch") + SOURCE_DATE_EPOCH=$(cat "$SCRIPT_DIR/Gitepoch") + export SOURCE_DATE_EPOCH fi REPO_ROOT="$SCRIPT_DIR" From 853847ab399b057399abcc40a0d95ec380f31680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:33:30 -0300 Subject: [PATCH 35/82] ci(genesis): test OpenEmbedded packages --- .github/workflows/genesis-openembedded.yml | 45 ++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/genesis-openembedded.yml diff --git a/.github/workflows/genesis-openembedded.yml b/.github/workflows/genesis-openembedded.yml new file mode 100644 index 0000000..ded2ec7 --- /dev/null +++ b/.github/workflows/genesis-openembedded.yml @@ -0,0 +1,45 @@ +name: Genesis OpenEmbedded packages + +on: + pull_request: + push: + branches: + - master + +permissions: + contents: read + +jobs: + package-tests: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Install build tools + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + apt-utils createrepo-c dpkg-dev gnupg \ + libparallel-forkmanager-perl libperl-critic-perl rpm shellcheck + + - name: Run static checks + run: | + perl -c genesis-openembedded/build + perl -c genesis-openembedded/package + perl -c genesis-openembedded/verify-release + perl -c genesis-openembedded/lib/XCAT/GenesisRelease.pm + perl -c mockbuild-all.pl + perlcritic \ + genesis-openembedded/build \ + genesis-openembedded/package \ + genesis-openembedded/verify-release \ + genesis-openembedded/lib/XCAT/GenesisRelease.pm \ + mockbuild-all.pl \ + t/genesis_openembedded_release.t \ + t/genesis_openembedded_consumer.t \ + t/lib/XCAT/GenesisReleaseTest.pm + bash -n build-apt-repo.sh + shellcheck build-apt-repo.sh + + - name: Run package tests + run: sudo -E prove -v -It/lib t From 2531e9f579c0c9c5d1519021a5d6cfa56c609688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:34:32 -0300 Subject: [PATCH 36/82] docs(genesis): clarify release publication --- BUILD.md | 7 +++++++ genesis-openembedded/README.md | 27 ++++++++++++++++++++++----- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/BUILD.md b/BUILD.md index 263b847..d9d1124 100644 --- a/BUILD.md +++ b/BUILD.md @@ -171,6 +171,13 @@ copied into each generated EL repository. Old Genesis packages are removed from those output directories so a previous run cannot satisfy the release by accident. +Repository publication requires a release containing every supported Genesis +architecture. The packages are `noarch`, and every management-node repository +receives the full set of target images. + +The release checksums cover the unsigned input packages. If repository signing +is enabled, `rpmsign` changes the deployed RPM bytes after collection. + Omit `--genesis-release` to keep using the existing Genesis builder. # Build Unified Repository Without xCAT (`--skip-xcat`) diff --git a/genesis-openembedded/README.md b/genesis-openembedded/README.md index 549ef77..4a94bbc 100644 --- a/genesis-openembedded/README.md +++ b/genesis-openembedded/README.md @@ -20,10 +20,14 @@ The default format is `all`, which produces RPM, SRPM, and DEB packages. Use supported image architectures are `x86`, `x86_64`, `ppc64`, `ppc64le`, `armv7hf`, `aarch64`, and `riscv64`. -Each package installs one exact-architecture image under -`/opt/xcat/share/xcat/netboot/genesis//`. The packages themselves -are `noarch` or `all` because they are installed on the management node, not run -on the target node. +Use `--architecture` for development builds. Repository publication requires a +complete release built with `--all`. + +Each package installs one exact-architecture export under +`/opt/xcat/share/xcat/netboot/genesis//`. This includes the kernel, +initramfs, manifests, checksums, and license evidence. The packages are `noarch` +or `all` because they are installed on the management node, not run on the +target node. The release directory contains: @@ -51,7 +55,9 @@ perl ./mockbuild-all.pl \ The RPM builder skips its old per-EL Genesis build when a release is supplied. The APT builder adds the DEB packages to every selected suite. Both consumers -verify package identities and checksums before collecting packages. +require all seven architectures and verify package identities and checksums +before collecting packages. Every management-node repository receives every +target image so it can provision nodes of another architecture. Without `--genesis-release`, both builders keep their existing behavior. The option remains opt-in until xcat-core uses the exact package names on every @@ -59,3 +65,14 @@ platform. The current RPM naming already matches `x86_64`, but older mappings still use `ppc64` for `ppc64le`, and the DEB packages use `amd64`. Those mappings belong in a separate xcat-core change. Generated images and packages belong in release storage, not in Git. + +The DEB packages replace the old `amd64` and `ppc64el` package spellings during +an upgrade. They do not provide aliases for those names. This keeps the old +workflow unchanged until the xcat-core package dependencies move to the exact +architecture names. + +Run the package tests on a Linux builder with RPM, DEB, and repository tools: + +```bash +sudo prove -It/lib t +``` From 4f38261407369f85a5bad82b1904bc808768d1d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:35:27 -0300 Subject: [PATCH 37/82] test(genesis): protect deployed release packages --- t/genesis_openembedded_consumer.t | 66 +++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index f65cec6..5745f43 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -38,24 +38,25 @@ my $epoch = 1787293573; my $tmp = tempdir(CLEANUP => 1); SKIP: { - skip 'RPM repository tools require a root Linux builder', 8 + skip 'RPM repository tools require a root Linux builder', 10 unless $^O eq 'linux' && $> == 0 && command_exists('rpmbuild') && command_exists('rpm') - && command_exists('createrepo_c') - && command_exists('cmp'); + && command_exists('createrepo_c'); test_rpm_consumer(); + test_partial_rpm_release(); } SKIP: { - skip 'APT repository tools are not installed', 8 + skip 'APT repository tools are not installed', 10 unless $^O eq 'linux' && command_exists('bash') && command_exists('dpkg-deb') && command_exists('apt-ftparchive') && command_exists('gpg'); test_deb_consumer(); + test_partial_deb_release(); } done_testing(); @@ -198,11 +199,62 @@ sub test_deb_consumer { 'collision failure identifies the package'); } +sub test_partial_rpm_release { + my $release_root = make_package_release("$tmp/rpm-partial", 'rpm', 'x86_64'); + my $output = "$tmp/partial-output"; + my $target = 'test+epel-10-' . capture('uname', '-m'); + my $deployed = "$output/xcat-dep/rh10/" . capture('uname', '-m'); + my $existing = "$deployed/xCAT-genesis-base-existing.noarch.rpm"; + make_path($deployed); + write_file($existing, 'existing release'); + + my $log = "$tmp/rpm-partial.log"; + my $status = run_capture( + $log, + $^X, $rpm_consumer, + '--repo-root', $repo_root, + '--output', $output, + '--target', $target, + '--run-id', 'partial', + '--build-timestamp', $epoch, + '--skip-build', '--skip-xcat', '--skip-xcat-dep', '--skip-perl', + '--skip-createrepo', '--skip-tarball', + '--genesis-release', $release_root, + ); + + isnt($status, 0, 'RPM repository rejects a partial Genesis release'); + ok(-f $existing, 'partial release does not remove the deployed package'); +} + +sub test_partial_deb_release { + my $release_root = make_package_release("$tmp/deb-partial", 'deb', 'x86_64'); + my $apt_root = "$tmp/partial-apt"; + my $pool = "$apt_root/pool/main/noble"; + my $existing = "$pool/xcat-genesis-base-existing.deb"; + make_path($pool); + write_file($existing, 'existing release'); + + my $log = "$tmp/deb-partial.log"; + my $status = run_capture( + $log, + 'bash', $deb_consumer, + '--repo-root', $repo_root, + '--apt-dir', $apt_root, + '--skip-sign', + '--genesis-release', $release_root, + 'ubuntu24.04', + ); + + isnt($status, 0, 'APT repository rejects a partial Genesis release'); + ok(-f $existing, 'partial DEB release does not remove the deployed package'); +} + sub make_package_release { - my ($root, $format) = @_; + my ($root, $format, @requested_architectures) = @_; + @requested_architectures = architectures() unless @requested_architectures; my $release_root = "$root/release"; make_path($release_root); - for my $architecture (architectures()) { + for my $architecture (@requested_architectures) { my $export = make_export("$root/exports/$architecture", $architecture); my $packages = "$root/packages/$architecture"; die "Cannot package test release for $architecture\n" @@ -240,7 +292,7 @@ sub make_package_release { } write_release_manifest( $release_root, $version, $release, $revision, $epoch, - join(',', architectures()), $format, + join(',', @requested_architectures), $format, ); write_checksums($release_root); return $release_root; From 6516f294231bd7e2066a8314a690d24bd9a604ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:11:30 -0300 Subject: [PATCH 38/82] fix(genesis): stage OpenEmbedded packages separately --- build-apt-repo.sh | 4 ++ genesis-openembedded/build | 20 +++++++- .../lib/XCAT/GenesisRelease.pm | 15 +----- genesis-openembedded/package | 33 ++---------- .../rpm/xCAT-genesis-base.spec | 51 ------------------- .../rpm/xCAT-genesis-openembedded.spec | 37 ++++++++++++++ genesis-openembedded/verify-release | 12 +++-- mockbuild-all.pl | 46 ++++++++++++++--- 8 files changed, 111 insertions(+), 107 deletions(-) delete mode 100644 genesis-openembedded/rpm/xCAT-genesis-base.spec create mode 100644 genesis-openembedded/rpm/xCAT-genesis-openembedded.spec diff --git a/build-apt-repo.sh b/build-apt-repo.sh index 02e9cf3..67f6102 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -219,9 +219,13 @@ for ver in "${SELECTED_VERS[@]}"; do echo "$ver -> pool/main/$codename/" if [[ $DRY_RUN -eq 0 ]]; then for deb in "$src"/*.deb; do + if [[ -n "$GENESIS_RELEASE" && ${deb##*/} == xcat-genesis-openembedded-*.deb ]]; then + continue + fi copy_deb "$deb" "$dst" done if [[ -n "$GENESIS_RELEASE" ]]; then + rm -f -- "$dst"/xcat-genesis-openembedded-*.deb for deb in "$GENESIS_RELEASE"/deb/*.deb; do copy_genesis_deb "$deb" "$dst" done diff --git a/genesis-openembedded/build b/genesis-openembedded/build index 342fdea..0121ad0 100755 --- a/genesis-openembedded/build +++ b/genesis-openembedded/build @@ -27,6 +27,7 @@ my $repo_root = abs_path("$FindBin::Bin/.."); my $xcat_source = "$repo_root/../xcat-core"; my $xcat_ref = ''; my $output_dir = ''; +my $work_dir = ''; my $format = 'all'; my $all = 0; my @requested_architectures; @@ -35,6 +36,7 @@ GetOptions( 'xcat-source=s' => \$xcat_source, 'xcat-ref=s' => \$xcat_ref, 'output-dir=s' => \$output_dir, + 'work-dir=s' => \$work_dir, 'format=s' => \$format, 'architecture=s@' => \@requested_architectures, 'all!' => \$all, @@ -49,6 +51,9 @@ die "Use either --all or --architecture\n" if $all && @requested_architectures; my %seen; @requested_architectures = grep { !$seen{$_}++ } @requested_architectures; validate_architecture($_) for @requested_architectures; +my %requested = map { $_ => 1 } @requested_architectures; +@requested_architectures = grep { $requested{$_} } architectures(); +umask(0022); $xcat_source = abs_path($xcat_source) or die "Cannot resolve xcat-core source\n"; for my $path (qw(Version xCAT-genesis-builder/oe/build xCAT-genesis-builder/oe/export)) { @@ -83,7 +88,14 @@ die "Output already exists: $output_dir\n" if -e $output_dir || -l $output_dir; my $work = tempdir('xcat-genesis-release.XXXXXX', TMPDIR => 1, CLEANUP => 1); my $staging = tempdir('.xcat-genesis-release.XXXXXX', DIR => $output_parent, CLEANUP => 1); -my $oe_work = "$work/openembedded"; +my $oe_work; +if ($work_dir ne '') { + make_path($work_dir) unless -d $work_dir; + $work_dir = abs_path($work_dir) or die "Cannot resolve work directory\n"; + $oe_work = "$work_dir/openembedded"; +} else { + $oe_work = "$work/openembedded"; +} my $oe_tmp = "$oe_work/build/tmp"; local $ENV{XCAT_GENESIS_WORK_DIR} = $oe_work; { @@ -132,8 +144,11 @@ _write_file( ); _write_checksums($staging); validate_release($staging); -_run("$FindBin::Bin/verify-release", '--format', $format, $staging); +my @verify_args = ('--format', $format); +push(@verify_args, '--complete') if $all; +_run("$FindBin::Bin/verify-release", @verify_args, $staging); +chmod(0755, $staging) or die "Cannot make release directory readable: $!\n"; rename($staging, $output_dir) or die "Cannot publish $output_dir: $!\n"; print "Built Genesis package release in $output_dir\n"; @@ -238,6 +253,7 @@ sub _write_file { sub usage { return <<'USAGE'; Usage: build [--xcat-source DIR] [--xcat-ref REF] [--output-dir DIR] + [--work-dir DIR] [--format all|rpm|deb] [--architecture ARCH ... | --all] USAGE } diff --git a/genesis-openembedded/lib/XCAT/GenesisRelease.pm b/genesis-openembedded/lib/XCAT/GenesisRelease.pm index 89d0e10..f10b2ee 100644 --- a/genesis-openembedded/lib/XCAT/GenesisRelease.pm +++ b/genesis-openembedded/lib/XCAT/GenesisRelease.pm @@ -12,7 +12,6 @@ our @EXPORT_OK = qw( architectures deb_package_name read_release_manifest - replaced_deb_package_names rpm_package_name validated_release_checksums validate_architecture @@ -24,10 +23,6 @@ our @EXPORT_OK = qw( my @ARCHITECTURES = qw(x86 x86_64 ppc64 ppc64le armv7hf aarch64 riscv64); my %ARCHITECTURE = map { $_ => 1 } @ARCHITECTURES; -my %REPLACED_DEB_PACKAGES = ( - x86_64 => [ 'xcat-genesis-base-amd64' ], - ppc64le => [ 'xcat-genesis-base-ppc64el' ], -); sub architectures { return @ARCHITECTURES; @@ -43,20 +38,14 @@ sub validate_architecture { sub rpm_package_name { my ($architecture) = @_; validate_architecture($architecture); - return "xCAT-genesis-base-$architecture"; + return "xCAT-genesis-openembedded-$architecture"; } sub deb_package_name { my ($architecture) = @_; validate_architecture($architecture); $architecture =~ tr/_/-/; - return "xcat-genesis-base-$architecture"; -} - -sub replaced_deb_package_names { - my ($architecture) = @_; - validate_architecture($architecture); - return @{ $REPLACED_DEB_PACKAGES{$architecture} // [] }; + return "xcat-genesis-openembedded-$architecture"; } sub _read_key_values { diff --git a/genesis-openembedded/package b/genesis-openembedded/package index a346cfe..cf212a3 100755 --- a/genesis-openembedded/package +++ b/genesis-openembedded/package @@ -15,7 +15,6 @@ use Getopt::Long qw(GetOptions); use lib "$FindBin::Bin/lib"; use XCAT::GenesisRelease qw( deb_package_name - replaced_deb_package_names rpm_package_name validate_architecture validate_export @@ -117,8 +116,8 @@ sub _build_rpm { "--mtime=\@$source_date_epoch", '--use-compress-program=gzip -n', '-cf', $archive, '-C', dirname($source_root), basename($source_root), ); - my $spec = "$FindBin::Bin/rpm/xCAT-genesis-base.spec"; - copy($spec, "$rpm_top/SPECS/xCAT-genesis-base.spec") + my $spec = "$FindBin::Bin/rpm/xCAT-genesis-openembedded.spec"; + copy($spec, "$rpm_top/SPECS/xCAT-genesis-openembedded.spec") or die "Cannot stage RPM spec: $!\n"; _run( 'rpmbuild', '-ba', @@ -132,7 +131,7 @@ sub _build_rpm { '--define', 'clamp_mtime_to_source_date_epoch 1', '--define', '_binary_payload w9.gzdio', '--define', '_source_payload w9.gzdio', - "$rpm_top/SPECS/xCAT-genesis-base.spec", + "$rpm_top/SPECS/xCAT-genesis-openembedded.spec", ); my $binary = "$rpm_top/RPMS/noarch/$package_name-$version-$release.noarch.rpm"; @@ -148,7 +147,7 @@ sub _build_rpm { sub _build_deb { my ($work, $staging, $source_root, $package_name) = @_; my $root = "$work/deb-root"; - my $image = "$root/opt/xcat/share/xcat/netboot/genesis/$architecture"; + my $image = "$root/opt/xcat/share/xcat/netboot/genesis-openembedded/$architecture"; my $doc = "$root/usr/share/doc/$package_name"; make_path("$root/DEBIAN", $image, $doc, "$staging/deb"); @@ -163,41 +162,17 @@ sub _build_deb { my $installed_kib = _tree_bytes($root); $installed_kib = int(($installed_kib + 1023) / 1024); - my @replaced = replaced_deb_package_names($architecture); - my $transition = @replaced - ? "Conflicts: " . join(', ', @replaced) . "\n" - . "Replaces: " . join(', ', @replaced) . "\n" - : ''; _write_file( "$root/DEBIAN/control", "Package: $package_name\n" . "Version: $version-$release\n" . "Section: admin\nPriority: optional\nArchitecture: all\n" - . $transition . "Installed-Size: $installed_kib\n" . "Maintainer: xCAT \n" . "Description: xCAT Genesis netboot image\n" . " The OpenEmbedded Genesis image used for node discovery, inventory,\n" . " and service actions.\n", ); - _write_file( - "$root/DEBIAN/postinst", - "#!/bin/sh\nset -e\n" - . "if [ \"\${1:-}\" = configure ]; then\n" - . " mkdir -p /etc/xcat\n" - . " touch /etc/xcat/genesis-base-updated\n" - . " root_stat=\$(stat -c '%i %d' / 2>/dev/null || :)\n" - . " init_stat=\$(stat -c '%i %d' /proc/1/root/. 2>/dev/null || :)\n" - . " if [ -x /opt/xcat/sbin/mknb ] && [ -r /proc/cmdline ] && \\\n" - . " [ -n \"\$root_stat\" ] && [ \"\$root_stat\" = \"\$init_stat\" ]; then\n" - . " if /opt/xcat/sbin/mknb '$architecture'; then\n" - . " rm -f /etc/xcat/genesis-base-updated\n" - . " else\n" - . " echo 'mknb $architecture was deferred; run it after xcatd is ready' >&2\n" - . " fi\n" - . " fi\nfi\nexit 0\n", - ); - chmod(0755, "$root/DEBIAN/postinst") or die "Cannot make postinst executable: $!\n"; _write_deb_md5sums($root); _normalize_mtime($root, $source_date_epoch); diff --git a/genesis-openembedded/rpm/xCAT-genesis-base.spec b/genesis-openembedded/rpm/xCAT-genesis-base.spec deleted file mode 100644 index e2db891..0000000 --- a/genesis-openembedded/rpm/xCAT-genesis-base.spec +++ /dev/null @@ -1,51 +0,0 @@ -%global genesis_arch %{?genesis_arch}%{!?genesis_arch:x86_64} - -Name: xCAT-genesis-base-%{genesis_arch} -Version: %{?version}%{!?version:0} -Release: %{?release}%{!?release:1} -Epoch: 2 -Summary: xCAT Genesis netboot image -License: Various -URL: https://xcat.org/ -Source0: %{name}-%{version}.tar.gz -BuildArch: noarch -AutoReqProv: no - -%description -The OpenEmbedded Genesis image used by xCAT for discovery, inventory, and -service actions before a node boots its installed operating system. - -%prep -%setup -q - -%build - -%install -rm -rf %{buildroot} -install -d -m 0755 %{buildroot}/opt/xcat/share/xcat/netboot/genesis/%{genesis_arch} -install -m 0644 image/* %{buildroot}/opt/xcat/share/xcat/netboot/genesis/%{genesis_arch}/ -install -d -m 0755 %{buildroot}%{_docdir}/%{name} -install -m 0644 xcat-core-revision %{buildroot}%{_docdir}/%{name}/ - -%post -mkdir -p /etc/xcat -touch /etc/xcat/genesis-base-updated -root_stat=$(stat -c '%%i %%d' / 2>/dev/null || :) -init_stat=$(stat -c '%%i %%d' /proc/1/root/. 2>/dev/null || :) -if [ -x /opt/xcat/sbin/mknb ] && [ -r /proc/cmdline ] && \ - [ -n "$root_stat" ] && [ "$root_stat" = "$init_stat" ]; then - if /opt/xcat/sbin/mknb %{genesis_arch}; then - rm -f /etc/xcat/genesis-base-updated - else - echo "mknb %{genesis_arch} was deferred; run it after xcatd is ready" >&2 - fi -fi - -%files -%dir /opt/xcat -%dir /opt/xcat/share -%dir /opt/xcat/share/xcat -%dir /opt/xcat/share/xcat/netboot -%dir /opt/xcat/share/xcat/netboot/genesis -/opt/xcat/share/xcat/netboot/genesis/%{genesis_arch} -%doc %{_docdir}/%{name}/xcat-core-revision diff --git a/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec b/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec new file mode 100644 index 0000000..1e36de0 --- /dev/null +++ b/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec @@ -0,0 +1,37 @@ +%global genesis_arch %{?genesis_arch}%{!?genesis_arch:x86_64} + +Name: xCAT-genesis-openembedded-%{genesis_arch} +Version: %{?version}%{!?version:0} +Release: %{?release}%{!?release:1} +Epoch: 1 +Summary: xCAT OpenEmbedded Genesis netboot image +License: Various +URL: https://xcat.org/ +Source0: %{name}-%{version}.tar.gz +BuildArch: noarch +AutoReqProv: no + +%description +The OpenEmbedded Genesis image used by xCAT for discovery, inventory, and +service actions before a node boots its installed operating system. + +%prep +%setup -q + +%build + +%install +rm -rf %{buildroot} +install -d -m 0755 %{buildroot}/opt/xcat/share/xcat/netboot/genesis-openembedded/%{genesis_arch} +install -m 0644 image/* %{buildroot}/opt/xcat/share/xcat/netboot/genesis-openembedded/%{genesis_arch}/ +install -d -m 0755 %{buildroot}/usr/share/doc/%{name} +install -m 0644 xcat-core-revision %{buildroot}/usr/share/doc/%{name}/ + +%files +%dir /opt/xcat +%dir /opt/xcat/share +%dir /opt/xcat/share/xcat +%dir /opt/xcat/share/xcat/netboot +%dir /opt/xcat/share/xcat/netboot/genesis-openembedded +/opt/xcat/share/xcat/netboot/genesis-openembedded/%{genesis_arch} +%doc /usr/share/doc/%{name}/xcat-core-revision diff --git a/genesis-openembedded/verify-release b/genesis-openembedded/verify-release index ca87905..c555e1e 100755 --- a/genesis-openembedded/verify-release +++ b/genesis-openembedded/verify-release @@ -10,7 +10,6 @@ use Getopt::Long qw(GetOptions); use lib "$FindBin::Bin/lib"; use XCAT::GenesisRelease qw( deb_package_name - replaced_deb_package_names rpm_package_name validate_complete_release validate_release @@ -67,7 +66,7 @@ sub _verify_rpm { ); my $expected = join( "\t", $name, $manifest->{xcat_version}, $manifest->{xcat_release}, - $expected_arch, '2', $expected_source, + $expected_arch, '1', $expected_source, ); die "Unexpected RPM identity for $path: $metadata\n" unless $metadata eq $expected; } @@ -81,14 +80,17 @@ sub _verify_deb { my $metadata = join( "\n", map { _capture('dpkg-deb', '-f', $path, $_) } - qw(Package Version Architecture Conflicts Replaces), + qw(Package Version Architecture), ); - my $replaced = join(', ', replaced_deb_package_names($architecture)); my $expected = join( "\n", $name, "$manifest->{xcat_version}-$manifest->{xcat_release}", - 'all', $replaced, $replaced, + 'all', ); die "Unexpected DEB identity for $path: $metadata\n" unless $metadata eq $expected; + for my $field (qw(Conflicts Replaces)) { + my $value = _capture('dpkg-deb', '-f', $path, $field); + die "Unexpected DEB $field for $path: $value\n" if $value ne ''; + } } sub _require_command { diff --git a/mockbuild-all.pl b/mockbuild-all.pl index dc723be..e7917c5 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -150,7 +150,7 @@ if ($genesis_release ne '') { or die "Cannot resolve --genesis-release directory\n"; die "Genesis release directory not found: $genesis_release\n" unless -d $genesis_release; - my $verifier = "$repo_root/genesis-openembedded/verify-release"; + my $verifier = "$script_dir/genesis-openembedded/verify-release"; die "Genesis release verifier not found: $verifier\n" unless -x $verifier; my $checksums_before = validated_release_checksums($genesis_release); run_argv($^X, $verifier, '--complete', '--format', 'rpm', $genesis_release); @@ -158,7 +158,6 @@ if ($genesis_release ne '') { die "Genesis release changed during verification\n" unless checksum_sets_match($checksums_before, $checksums_after); $genesis_release_checksums = $checksums_before; - $skip_genesis = 1; } # An explicit --target builds just that target; otherwise build the current host @@ -301,7 +300,6 @@ print "srpm_repo_dir: $srpm_repo_dir\n"; print "srpm_tarball: $srpm_tarball\n"; my @collect_roots; -push @collect_roots, "$genesis_release/rpm" if $genesis_release; if ($scrub_all_chroots) { run_step( @@ -468,7 +466,6 @@ push @collect_roots, @extra_collect_dirs; my @srpm_collect_roots = (!$skip_xcat) ? uniq(@collect_roots, $xcat_srpms_dir) : uniq(@collect_roots); -push @srpm_collect_roots, "$genesis_release/srpm" if $genesis_release; @srpm_collect_roots = uniq(@srpm_collect_roots); if ($genesis_release && !$dry_run) { @@ -486,6 +483,9 @@ my ($copied, $skipped_src, $missing_roots) = collect_rpms( dry_run => $dry_run, ); +$copied += install_genesis_release_packages('rpm', $repo_dir) + if $genesis_release && !$dry_run; + if (!$dry_run && $copied == 0) { die "No binary RPMs were collected. Check build logs and collection roots.\n"; } @@ -511,6 +511,9 @@ my ($copied_srpms, $skipped_non_src, $missing_srpm_roots) = collect_srpms( dry_run => $dry_run, ); +$copied_srpms += install_genesis_release_packages('srpm', $srpm_repo_dir) + if $genesis_release && !$dry_run; + assert_genesis_release_copied($repo_dir, $srpm_repo_dir) if $genesis_release && !$dry_run; @@ -739,8 +742,8 @@ Options: --skip-genesis Skip the existing per-EL Genesis image build --skip-createrepo Skip createrepo --skip-tarball Skip binary/SRPM tarball creation - --genesis-release PATH Consume a verified OpenEmbedded Genesis RPM release and skip the - legacy per-EL Genesis build + --genesis-release PATH Add a verified OpenEmbedded Genesis RPM release alongside the + existing per-EL Genesis packages --scrub-all-chroots Run mock -r --scrub=all before build/collect --collect-dir PATH Additional directory to scan recursively for RPMs (repeatable) --dry-run Print planned commands without executing @@ -934,13 +937,38 @@ sub assert_required_deps { sub remove_genesis_packages { my ($directory, $source) = @_; return unless -d $directory; - for my $path (bsd_glob("$directory/xCAT-genesis-base-*.rpm")) { + opendir(my $dh, $directory) or die "Cannot read $directory: $!\n"; + my @names = grep { /^xCAT-genesis-openembedded-.*\.rpm\z/ } readdir($dh); + closedir($dh) or die "Cannot close $directory: $!\n"; + for my $name (@names) { + my $path = "$directory/$name"; next if $source && $path !~ /\.src\.rpm\z/; next if !$source && $path =~ /\.src\.rpm\z/; unlink($path) or die "Cannot remove stale Genesis package $path: $!\n"; } } +sub install_genesis_release_packages { + my ($prefix, $destination_root) = @_; + remove_genesis_packages($destination_root, $prefix eq 'srpm'); + + my $copied = 0; + for my $relative ( + sort grep { /^\Q$prefix\E\/xCAT-genesis-openembedded-[^\/]+\.rpm\z/ } + keys %{$genesis_release_checksums} + ) + { + my $source = "$genesis_release/$relative"; + my $destination = "$destination_root/" . basename($relative); + copy($source, $destination) + or die "Cannot install Genesis release package $source: $!\n"; + verify_release_file($genesis_release_checksums, $relative, $destination); + $copied++; + } + die "Genesis release has no $prefix packages\n" unless $copied; + return $copied; +} + sub assert_genesis_release_copied { my ($binary_directory, $source_directory) = @_; for my $entry ( @@ -1001,6 +1029,8 @@ sub collect_rpms { next; } my $base = basename($rpm); + next if $genesis_release + && $base =~ /^xCAT-genesis-openembedded-/; next if $seen{$base}++; if ($is_dry) { print "DRY-RUN copy: $rpm -> $dest/$base\n"; @@ -1050,6 +1080,8 @@ sub collect_srpms { next; } my $base = basename($rpm); + next if $genesis_release + && $base =~ /^xCAT-genesis-openembedded-/; next if $seen{$base}++; if ($is_dry) { print "DRY-RUN copy source: $rpm -> $dest/$base\n"; From 6e31a24c3048712a4667eb5fdcee130f1e07b105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:14:26 -0300 Subject: [PATCH 39/82] test(genesis): keep legacy packages available --- t/genesis_openembedded_consumer.t | 72 +++++++++++++++++++++---------- t/genesis_openembedded_release.t | 60 +++++++++++--------------- 2 files changed, 75 insertions(+), 57 deletions(-) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index 5745f43..0ba74d6 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -37,8 +37,15 @@ my $release = 'snap202608210726'; my $epoch = 1787293573; my $tmp = tempdir(CLEANUP => 1); +if ($ENV{XCAT_GENESIS_CI}) { + BAIL_OUT('CI requires Linux root') unless $^O eq 'linux' && $> == 0; + for my $command (qw(apt-ftparchive bash createrepo_c dpkg-deb gpg rpm rpmbuild)) { + BAIL_OUT("CI requires $command") unless command_exists($command); + } +} + SKIP: { - skip 'RPM repository tools require a root Linux builder', 10 + skip 'RPM repository tools require a root Linux builder', 13 unless $^O eq 'linux' && $> == 0 && command_exists('rpmbuild') @@ -49,7 +56,7 @@ SKIP: { } SKIP: { - skip 'APT repository tools are not installed', 10 + skip 'APT repository tools are not installed', 12 unless $^O eq 'linux' && command_exists('bash') && command_exists('dpkg-deb') @@ -63,8 +70,8 @@ done_testing(); sub test_rpm_consumer { my $release_root = make_package_release("$tmp/rpm", 'rpm'); - my $package = "xCAT-genesis-base-x86_64-$version-$release.noarch.rpm"; - my $source_package = "xCAT-genesis-base-x86_64-$version-$release.src.rpm"; + my $package = "xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; + my $source_package = "xCAT-genesis-openembedded-x86_64-$version-$release.src.rpm"; my $output = "$tmp/rpm output"; my $target = 'test+epel-10-' . capture('uname', '-m'); my $run = "$target-consumer"; @@ -72,19 +79,28 @@ sub test_rpm_consumer { my $source_repo = "$output/mockbuild-all/$run/repo-src"; my $deploy_repo = "$output/xcat-dep/rh10/" . capture('uname', '-m'); make_path($run_repo, $source_repo, $deploy_repo); - write_file("$run_repo/xCAT-genesis-base-stale.noarch.rpm", 'stale'); - write_file("$source_repo/xCAT-genesis-base-stale.src.rpm", 'stale'); - write_file("$deploy_repo/xCAT-genesis-base-stale.noarch.rpm", 'stale'); + write_file("$run_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale'); + write_file("$source_repo/xCAT-genesis-openembedded-stale.src.rpm", 'stale'); + write_file("$deploy_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale'); my $dependencies = "$tmp/rpm-dependencies"; make_path($dependencies); for my $name (qw( ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB + xCAT-genesis-base-x86_64 )) { copy("$release_root/rpm/$package", "$dependencies/$name-1.noarch.rpm") or die $!; } + write_file( + "$dependencies/xCAT-genesis-openembedded-x86_64-$version-old.noarch.rpm", + 'stale OpenEmbedded RPM', + ); + write_file( + "$dependencies/xCAT-genesis-openembedded-x86_64-$version-old.src.rpm", + 'stale OpenEmbedded SRPM', + ); my $stub = "$tmp/perl-stub/Parallel/ForkManager.pm"; make_path("$tmp/perl-stub/Parallel"); @@ -121,39 +137,46 @@ sub test_rpm_consumer { is($status, 0, 'RPM repository accepts a verified Genesis release'); is(file_sha("$deploy_repo/$package"), file_sha("$release_root/rpm/$package"), 'deployed RPM matches the release'); - ok(!-e "$run_repo/xCAT-genesis-base-stale.noarch.rpm", + ok(!-e "$run_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale run RPM is removed'); - ok(!-e "$source_repo/xCAT-genesis-base-stale.src.rpm", + ok(!-e "$source_repo/xCAT-genesis-openembedded-stale.src.rpm", 'stale source RPM is removed'); - ok(!-e "$deploy_repo/xCAT-genesis-base-stale.noarch.rpm", + ok(!-e "$deploy_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale deployed RPM is removed'); is(file_sha("$source_repo/$source_package"), file_sha("$release_root/srpm/$source_package"), 'source RPM matches the release'); ok(-f "$deploy_repo/repodata/repomd.xml", 'RPM repository metadata is generated'); - like(read_file("$output/mockbuild-all/$run/summary.txt"), qr/^copied_rpms=14$/m, - 'release RPM is counted with required dependencies'); + ok(-f "$deploy_repo/xCAT-genesis-base-x86_64-1.noarch.rpm", + 'legacy Genesis package remains available'); + ok(!-e "$run_repo/xCAT-genesis-openembedded-x86_64-$version-old.noarch.rpm", + 'stale OpenEmbedded RPM is not collected'); + ok(!-e "$source_repo/xCAT-genesis-openembedded-x86_64-$version-old.src.rpm", + 'stale OpenEmbedded SRPM is not collected'); + like(read_file("$output/mockbuild-all/$run/summary.txt"), qr/^copied_rpms=15$/m, + 'release RPM is counted alongside required dependencies'); } sub test_deb_consumer { my $release_root = make_package_release("$tmp/deb", 'deb'); - my $package = "xcat-genesis-base-x86-64_${version}-${release}_all.deb"; + my $package = "xcat-genesis-openembedded-x86-64_${version}-${release}_all.deb"; my $apt_root = "$tmp/apt"; my $input = "$apt_root/ubuntu24.04"; my $dummy = "$tmp/dummy-deb"; make_path("$dummy/DEBIAN", $input, "$apt_root/pool/main/noble"); write_file( "$dummy/DEBIAN/control", - "Package: xcat-dep-test\nVersion: 1\nArchitecture: all\n" + "Package: xcat-genesis-base-amd64\nVersion: 1\nArchitecture: all\n" . "Maintainer: xCAT \n" . "Description: repository test package\n", ); die "Cannot build test DEB\n" if run_capture( "$tmp/dummy-deb.log", 'dpkg-deb', '--root-owner-group', '--build', - $dummy, "$input/xcat-dep-test_1_all.deb", + $dummy, "$input/xcat-genesis-base-amd64_1_all.deb", ); - write_file("$apt_root/pool/main/noble/xcat-genesis-base-stale.deb", 'stale'); + write_file("$input/xcat-genesis-openembedded-stale.deb", 'stale'); + write_file("$apt_root/pool/main/noble/xcat-genesis-openembedded-old.deb", 'stale'); local $ENV{SOURCE_DATE_EPOCH} = $epoch; my $log = "$tmp/deb-consumer.log"; @@ -173,13 +196,17 @@ sub test_deb_consumer { is($status, 0, 'APT repository accepts a verified Genesis release'); is(file_sha($pool_package), file_sha("$release_root/deb/$package"), 'pooled DEB matches the release'); - ok(!-e "$apt_root/pool/main/noble/xcat-genesis-base-stale.deb", + ok(!-e "$apt_root/pool/main/noble/xcat-genesis-openembedded-old.deb", 'stale pooled DEB is removed'); - like(read_file($amd64), qr/^Package: xcat-genesis-base-x86-64$/m, + like(read_file($amd64), qr/^Package: xcat-genesis-openembedded-x86-64$/m, 'all-architecture DEB is indexed for amd64'); - like(read_file($ppc64el), qr/^Package: xcat-genesis-base-x86-64$/m, + like(read_file($ppc64el), qr/^Package: xcat-genesis-openembedded-x86-64$/m, 'all-architecture DEB is indexed for ppc64el'); ok(-f "$apt_root/dists/noble/Release", 'APT Release metadata is generated'); + like(read_file($amd64), qr/^Package: xcat-genesis-base-amd64$/m, + 'legacy Genesis DEB remains available'); + ok(!-e "$apt_root/pool/main/noble/xcat-genesis-openembedded-stale.deb", + 'stale OpenEmbedded DEB is not collected'); my $collision = "$tmp/apt-collision"; make_path("$collision/ubuntu24.04"); @@ -194,9 +221,10 @@ sub test_deb_consumer { '--genesis-release', $release_root, 'ubuntu24.04', ); - isnt($collision_status, 0, 'APT repository rejects a package collision'); - like(read_file($collision_log), qr/Package collision with different content/, - 'collision failure identifies the package'); + is($collision_status, 0, 'verified release replaces a stale source package'); + is(file_sha("$collision/pool/main/noble/$package"), + file_sha("$release_root/deb/$package"), + 'pooled package still matches the verified release'); } sub test_partial_rpm_release { diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index b0a571f..e10fc56 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -13,7 +13,6 @@ use lib "$FindBin::Bin/lib"; use XCAT::GenesisRelease qw( architectures deb_package_name - replaced_deb_package_names rpm_package_name validated_release_checksums validate_architecture @@ -44,30 +43,21 @@ my $version = '2.19.0'; my $release = 'snap202608210726'; my $epoch = 1787293573; +if ($ENV{XCAT_GENESIS_CI}) { + for my $command (qw(git dpkg-deb rpm rpmbuild tar)) { + BAIL_OUT("CI requires $command") unless command_exists($command); + } +} + is_deeply( [ architectures() ], [ qw(x86 x86_64 ppc64 ppc64le armv7hf aarch64 riscv64) ], 'supported architectures keep their exact xCAT names', ); -is(rpm_package_name('ppc64le'), 'xCAT-genesis-base-ppc64le', +is(rpm_package_name('ppc64le'), 'xCAT-genesis-openembedded-ppc64le', 'RPM package keeps ppc64le distinct'); -is(deb_package_name('x86_64'), 'xcat-genesis-base-x86-64', +is(deb_package_name('x86_64'), 'xcat-genesis-openembedded-x86-64', 'DEB package uses a legal spelling of x86_64'); -is_deeply( - [ replaced_deb_package_names('x86_64') ], - [ 'xcat-genesis-base-amd64' ], - 'x86_64 DEB replaces the old host-spelled package', -); -is_deeply( - [ replaced_deb_package_names('ppc64le') ], - [ 'xcat-genesis-base-ppc64el' ], - 'ppc64le DEB replaces the old host-spelled package', -); -is_deeply( - [ replaced_deb_package_names('ppc64') ], - [], - 'ppc64 and ppc64le packages can be installed together', -); dies_like(sub { validate_architecture('ppc') }, qr/Unsupported Genesis architecture/, 'legacy ppc alias is rejected'); @@ -135,7 +125,7 @@ write_release_manifest( write_checksums($qualified_release); ok(validate_release($qualified_release), 'package filenames accept valid release qualifiers'); my $verified_checksums = validated_release_checksums($release_dir); -my $verified_relative = "rpm/xCAT-genesis-base-x86_64-$version-$release.noarch.rpm"; +my $verified_relative = "rpm/xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; my $verified_copy = "$tmp/verified-copy.rpm"; copy("$release_dir/$verified_relative", $verified_copy) or die $!; ok(verify_release_file($verified_checksums, $verified_relative, $verified_copy), @@ -194,7 +184,7 @@ dies_like(sub { validate_release($bad_release) }, qr/Unexpected Genesis release my $missing_release = "$tmp/missing-release"; copy_tree($release_dir, $missing_release); -unlink("$missing_release/rpm/xCAT-genesis-base-ppc64le-$version-$release.noarch.rpm") or die $!; +unlink("$missing_release/rpm/xCAT-genesis-openembedded-ppc64le-$version-$release.noarch.rpm") or die $!; write_checksums($missing_release); dies_like(sub { validate_release($missing_release) }, qr/Genesis release is missing/, 'incomplete architecture set fails'); @@ -280,10 +270,10 @@ sub exercise_packager { my ($relative, $source_relative); if ($format eq 'rpm') { - $relative = "rpm/xCAT-genesis-base-x86_64-$version-$release.noarch.rpm"; - $source_relative = "srpm/xCAT-genesis-base-x86_64-$version-$release.src.rpm"; + $relative = "rpm/xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; + $source_relative = "srpm/xCAT-genesis-openembedded-x86_64-$version-$release.src.rpm"; } else { - $relative = "deb/xcat-genesis-base-x86-64_${version}-${release}_all.deb"; + $relative = "deb/xcat-genesis-openembedded-x86-64_${version}-${release}_all.deb"; } ok(-f "$first/$relative", "$format binary exists"); is(file_sha("$first/$relative"), file_sha("$second/$relative"), @@ -304,19 +294,19 @@ sub exercise_packager { ok(validate_release($release_root), "$format release layout passes"); is(system($verifier, '--format', $format, $release_root), 0, "$format package metadata passes"); - my $hook_log = "$tmp/$format-install-hook.log"; + my $contents_log = "$tmp/$format-contents.log"; if ($format eq 'rpm') { - is(run_capture($hook_log, 'rpm', '-qp', '--scripts', "$first/$relative"), 0, - 'RPM install hook can be read'); - like(read_file($hook_log), qr/root_stat=.*\[ -n \"\$root_stat\" \]/s, - 'RPM install hook rejects an empty container identity'); + is(run_capture($contents_log, 'rpm', '-qpl', "$first/$relative"), 0, + 'RPM payload can be listed'); + like(read_file($contents_log), + qr{/opt/xcat/share/xcat/netboot/genesis-openembedded/x86_64/kernel}, + 'RPM uses the OpenEmbedded staging namespace'); } else { - my $control = "$tmp/deb-control"; - make_path($control); - is(run_capture($hook_log, 'dpkg-deb', '-e', "$first/$relative", $control), 0, - 'DEB install hook can be extracted'); - like(read_file("$control/postinst"), qr/root_stat=.*\[ -n \"\$root_stat\" \]/s, - 'DEB install hook rejects an empty container identity'); + is(run_capture($contents_log, 'dpkg-deb', '-c', "$first/$relative"), 0, + 'DEB payload can be listed'); + like(read_file($contents_log), + qr{/opt/xcat/share/xcat/netboot/genesis-openembedded/x86_64/kernel}, + 'DEB uses the OpenEmbedded staging namespace'); } } @@ -351,7 +341,7 @@ sub exercise_packager_from_unsearchable_cwd { is($status, 0, 'RPM package ignores an inaccessible inherited working directory'); ok( - -f "$output/rpm/xCAT-genesis-base-x86_64-$version-$release.noarch.rpm", + -f "$output/rpm/xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm", 'RPM package is published from an inaccessible inherited working directory', ); } From fd0cffadac1a201bbfd093e428864d516c8dfe05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:15:04 -0300 Subject: [PATCH 40/82] docs(genesis): describe staged package rollout --- BUILD.md | 15 ++++++++----- genesis-openembedded/README.md | 40 +++++++++++++++++----------------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/BUILD.md b/BUILD.md index d9d1124..eac380b 100644 --- a/BUILD.md +++ b/BUILD.md @@ -79,8 +79,8 @@ Use these flags to skip specific operations: - `--skip-genesis` - Skips the existing per-EL Genesis image build. - `--genesis-release ` - - Adds a verified OpenEmbedded Genesis RPM release and skips the existing - per-EL Genesis image build for that run. + - Adds a verified OpenEmbedded Genesis RPM release alongside the existing + per-EL Genesis packages. - `--scrub-all-chroots` - Runs `mock -r --scrub=all` before build and collection. - `--collect-dir ` @@ -167,9 +167,9 @@ perl ./mockbuild-all.pl \ ``` The release is checked before any package is collected. Its RPMs and SRPMs are -copied into each generated EL repository. Old Genesis packages are removed from -those output directories so a previous run cannot satisfy the release by -accident. +copied into each generated EL repository. Stale OpenEmbedded Genesis packages +are removed from the output first, while the existing per-EL Genesis package +remains available. Repository publication requires a release containing every supported Genesis architecture. The packages are `noarch`, and every management-node repository @@ -178,6 +178,11 @@ receives the full set of target images. The release checksums cover the unsigned input packages. If repository signing is enabled, `rpmsign` changes the deployed RPM bytes after collection. +The OpenEmbedded packages use their own names and install under +`/opt/xcat/share/xcat/netboot/genesis-openembedded/`. Publishing them does not +replace the Genesis packages used by current xcat-core releases. Activation is +a separate xcat-core change. + Omit `--genesis-release` to keep using the existing Genesis builder. # Build Unified Repository Without xCAT (`--skip-xcat`) diff --git a/genesis-openembedded/README.md b/genesis-openembedded/README.md index 4a94bbc..dd48fe4 100644 --- a/genesis-openembedded/README.md +++ b/genesis-openembedded/README.md @@ -12,6 +12,7 @@ to the intended commit; it does not change the checkout for you. --xcat-source /path/to/xcat-core \ --xcat-ref \ --all \ + --work-dir /path/to/oe-work \ --output-dir /path/to/xcat-genesis-release ``` @@ -24,10 +25,15 @@ Use `--architecture` for development builds. Repository publication requires a complete release built with `--all`. Each package installs one exact-architecture export under -`/opt/xcat/share/xcat/netboot/genesis//`. This includes the kernel, -initramfs, manifests, checksums, and license evidence. The packages are `noarch` -or `all` because they are installed on the management node, not run on the -target node. +`/opt/xcat/share/xcat/netboot/genesis-openembedded//`. The package +and install namespaces are separate from the old Genesis packages, so both +generations can be published and installed without replacing one another. The +packages are `noarch` or `all` because they are installed on the management +node, not run on the target node. + +`--work-dir` keeps the OpenEmbedded downloads and build state between release +builds. Without it, the builder uses a temporary directory and removes it when +the command finishes. The release directory contains: @@ -38,7 +44,7 @@ The release directory contains: Validate the directory before publishing it: ```bash -./genesis-openembedded/verify-release /path/to/xcat-genesis-release +./genesis-openembedded/verify-release --complete /path/to/xcat-genesis-release ``` Pass that same directory to the repository builders: @@ -53,23 +59,17 @@ perl ./mockbuild-all.pl \ [DIST ...] ``` -The RPM builder skips its old per-EL Genesis build when a release is supplied. -The APT builder adds the DEB packages to every selected suite. Both consumers -require all seven architectures and verify package identities and checksums -before collecting packages. Every management-node repository receives every -target image so it can provision nodes of another architecture. +The RPM builder keeps its old per-EL Genesis build and adds the new packages. +The APT builder does the same for each selected suite. Both consumers require +all seven architectures and verify package identities and checksums before +collecting packages. Every management-node repository receives every target +image so it can provision nodes of another architecture. Without `--genesis-release`, both builders keep their existing behavior. The -option remains opt-in until xcat-core uses the exact package names on every -platform. The current RPM naming already matches `x86_64`, but older mappings -still use `ppc64` for `ppc64le`, and the DEB packages use `amd64`. Those mappings -belong in a separate xcat-core change. Generated images and packages belong in -release storage, not in Git. - -The DEB packages replace the old `amd64` and `ppc64el` package spellings during -an upgrade. They do not provide aliases for those names. This keeps the old -workflow unchanged until the xcat-core package dependencies move to the exact -architecture names. +new packages do not provide, replace, or obsolete the old package names. A +separate xcat-core change will select the OpenEmbedded package and install +namespace after the repositories carry it. Generated images and packages +belong in release storage, not in Git. Run the package tests on a Linux builder with RPM, DEB, and repository tools: From b1eed082e6d1238fbadd8384012fb5456f2e8032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:15:04 -0300 Subject: [PATCH 41/82] ci(genesis): require package build tools --- .github/workflows/genesis-openembedded.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/genesis-openembedded.yml b/.github/workflows/genesis-openembedded.yml index ded2ec7..a2ce8c5 100644 --- a/.github/workflows/genesis-openembedded.yml +++ b/.github/workflows/genesis-openembedded.yml @@ -12,6 +12,8 @@ permissions: jobs: package-tests: runs-on: ubuntu-24.04 + env: + XCAT_GENESIS_CI: 1 steps: - uses: actions/checkout@v4 @@ -29,6 +31,11 @@ jobs: perl -c genesis-openembedded/verify-release perl -c genesis-openembedded/lib/XCAT/GenesisRelease.pm perl -c mockbuild-all.pl + rpmspec -P \ + -D 'genesis_arch x86_64' \ + -D 'version 2.19.0' \ + -D 'release snap202608210726' \ + genesis-openembedded/rpm/xCAT-genesis-openembedded.spec >/dev/null perlcritic \ genesis-openembedded/build \ genesis-openembedded/package \ From 1b21fdb827db5d735e3ddac54b645d08a80ce42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:18:01 -0300 Subject: [PATCH 42/82] test(genesis): verify staged package payloads --- t/genesis_openembedded_release.t | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index e10fc56..c5dce52 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -220,7 +220,7 @@ SKIP: { } SKIP: { - skip 'rpmbuild and rpm are not installed', 10 + skip 'rpmbuild and rpm are not installed', 12 unless command_exists('rpmbuild') && command_exists('rpm'); exercise_packager('rpm'); } @@ -233,12 +233,12 @@ SKIP: { } SKIP: { - skip 'dpkg-deb is not installed', 8 unless command_exists('dpkg-deb'); + skip 'dpkg-deb is not installed', 10 unless command_exists('dpkg-deb'); exercise_packager('deb'); } SKIP: { - skip 'git and dpkg-deb are not installed', 4 + skip 'git and dpkg-deb are not installed', 6 unless command_exists('git') && command_exists('dpkg-deb'); exercise_builder_tmpdir(); } @@ -301,12 +301,21 @@ sub exercise_packager { like(read_file($contents_log), qr{/opt/xcat/share/xcat/netboot/genesis-openembedded/x86_64/kernel}, 'RPM uses the OpenEmbedded staging namespace'); + is(run_capture($contents_log, 'rpm', '-qp', '--scripts', "$first/$relative"), 0, + 'RPM script metadata can be read'); + unlike(read_file($contents_log), qr/postinstall/i, + 'RPM does not activate the staged image'); } else { is(run_capture($contents_log, 'dpkg-deb', '-c', "$first/$relative"), 0, 'DEB payload can be listed'); like(read_file($contents_log), qr{/opt/xcat/share/xcat/netboot/genesis-openembedded/x86_64/kernel}, 'DEB uses the OpenEmbedded staging namespace'); + my $control = "$tmp/deb-control"; + make_path($control); + is(run_capture($contents_log, 'dpkg-deb', '-e', "$first/$relative", $control), 0, + 'DEB control files can be extracted'); + ok(!-e "$control/postinst", 'DEB does not activate the staged image'); } } @@ -397,6 +406,7 @@ EXPORT } my $ambient_tmp = "$tmp/ambient-tmp"; + my $persistent_work = "$tmp/persistent-work"; my $output = "$tmp/tmpdir-release"; my $log = "$tmp/tmpdir-builder.log"; make_path($ambient_tmp); @@ -405,12 +415,16 @@ EXPORT local $ENV{TMPDIR} = $ambient_tmp; $status = run_capture( $log, $builder, '--xcat-source', $source, - '--output-dir', $output, '--format', 'deb', + '--output-dir', $output, '--work-dir', $persistent_work, + '--format', 'deb', ); } is($status, 0, 'release builder isolates the OpenEmbedded tmpdir'); unlike(read_file($log), qr/Invalid OpenEmbedded deploy directory/, 'release builder finds the configured deploy directory'); + ok(-d "$persistent_work/openembedded/build/tmp", + 'release builder preserves the requested work directory'); + is((stat($output))[2] & 07777, 0755, 'release directory is readable by other users'); my $built = validate_release($output); is($built->{architectures}, 'x86_64', 'isolated build keeps the target architecture'); is($built->{formats}, 'deb', 'isolated build keeps the requested format'); From 17bd87c53c165e17aa2b289f17e5a3f4f773ff94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:39:38 -0300 Subject: [PATCH 43/82] fix(genesis): harden staged release publication --- build-apt-repo.sh | 17 ++---- genesis-openembedded/package | 2 +- .../rpm/xCAT-genesis-openembedded.spec | 2 +- genesis-openembedded/verify-release | 15 ++++-- mockbuild-all.pl | 53 +++++++++++++------ 5 files changed, 55 insertions(+), 34 deletions(-) diff --git a/build-apt-repo.sh b/build-apt-repo.sh index 67f6102..55ef407 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -120,12 +120,11 @@ command -v gpg >/dev/null 2>&1 \ || die "gpg not found. Install: sudo apt-get install gnupg" command -v cmp >/dev/null 2>&1 \ || die "cmp not found. Install: sudo apt-get install diffutils" -command -v dpkg-deb >/dev/null 2>&1 \ - || die "dpkg-deb not found. Install: sudo apt-get install dpkg" -command -v sha256sum >/dev/null 2>&1 \ - || die "sha256sum not found. Install: sudo apt-get install coreutils" - if [[ -n "$GENESIS_RELEASE" ]]; then + command -v dpkg-deb >/dev/null 2>&1 \ + || die "dpkg-deb not found. Install: sudo apt-get install dpkg" + command -v sha256sum >/dev/null 2>&1 \ + || die "sha256sum not found. Install: sudo apt-get install coreutils" GENESIS_CHECKSUMS=$(mktemp "${TMPDIR:-/tmp}/xcat-genesis-checksums.XXXXXX") cp -- "$GENESIS_RELEASE/SHA256SUMS" "$GENESIS_CHECKSUMS" "$GENESIS_VERIFIER" --complete --format deb "$GENESIS_RELEASE" @@ -198,12 +197,7 @@ copy_genesis_deb() { name=$(basename "$source") destination="$directory/$name" relative="deb/$name" - if [[ -e "$destination" ]]; then - cmp -s "$source" "$destination" \ - || die "Package collision with different content: $destination" - else - cp -- "$source" "$destination" - fi + cp -- "$source" "$destination" expected=$(awk -v path="$relative" '$2 == path { print $1 }' "$GENESIS_CHECKSUMS") [[ -n "$expected" ]] || die "Missing verified checksum for $relative" actual=$(sha256sum "$destination") @@ -225,7 +219,6 @@ for ver in "${SELECTED_VERS[@]}"; do copy_deb "$deb" "$dst" done if [[ -n "$GENESIS_RELEASE" ]]; then - rm -f -- "$dst"/xcat-genesis-openembedded-*.deb for deb in "$GENESIS_RELEASE"/deb/*.deb; do copy_genesis_deb "$deb" "$dst" done diff --git a/genesis-openembedded/package b/genesis-openembedded/package index cf212a3..977cee8 100755 --- a/genesis-openembedded/package +++ b/genesis-openembedded/package @@ -112,7 +112,7 @@ sub _build_rpm { my $archive = "$rpm_top/SOURCES/$source_name.tar.gz"; _run( - 'tar', '--sort=name', '--owner=0', '--group=0', + 'tar', '--sort=name', '--owner=0', '--group=0', '--numeric-owner', "--mtime=\@$source_date_epoch", '--use-compress-program=gzip -n', '-cf', $archive, '-C', dirname($source_root), basename($source_root), ); diff --git a/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec b/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec index 1e36de0..689b25a 100644 --- a/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec +++ b/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec @@ -3,7 +3,6 @@ Name: xCAT-genesis-openembedded-%{genesis_arch} Version: %{?version}%{!?version:0} Release: %{?release}%{!?release:1} -Epoch: 1 Summary: xCAT OpenEmbedded Genesis netboot image License: Various URL: https://xcat.org/ @@ -24,6 +23,7 @@ service actions before a node boots its installed operating system. rm -rf %{buildroot} install -d -m 0755 %{buildroot}/opt/xcat/share/xcat/netboot/genesis-openembedded/%{genesis_arch} install -m 0644 image/* %{buildroot}/opt/xcat/share/xcat/netboot/genesis-openembedded/%{genesis_arch}/ +# Keep this path stable across RPM build hosts. %%{_docdir} differs on SUSE. install -d -m 0755 %{buildroot}/usr/share/doc/%{name} install -m 0644 xcat-core-revision %{buildroot}/usr/share/doc/%{name}/ diff --git a/genesis-openembedded/verify-release b/genesis-openembedded/verify-release index c555e1e..623a748 100755 --- a/genesis-openembedded/verify-release +++ b/genesis-openembedded/verify-release @@ -61,14 +61,23 @@ sub _verify_rpm { my ($path, $expected_arch, $expected_source) = @{$package}; my $metadata = _capture( 'rpm', '-qp', '--qf', - "%{NAME}\t%{VERSION}\t%{RELEASE}\t%{ARCH}\t%{EPOCHNUM}\t%{SOURCERPM}\n", + "%{NAME}\t%{VERSION}\t%{RELEASE}\t%{ARCH}\t%{EPOCHNUM}\t%{SOURCERPM}" + . "\t%{BUILDHOST}\t%{BUILDTIME}\n", $path, ); my $expected = join( "\t", $name, $manifest->{xcat_version}, $manifest->{xcat_release}, - $expected_arch, '1', $expected_source, + $expected_arch, '0', $expected_source, 'xcat-release', + $manifest->{source_date_epoch}, ); die "Unexpected RPM identity for $path: $metadata\n" unless $metadata eq $expected; + for my $relationship (qw(conflicts obsoletes)) { + my $value = _capture('rpm', '-qp', "--$relationship", $path); + die "Unexpected RPM $relationship for $path: $value\n" if $value ne ''; + } + my $provides = _capture('rpm', '-qp', '--provides', $path); + die "Legacy RPM relationship in $path: $provides\n" + if $provides =~ /xCAT-genesis-base/; } } @@ -87,7 +96,7 @@ sub _verify_deb { 'all', ); die "Unexpected DEB identity for $path: $metadata\n" unless $metadata eq $expected; - for my $field (qw(Conflicts Replaces)) { + for my $field (qw(Breaks Conflicts Provides Replaces)) { my $value = _capture('dpkg-deb', '-f', $path, $field); die "Unexpected DEB $field for $path: $value\n" if $value ne ''; } diff --git a/mockbuild-all.pl b/mockbuild-all.pl index e7917c5..5c26d67 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -466,7 +466,6 @@ push @collect_roots, @extra_collect_dirs; my @srpm_collect_roots = (!$skip_xcat) ? uniq(@collect_roots, $xcat_srpms_dir) : uniq(@collect_roots); -@srpm_collect_roots = uniq(@srpm_collect_roots); if ($genesis_release && !$dry_run) { remove_genesis_packages($repo_dir, 0); @@ -609,11 +608,21 @@ sub deploy_target { print_step("Deploy $tgt -> $dest"); return if $dry_run; make_path($dest); - remove_genesis_packages($dest, 0) if $genesis_release; for my $rpm (bsd_glob("$src/*.rpm")) { next if $rpm =~ /\.src\.rpm$/; - copy($rpm, "$dest/" . basename($rpm)) - or die "Failed to copy $rpm -> $dest: $!\n"; + my $destination = "$dest/" . basename($rpm); + my $temporary = "$destination.$$.new"; + die "Deployment staging file already exists: $temporary\n" + if -e $temporary || -l $temporary; + copy($rpm, $temporary) + or die "Failed to stage $rpm -> $temporary: $!\n"; + rename($temporary, $destination) + or die "Failed to publish $temporary -> $destination: $!\n"; + } + if ($genesis_release) { + my @keep = map { basename($_) } genesis_release_files('rpm'); + remove_genesis_packages($dest, 0, \@keep); + verify_genesis_release_packages('rpm', $dest); } assert_required_deps($dest); sign_and_index_repo($dest); @@ -935,12 +944,14 @@ sub assert_required_deps { } sub remove_genesis_packages { - my ($directory, $source) = @_; + my ($directory, $source, $keep_names) = @_; return unless -d $directory; + my %keep = map { $_ => 1 } @{ $keep_names // [] }; opendir(my $dh, $directory) or die "Cannot read $directory: $!\n"; my @names = grep { /^xCAT-genesis-openembedded-.*\.rpm\z/ } readdir($dh); closedir($dh) or die "Cannot close $directory: $!\n"; for my $name (@names) { + next if $keep{$name}; my $path = "$directory/$name"; next if $source && $path !~ /\.src\.rpm\z/; next if !$source && $path =~ /\.src\.rpm\z/; @@ -953,11 +964,7 @@ sub install_genesis_release_packages { remove_genesis_packages($destination_root, $prefix eq 'srpm'); my $copied = 0; - for my $relative ( - sort grep { /^\Q$prefix\E\/xCAT-genesis-openembedded-[^\/]+\.rpm\z/ } - keys %{$genesis_release_checksums} - ) - { + for my $relative (genesis_release_files($prefix)) { my $source = "$genesis_release/$relative"; my $destination = "$destination_root/" . basename($relative); copy($source, $destination) @@ -969,6 +976,24 @@ sub install_genesis_release_packages { return $copied; } +sub genesis_release_files { + my ($prefix) = @_; + return sort grep { + /^\Q$prefix\E\/xCAT-genesis-openembedded-[^\/]+\.rpm\z/ + } keys %{$genesis_release_checksums}; +} + +sub verify_genesis_release_packages { + my ($prefix, $destination_root) = @_; + my @files = genesis_release_files($prefix); + die "Genesis release has no $prefix packages\n" unless @files; + for my $relative (@files) { + my $destination = "$destination_root/" . basename($relative); + verify_release_file($genesis_release_checksums, $relative, $destination); + } + return scalar(@files); +} + sub assert_genesis_release_copied { my ($binary_directory, $source_directory) = @_; for my $entry ( @@ -976,13 +1001,7 @@ sub assert_genesis_release_copied { [ 'srpm', $source_directory ], ) { my ($prefix, $destination_root) = @{$entry}; - for my $relative ( - sort grep { /^\Q$prefix\E\/[^\/]+\.rpm\z/ } keys %{$genesis_release_checksums} - ) - { - my $destination = "$destination_root/" . basename($relative); - verify_release_file($genesis_release_checksums, $relative, $destination); - } + verify_genesis_release_packages($prefix, $destination_root); } } From 4bfa7104d415f7de2bd630f8020cc69a35b7b91c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:39:39 -0300 Subject: [PATCH 44/82] test(genesis): isolate release consumers --- t/genesis_openembedded_consumer.t | 5 +++-- t/genesis_openembedded_release.t | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index 0ba74d6..0bf87ef 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -84,7 +84,8 @@ sub test_rpm_consumer { write_file("$deploy_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale'); my $dependencies = "$tmp/rpm-dependencies"; - make_path($dependencies); + my $scratch_repo_root = "$tmp/rpm-repo-root"; + make_path($dependencies, $scratch_repo_root); for my $name (qw( ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB @@ -123,7 +124,7 @@ sub test_rpm_consumer { my $status = run_capture( $log, $^X, $rpm_consumer, - '--repo-root', $repo_root, + '--repo-root', $scratch_repo_root, '--output', $output, '--target', $target, '--run-id', 'consumer', diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index c5dce52..3db6644 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -303,8 +303,7 @@ sub exercise_packager { 'RPM uses the OpenEmbedded staging namespace'); is(run_capture($contents_log, 'rpm', '-qp', '--scripts', "$first/$relative"), 0, 'RPM script metadata can be read'); - unlike(read_file($contents_log), qr/postinstall/i, - 'RPM does not activate the staged image'); + is(read_file($contents_log), '', 'RPM has no package scripts'); } else { is(run_capture($contents_log, 'dpkg-deb', '-c', "$first/$relative"), 0, 'DEB payload can be listed'); @@ -315,7 +314,11 @@ sub exercise_packager { make_path($control); is(run_capture($contents_log, 'dpkg-deb', '-e', "$first/$relative", $control), 0, 'DEB control files can be extracted'); - ok(!-e "$control/postinst", 'DEB does not activate the staged image'); + opendir(my $control_dh, $control) or die $!; + my @control_files = sort grep { $_ ne '.' && $_ ne '..' } readdir($control_dh); + closedir($control_dh) or die $!; + is_deeply(\@control_files, [ qw(control md5sums) ], + 'DEB has no maintainer scripts or triggers'); } } From 861cec2d8fa67b6e378df2c6ea4ff295fb8183cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:43:01 -0300 Subject: [PATCH 45/82] ci(genesis): test unprivileged package builds --- .github/workflows/genesis-openembedded.yml | 4 +++- genesis-openembedded/README.md | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/genesis-openembedded.yml b/.github/workflows/genesis-openembedded.yml index a2ce8c5..03c0c4d 100644 --- a/.github/workflows/genesis-openembedded.yml +++ b/.github/workflows/genesis-openembedded.yml @@ -49,4 +49,6 @@ jobs: shellcheck build-apt-repo.sh - name: Run package tests - run: sudo -E prove -v -It/lib t + run: | + prove -v -It/lib t/genesis_openembedded_release.t + sudo -E prove -v -It/lib t/genesis_openembedded_consumer.t diff --git a/genesis-openembedded/README.md b/genesis-openembedded/README.md index dd48fe4..6d4a21d 100644 --- a/genesis-openembedded/README.md +++ b/genesis-openembedded/README.md @@ -74,5 +74,6 @@ belong in release storage, not in Git. Run the package tests on a Linux builder with RPM, DEB, and repository tools: ```bash -sudo prove -It/lib t +prove -It/lib t/genesis_openembedded_release.t +sudo -E prove -It/lib t/genesis_openembedded_consumer.t ``` From a3af2b703190879128f172482f8470a8770db0af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 11:48:07 -0300 Subject: [PATCH 46/82] fix(genesis): return release packages safely --- mockbuild-all.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 5c26d67..9de89af 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -978,9 +978,10 @@ sub install_genesis_release_packages { sub genesis_release_files { my ($prefix) = @_; - return sort grep { + my @files = sort grep { /^\Q$prefix\E\/xCAT-genesis-openembedded-[^\/]+\.rpm\z/ } keys %{$genesis_release_checksums}; + return @files; } sub verify_genesis_release_packages { From f86ba2fb2a171cf50e56f05d0297d62c7568ec1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 11:48:25 -0300 Subject: [PATCH 47/82] test(genesis): express release modes portably --- t/genesis_openembedded_release.t | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 3db6644..110a6d3 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -427,7 +427,8 @@ EXPORT 'release builder finds the configured deploy directory'); ok(-d "$persistent_work/openembedded/build/tmp", 'release builder preserves the requested work directory'); - is((stat($output))[2] & 07777, 0755, 'release directory is readable by other users'); + is((stat($output))[2] & oct('07777'), oct('0755'), + 'release directory is readable by other users'); my $built = validate_release($output); is($built->{architectures}, 'x86_64', 'isolated build keeps the target architecture'); is($built->{formats}, 'deb', 'isolated build keeps the requested format'); From 2b1c7724cb9c120d97b666280a650e6b1dcc60af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:16:16 -0300 Subject: [PATCH 48/82] test(genesis): prepare shared library path --- t/genesis_openembedded_consumer.t | 1 + t/genesis_openembedded_release.t | 1 + 2 files changed, 2 insertions(+) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index 0bf87ef..36070f2 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -9,6 +9,7 @@ use File::Temp qw(tempdir); use FindBin; use Test::More; +use lib "$FindBin::Bin/../lib"; use lib "$FindBin::Bin/../genesis-openembedded/lib"; use lib "$FindBin::Bin/lib"; use XCAT::GenesisRelease qw( diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 110a6d3..2c3ba85 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -8,6 +8,7 @@ use File::Temp qw(tempdir); use FindBin; use Test::More; +use lib "$FindBin::Bin/../lib"; use lib "$FindBin::Bin/../genesis-openembedded/lib"; use lib "$FindBin::Bin/lib"; use XCAT::GenesisRelease qw( From 881605847cbc53daeca9463acf46678b49ac65d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:20:23 -0300 Subject: [PATCH 49/82] ci(genesis): install shared Perl helpers --- .github/workflows/genesis-openembedded.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/genesis-openembedded.yml b/.github/workflows/genesis-openembedded.yml index 03c0c4d..af934fb 100644 --- a/.github/workflows/genesis-openembedded.yml +++ b/.github/workflows/genesis-openembedded.yml @@ -22,6 +22,7 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends \ apt-utils createrepo-c dpkg-dev gnupg \ + libfile-slurper-perl \ libparallel-forkmanager-perl libperl-critic-perl rpm shellcheck - name: Run static checks From 69a00bf43f017e25a550e9b6d3e75ed8a9e896f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:24:45 -0300 Subject: [PATCH 50/82] refactor(genesis): share release build helpers --- genesis-openembedded/build | 110 +++--------- genesis-openembedded/package | 100 +++-------- genesis-openembedded/verify-release | 36 +--- lib/XCAT/BuildUtils.pm | 160 ++++++++++++++++++ .../lib => lib}/XCAT/GenesisRelease.pm | 55 +----- mockbuild-all.pl | 151 ++++++----------- 6 files changed, 282 insertions(+), 330 deletions(-) create mode 100644 lib/XCAT/BuildUtils.pm rename {genesis-openembedded/lib => lib}/XCAT/GenesisRelease.pm (81%) diff --git a/genesis-openembedded/build b/genesis-openembedded/build index 0121ad0..41fa3ba 100755 --- a/genesis-openembedded/build +++ b/genesis-openembedded/build @@ -4,17 +4,22 @@ use strict; use warnings; use Cwd qw(abs_path); -use Digest::SHA (); use File::Basename qw(basename dirname); use File::Copy qw(copy); -use File::Find qw(find); use File::Path qw(make_path); -use File::Spec; use File::Temp qw(tempdir); use FindBin; use Getopt::Long qw(GetOptions); use POSIX qw(strftime); -use lib "$FindBin::Bin/lib"; +use lib "$FindBin::Bin/../lib"; +use XCAT::BuildUtils qw( + capture_command + digest_manifest + read_first_line + relative_files + run_command + write_binary +); use XCAT::GenesisRelease qw( architectures deb_package_name @@ -60,20 +65,24 @@ for my $path (qw(Version xCAT-genesis-builder/oe/build xCAT-genesis-builder/oe/e die "xcat-core source is missing $path\n" unless -f "$xcat_source/$path"; } die "xcat-core checkout is not clean\n" - if _capture('git', '-C', $xcat_source, 'status', '--porcelain') ne ''; + if capture_command('git', '-C', $xcat_source, 'status', '--porcelain') ne ''; -my $revision = _capture('git', '-C', $xcat_source, 'rev-parse', 'HEAD'); +my $revision = capture_command('git', '-C', $xcat_source, 'rev-parse', 'HEAD'); die "Invalid xcat-core revision: $revision\n" unless $revision =~ /^[0-9a-f]{40}$/; if ($xcat_ref ne '') { die "Invalid xcat-core ref: $xcat_ref\n" unless $xcat_ref =~ /^[A-Za-z0-9][A-Za-z0-9._\/-]*$/; - my $expected = _capture('git', '-C', $xcat_source, 'rev-parse', '--verify', "$xcat_ref^{commit}"); + my $expected = capture_command( + 'git', '-C', $xcat_source, 'rev-parse', '--verify', "$xcat_ref^{commit}", + ); die "xcat-core HEAD $revision does not match $xcat_ref ($expected)\n" unless $revision eq $expected; } -my $version = _read_line("$xcat_source/Version"); +my $version = read_first_line("$xcat_source/Version"); die "Invalid xCAT version: $version\n" unless $version =~ /^\d+(?:\.\d+){1,3}$/; -my $source_date_epoch = _capture('git', '-C', $xcat_source, 'show', '-s', '--format=%ct', 'HEAD'); +my $source_date_epoch = capture_command( + 'git', '-C', $xcat_source, 'show', '-s', '--format=%ct', 'HEAD', +); die "Invalid xcat-core commit time\n" unless $source_date_epoch =~ /^\d+$/; my $release = strftime('snap%Y%m%d%H%M', gmtime($source_date_epoch)); @@ -101,7 +110,7 @@ local $ENV{XCAT_GENESIS_WORK_DIR} = $oe_work; { local $ENV{TMPDIR} = $oe_tmp; make_path($oe_tmp); - _run("$xcat_source/xCAT-genesis-builder/oe/build", @requested_architectures); + run_command("$xcat_source/xCAT-genesis-builder/oe/build", @requested_architectures); } my $effective_deploy = "$oe_tmp/deploy"; die "Invalid OpenEmbedded deploy directory: $effective_deploy\n" @@ -110,13 +119,13 @@ die "Invalid OpenEmbedded deploy directory: $effective_deploy\n" for my $architecture (@requested_architectures) { my $export = "$work/exports/$architecture"; make_path(dirname($export)); - _run( + run_command( "$xcat_source/xCAT-genesis-builder/oe/export", $architecture, $effective_deploy, $export, ); my $packages = "$work/packages/$architecture"; - _run( + run_command( "$FindBin::Bin/package", '--architecture', $architecture, '--export-dir', $export, @@ -131,7 +140,7 @@ for my $architecture (@requested_architectures) { } my @formats = $format eq 'all' ? qw(deb rpm) : ($format); -_write_file( +write_binary( "$staging/release.manifest", "format=xcat-genesis-packages\n" . "version=1\n" @@ -142,11 +151,15 @@ _write_file( . "architectures=" . join(',', @requested_architectures) . "\n" . "formats=" . join(',', @formats) . "\n", ); -_write_checksums($staging); +my @release_files = grep { $_ ne 'SHA256SUMS' } relative_files($staging); +write_binary( + "$staging/SHA256SUMS", + digest_manifest($staging, 'sha256', @release_files), +); validate_release($staging); my @verify_args = ('--format', $format); push(@verify_args, '--complete') if $all; -_run("$FindBin::Bin/verify-release", @verify_args, $staging); +run_command("$FindBin::Bin/verify-release", @verify_args, $staging); chmod(0755, $staging) or die "Cannot make release directory readable: $!\n"; rename($staging, $output_dir) or die "Cannot publish $output_dir: $!\n"; @@ -183,73 +196,6 @@ sub _collect_one { copy($source, $destination) or die "Cannot collect $source: $!\n"; } -sub _write_checksums { - my ($directory) = @_; - my @files; - find( - { - no_chdir => 1, - wanted => sub { - return unless -f $_ && !-l $_; - my $relative = File::Spec->abs2rel($File::Find::name, $directory); - $relative =~ tr{\\}{/}; - push(@files, $relative) unless $relative eq 'SHA256SUMS'; - }, - }, - $directory, - ); - my $content = ''; - for my $relative (sort @files) { - open(my $fh, '<:raw', "$directory/$relative") - or die "Cannot read $directory/$relative: $!\n"; - my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest; - close($fh) or die "Cannot close $directory/$relative: $!\n"; - $content .= "$digest $relative\n"; - } - _write_file("$directory/SHA256SUMS", $content); -} - -sub _read_line { - my ($path) = @_; - open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; - my $line = <$fh>; - close($fh) or die "Cannot close $path: $!\n"; - die "Empty file: $path\n" unless defined($line); - chomp($line); - $line =~ s/\r\z//; - return $line; -} - -sub _capture { - my (@command) = @_; - open(my $fh, '-|', @command) or die "Cannot run $command[0]: $!\n"; - local $/; - my $output = <$fh> // ''; - close($fh) or die "Command failed: $command[0]\n"; - $output =~ s/\s+\z//; - return $output; -} - -sub _run { - my (@command) = @_; - print '+ ', join(' ', map { _display_quote($_) } @command), "\n"; - system(@command) == 0 or die "Command failed: $command[0]\n"; -} - -sub _display_quote { - my ($value) = @_; - return $value if $value =~ /^[A-Za-z0-9_.,+\/:=@~-]+$/; - $value =~ s/'/'\\''/g; - return "'$value'"; -} - -sub _write_file { - my ($path, $content) = @_; - open(my $fh, '>:raw', $path) or die "Cannot write $path: $!\n"; - print {$fh} $content or die "Cannot write $path: $!\n"; - close($fh) or die "Cannot close $path: $!\n"; -} - sub usage { return <<'USAGE'; Usage: build [--xcat-source DIR] [--xcat-ref REF] [--output-dir DIR] diff --git a/genesis-openembedded/package b/genesis-openembedded/package index 977cee8..683ed25 100755 --- a/genesis-openembedded/package +++ b/genesis-openembedded/package @@ -4,15 +4,21 @@ use strict; use warnings; use Cwd qw(abs_path); -use Digest::MD5 (); use File::Basename qw(basename dirname); use File::Copy qw(copy); use File::Path qw(make_path remove_tree); -use File::Spec; use File::Temp qw(tempdir); use FindBin; use Getopt::Long qw(GetOptions); -use lib "$FindBin::Bin/lib"; +use lib "$FindBin::Bin/../lib"; +use XCAT::BuildUtils qw( + capture_command + digest_manifest + relative_files + require_command + run_command + write_binary +); use XCAT::GenesisRelease qw( deb_package_name rpm_package_name @@ -84,16 +90,16 @@ for my $entry (_flat_files($export_dir)) { copy("$export_dir/$entry", "$source_root/image/$entry") or die "Cannot copy export file $entry: $!\n"; } -_write_file("$source_root/xcat-core-revision", "$revision\n"); +write_binary("$source_root/xcat-core-revision", "$revision\n"); local $ENV{SOURCE_DATE_EPOCH} = $source_date_epoch; if ($format eq 'all' || $format eq 'rpm') { - _require_command('rpmbuild'); + require_command('rpmbuild'); _build_rpm($work, $staging, $source_root, $source_name, $rpm_name); } if ($format eq 'all' || $format eq 'deb') { - _require_command('dpkg-deb'); + require_command('dpkg-deb'); _build_deb($work, $staging, $source_root, $deb_name); } @@ -111,7 +117,7 @@ sub _build_rpm { make_path("$staging/rpm", "$staging/srpm"); my $archive = "$rpm_top/SOURCES/$source_name.tar.gz"; - _run( + run_command( 'tar', '--sort=name', '--owner=0', '--group=0', '--numeric-owner', "--mtime=\@$source_date_epoch", '--use-compress-program=gzip -n', '-cf', $archive, '-C', dirname($source_root), basename($source_root), @@ -119,7 +125,7 @@ sub _build_rpm { my $spec = "$FindBin::Bin/rpm/xCAT-genesis-openembedded.spec"; copy($spec, "$rpm_top/SPECS/xCAT-genesis-openembedded.spec") or die "Cannot stage RPM spec: $!\n"; - _run( + run_command( 'rpmbuild', '-ba', '--define', "_topdir $rpm_top", '--define', "genesis_arch $architecture", @@ -162,7 +168,7 @@ sub _build_deb { my $installed_kib = _tree_bytes($root); $installed_kib = int(($installed_kib + 1023) / 1024); - _write_file( + write_binary( "$root/DEBIAN/control", "Package: $package_name\n" . "Version: $version-$release\n" @@ -177,36 +183,17 @@ sub _build_deb { _normalize_mtime($root, $source_date_epoch); my $deb = "$staging/deb/${package_name}_${version}-${release}_all.deb"; - _run('dpkg-deb', '--root-owner-group', '-Zgzip', '-z9', '--build', $root, $deb); + run_command('dpkg-deb', '--root-owner-group', '-Zgzip', '-z9', '--build', $root, $deb); die "DEB build did not produce $deb\n" unless -f $deb; } sub _write_deb_md5sums { my ($root) = @_; - my @files; - require File::Find; - File::Find::find( - { - no_chdir => 1, - wanted => sub { - return unless -f $_ && !-l $_; - return if $File::Find::name =~ m{\Q$root/DEBIAN/\E}; - my $relative = File::Spec->abs2rel($File::Find::name, $root); - $relative =~ tr{\\}{/}; - push(@files, $relative); - }, - }, - $root, + my @files = grep { $_ !~ m{\ADEBIAN/} } relative_files($root); + write_binary( + "$root/DEBIAN/md5sums", + digest_manifest($root, 'md5', @files), ); - my $content = ''; - for my $relative (sort @files) { - open(my $fh, '<:raw', "$root/$relative") - or die "Cannot read $root/$relative: $!\n"; - my $digest = Digest::MD5->new->addfile($fh)->hexdigest; - close($fh) or die "Cannot close $root/$relative: $!\n"; - $content .= "$digest $relative\n"; - } - _write_file("$root/DEBIAN/md5sums", $content); } sub _normalize_mtime { @@ -226,12 +213,9 @@ sub _normalize_mtime { sub _flat_files { my ($directory) = @_; - opendir(my $dh, $directory) or die "Cannot read $directory: $!\n"; - my @entries = sort grep { $_ ne '.' && $_ ne '..' } readdir($dh); - closedir($dh) or die "Cannot close $directory: $!\n"; + my @entries = relative_files($directory); for my $entry (@entries) { - my $path = "$directory/$entry"; - die "Expected a regular file: $path\n" unless -f $path && !-l $path; + die "Expected a flat directory: $directory\n" if $entry =~ m{/}; } return @entries; } @@ -239,51 +223,17 @@ sub _flat_files { sub _tree_bytes { my ($directory) = @_; my $bytes = 0; - require File::Find; - File::Find::find( - { no_chdir => 1, wanted => sub { $bytes += -s $_ if -f $_ && !-l $_ } }, - $directory, - ); + $bytes += -s "$directory/$_" for relative_files($directory); return $bytes; } -sub _require_command { - my ($command) = @_; - for my $directory (File::Spec->path()) { - return if -x "$directory/$command"; - } - die "Required command not found: $command\n"; -} - sub _require_gnu_tar { - _require_command('tar'); - open(my $fh, '-|', 'tar', '--version') or die "Cannot run tar: $!\n"; - my $version_line = <$fh> // ''; - close($fh) or die "Cannot query tar version\n"; + require_command('tar'); + my $version_line = capture_command('tar', '--version'); die "GNU tar is required to build Genesis source packages\n" unless $version_line =~ /^tar \(GNU tar\)/; } -sub _run { - my (@command) = @_; - print '+ ', join(' ', map { _display_quote($_) } @command), "\n"; - system(@command) == 0 or die "Command failed: $command[0]\n"; -} - -sub _display_quote { - my ($value) = @_; - return $value if $value =~ /^[A-Za-z0-9_.,+\/:=@~-]+$/; - $value =~ s/'/'\\''/g; - return "'$value'"; -} - -sub _write_file { - my ($path, $content) = @_; - open(my $fh, '>:raw', $path) or die "Cannot write $path: $!\n"; - print {$fh} $content or die "Cannot write $path: $!\n"; - close($fh) or die "Cannot close $path: $!\n"; -} - sub usage { return <<'USAGE'; Usage: package --architecture ARCH --export-dir DIR --output-dir DIR diff --git a/genesis-openembedded/verify-release b/genesis-openembedded/verify-release index 623a748..da7563a 100755 --- a/genesis-openembedded/verify-release +++ b/genesis-openembedded/verify-release @@ -4,10 +4,10 @@ use strict; use warnings; use Cwd qw(abs_path); -use File::Spec; use FindBin; use Getopt::Long qw(GetOptions); -use lib "$FindBin::Bin/lib"; +use lib "$FindBin::Bin/../lib"; +use XCAT::BuildUtils qw(capture_command require_command); use XCAT::GenesisRelease qw( deb_package_name rpm_package_name @@ -49,7 +49,7 @@ print "Verified Genesis package release: $directory\n"; sub _verify_rpm { my ($directory, $manifest, $architecture) = @_; - _require_command('rpm'); + require_command('rpm'); my $name = rpm_package_name($architecture); my $source_name = "$name-$manifest->{xcat_version}-$manifest->{xcat_release}.src.rpm"; my @packages = ( @@ -59,7 +59,7 @@ sub _verify_rpm { ); for my $package (@packages) { my ($path, $expected_arch, $expected_source) = @{$package}; - my $metadata = _capture( + my $metadata = capture_command( 'rpm', '-qp', '--qf', "%{NAME}\t%{VERSION}\t%{RELEASE}\t%{ARCH}\t%{EPOCHNUM}\t%{SOURCERPM}" . "\t%{BUILDHOST}\t%{BUILDTIME}\n", @@ -72,10 +72,10 @@ sub _verify_rpm { ); die "Unexpected RPM identity for $path: $metadata\n" unless $metadata eq $expected; for my $relationship (qw(conflicts obsoletes)) { - my $value = _capture('rpm', '-qp', "--$relationship", $path); + my $value = capture_command('rpm', '-qp', "--$relationship", $path); die "Unexpected RPM $relationship for $path: $value\n" if $value ne ''; } - my $provides = _capture('rpm', '-qp', '--provides', $path); + my $provides = capture_command('rpm', '-qp', '--provides', $path); die "Legacy RPM relationship in $path: $provides\n" if $provides =~ /xCAT-genesis-base/; } @@ -83,12 +83,12 @@ sub _verify_rpm { sub _verify_deb { my ($directory, $manifest, $architecture) = @_; - _require_command('dpkg-deb'); + require_command('dpkg-deb'); my $name = deb_package_name($architecture); my $path = "$directory/deb/${name}_$manifest->{xcat_version}-$manifest->{xcat_release}_all.deb"; my $metadata = join( "\n", - map { _capture('dpkg-deb', '-f', $path, $_) } + map { capture_command('dpkg-deb', '-f', $path, $_) } qw(Package Version Architecture), ); my $expected = join( @@ -97,29 +97,11 @@ sub _verify_deb { ); die "Unexpected DEB identity for $path: $metadata\n" unless $metadata eq $expected; for my $field (qw(Breaks Conflicts Provides Replaces)) { - my $value = _capture('dpkg-deb', '-f', $path, $field); + my $value = capture_command('dpkg-deb', '-f', $path, $field); die "Unexpected DEB $field for $path: $value\n" if $value ne ''; } } -sub _require_command { - my ($command) = @_; - for my $directory (File::Spec->path()) { - return if -x "$directory/$command"; - } - die "Required command not found: $command\n"; -} - -sub _capture { - my (@command) = @_; - open(my $fh, '-|', @command) or die "Cannot run $command[0]: $!\n"; - local $/; - my $output = <$fh> // ''; - close($fh) or die "Command failed: $command[0]\n"; - $output =~ s/\s+\z//; - return $output; -} - sub usage { return "Usage: verify-release [--format all|rpm|deb] [--complete] RELEASE_DIRECTORY\n"; } diff --git a/lib/XCAT/BuildUtils.pm b/lib/XCAT/BuildUtils.pm new file mode 100644 index 0000000..c8fa7b0 --- /dev/null +++ b/lib/XCAT/BuildUtils.pm @@ -0,0 +1,160 @@ +package XCAT::BuildUtils; + +use strict; +use warnings; + +use Digest::MD5 (); +use Digest::SHA (); +use Exporter qw(import); +use File::Find qw(find); +use File::Slurper qw(read_binary write_binary); +use File::Spec; +use IPC::Cmd qw(can_run); + +our @EXPORT_OK = qw( + capture_command + command_exists + digest_file + digest_manifest + display_quote + hashes_equal + print_step + read_binary + read_first_line + read_lines + relative_files + require_command + run_command + shell_quote + write_binary +); + +sub command_exists { + my ($command) = @_; + return defined(can_run($command)); +} + +sub require_command { + my ($command) = @_; + return can_run($command) + // die "Required command not found: $command\n"; +} + +sub capture_command { + my (@command) = @_; + open(my $fh, '-|', @command) or die "Cannot run $command[0]: $!\n"; + local $/; + my $output = <$fh> // ''; + close($fh) or die "Command failed: $command[0]\n"; + $output =~ s/\s+\z//; + return $output; +} + +sub run_command { + my (@command) = @_; + print '+ ', join(' ', map { display_quote($_) } @command), "\n"; + my $status = system(@command); + return 1 if $status == 0; + + my $exit = $status == -1 + ? 255 + : ($status & 127) ? 128 + ($status & 127) : $status >> 8; + die "Command failed (rc=$exit): " + . join(' ', map { display_quote($_) } @command) . "\n"; +} + +sub display_quote { + my ($value) = @_; + return $value if $value =~ /^[A-Za-z0-9_.,+\/:=@~-]+$/; + return shell_quote($value); +} + +sub shell_quote { + my ($value) = @_; + $value = '' unless defined($value); + $value =~ s/'/'"'"'/g; + return "'$value'"; +} + +sub print_step { + my ($message) = @_; + print "\n== $message ==\n"; +} + +sub read_lines { + my ($path) = @_; + my $content = read_binary($path); + return () if $content eq ''; + + my @lines = split(/\n/, $content, -1); + pop(@lines) if @lines && $lines[-1] eq ''; + s/\r\z// for @lines; + return @lines; +} + +sub read_first_line { + my ($path) = @_; + my @lines = read_lines($path); + die "Empty file: $path\n" unless @lines; + return $lines[0]; +} + +sub relative_files { + my ($root) = @_; + die "Invalid directory: $root\n" unless -d $root && !-l $root; + + my $absolute = File::Spec->rel2abs($root); + my @files; + find( + { + no_chdir => 1, + wanted => sub { + my $path = $File::Find::name; + return if $path eq $absolute; + die "Symbolic links are not allowed: $path\n" if -l $path; + return if -d $path; + die "Non-regular entry: $path\n" unless -f $path; + my $relative = File::Spec->abs2rel($path, $absolute); + $relative =~ tr{\\}{/}; + push(@files, $relative); + }, + }, + $absolute, + ); + my @sorted = sort @files; + return @sorted; +} + +sub digest_file { + my ($path, $algorithm) = @_; + $algorithm //= 'sha256'; + + my $digest = $algorithm eq 'sha256' ? Digest::SHA->new(256) + : $algorithm eq 'md5' ? Digest::MD5->new + : die "Unsupported digest algorithm: $algorithm\n"; + open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; + my $value = $digest->addfile($fh)->hexdigest; + close($fh) or die "Cannot close $path: $!\n"; + return $value; +} + +sub digest_manifest { + my ($root, $algorithm, @files) = @_; + return join( + '', + map { digest_file("$root/$_", $algorithm) . " $_\n" } + sort @files, + ); +} + +sub hashes_equal { + my ($left, $right) = @_; + return 0 unless keys(%{$left}) == keys(%{$right}); + for my $name (keys %{$left}) { + return 0 unless exists($right->{$name}) + && $left->{$name} eq $right->{$name}; + } + return 1; +} + +1; diff --git a/genesis-openembedded/lib/XCAT/GenesisRelease.pm b/lib/XCAT/GenesisRelease.pm similarity index 81% rename from genesis-openembedded/lib/XCAT/GenesisRelease.pm rename to lib/XCAT/GenesisRelease.pm index f10b2ee..0bcc9ae 100644 --- a/genesis-openembedded/lib/XCAT/GenesisRelease.pm +++ b/lib/XCAT/GenesisRelease.pm @@ -3,10 +3,8 @@ package XCAT::GenesisRelease; use strict; use warnings; -use Digest::SHA (); use Exporter qw(import); -use File::Find qw(find); -use File::Spec; +use XCAT::BuildUtils qw(digest_file read_lines relative_files); our @EXPORT_OK = qw( architectures @@ -50,11 +48,8 @@ sub deb_package_name { sub _read_key_values { my ($path, $allowed) = @_; - open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; my %values; - while (my $line = <$fh>) { - chomp($line); - $line =~ s/\r\z//; + for my $line (read_lines($path)) { die "Invalid manifest entry in $path: $line\n" unless $line =~ /^([a-z][a-z0-9_]*)=([A-Za-z0-9][A-Za-z0-9.,_+~-]*)$/; my ($key, $value) = ($1, $2); @@ -62,43 +57,14 @@ sub _read_key_values { die "Duplicate manifest key in $path: $key\n" if exists($values{$key}); $values{$key} = $value; } - close($fh) or die "Cannot close $path: $!\n"; return \%values; } -sub _regular_files { - my ($root) = @_; - die "Invalid directory: $root\n" unless -d $root && !-l $root; - - my @files; - find( - { - no_chdir => 1, - wanted => sub { - my $path = $File::Find::name; - return if $path eq $root; - die "Symbolic links are not allowed: $path\n" if -l $path; - return if -d $path; - die "Non-regular release entry: $path\n" unless -f $path; - my $relative = File::Spec->abs2rel($path, $root); - $relative =~ tr{\\}{/}; - push(@files, $relative); - }, - }, - $root, - ); - my @sorted = sort @files; - return @sorted; -} - sub _read_checksums { my ($root) = @_; my $path = "$root/SHA256SUMS"; - open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; my %checksums; - while (my $line = <$fh>) { - chomp($line); - $line =~ s/\r\z//; + for my $line (read_lines($path)) { die "Invalid checksum entry in $path: $line\n" unless $line =~ /^([0-9a-f]{64}) ([A-Za-z0-9][A-Za-z0-9._\/+~-]*)$/; my ($digest, $name) = ($1, $2); @@ -108,21 +74,18 @@ sub _read_checksums { if exists($checksums{$name}); $checksums{$name} = $digest; } - close($fh) or die "Cannot close $path: $!\n"; return \%checksums; } sub _verify_checksums { my ($root) = @_; - my @files = grep { $_ ne 'SHA256SUMS' } _regular_files($root); + my @files = grep { $_ ne 'SHA256SUMS' } relative_files($root); my $checksums = _read_checksums($root); my %files = map { $_ => 1 } @files; for my $name (@files) { die "Missing checksum for $name\n" unless exists($checksums->{$name}); - open(my $fh, '<:raw', "$root/$name") or die "Cannot read $root/$name: $!\n"; - my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest; - close($fh) or die "Cannot close $root/$name: $!\n"; + my $digest = digest_file("$root/$name", 'sha256'); die "Checksum mismatch for $name\n" unless $digest eq $checksums->{$name}; } for my $name (keys %{$checksums}) { @@ -148,7 +111,7 @@ sub validate_export { ); $required{'fw_jump.elf'} = 1 if $architecture eq 'riscv64'; - my @files = _regular_files($directory); + my @files = relative_files($directory); my %files = map { $_ => 1 } @files; for my $name (sort keys %required) { die "Genesis export is missing $name\n" unless $files{$name}; @@ -240,7 +203,7 @@ sub validate_release { $expected{"deb/${deb}_$manifest->{xcat_version}-$manifest->{xcat_release}_all.deb"} = 1; } } - for my $file (grep { $_ ne 'SHA256SUMS' } _regular_files($directory)) { + for my $file (grep { $_ ne 'SHA256SUMS' } relative_files($directory)) { die "Unexpected Genesis release artifact: $file\n" unless $expected{$file}; delete($expected{$file}); } @@ -269,9 +232,7 @@ sub verify_release_file { die "Missing verified checksum for $relative\n" unless ref($checksums) eq 'HASH' && exists($checksums->{$relative}); die "Invalid collected release file: $path\n" unless -f $path && !-l $path; - open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; - my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest; - close($fh) or die "Cannot close $path: $!\n"; + my $digest = digest_file($path, 'sha256'); die "Collected release file checksum mismatch: $path\n" unless $digest eq $checksums->{$relative}; return 1; diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 9de89af..8711969 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -13,7 +13,16 @@ use FindBin; use Getopt::Long qw(GetOptions); use Parallel::ForkManager; use POSIX qw(strftime); -use lib "$FindBin::Bin/genesis-openembedded/lib"; +use lib "$FindBin::Bin/lib"; +use XCAT::BuildUtils qw( + capture_command + hashes_equal + print_step + read_first_line + require_command + run_command + shell_quote +); use XCAT::GenesisRelease qw( validated_release_checksums verify_release_file @@ -96,7 +105,7 @@ $repo_root = abs_path($repo_root); my $SOURCE_DATE_EPOCH; $SOURCE_DATE_EPOCH = $build_timestamp if defined $build_timestamp; if (!$SOURCE_DATE_EPOCH && -f "$repo_root/Gitepoch") { - $SOURCE_DATE_EPOCH = slurp_chomp("$repo_root/Gitepoch"); + $SOURCE_DATE_EPOCH = read_first_line("$repo_root/Gitepoch"); } unless ($SOURCE_DATE_EPOCH && $SOURCE_DATE_EPOCH =~ /^\d+$/) { $SOURCE_DATE_EPOCH = `git -C \Q$repo_root\E log -1 --format=%ct HEAD 2>/dev/null`; @@ -128,7 +137,7 @@ acquire_output_lock($output_base, $force_unlock); $xcat_src = resolve_xcat_source($xcat_src, $repo_root); -my $arch = capture('uname -m'); +my $arch = capture_command('uname', '-m'); my %os = read_os_release('/etc/os-release'); my $os_id = $os{ID} // ''; my $version_id = $os{VERSION_ID} // ''; @@ -153,10 +162,10 @@ if ($genesis_release ne '') { my $verifier = "$script_dir/genesis-openembedded/verify-release"; die "Genesis release verifier not found: $verifier\n" unless -x $verifier; my $checksums_before = validated_release_checksums($genesis_release); - run_argv($^X, $verifier, '--complete', '--format', 'rpm', $genesis_release); + run_command($^X, $verifier, '--complete', '--format', 'rpm', $genesis_release); my $checksums_after = validated_release_checksums($genesis_release); die "Genesis release changed during verification\n" - unless checksum_sets_match($checksums_before, $checksums_after); + unless hashes_equal($checksums_before, $checksums_after); $genesis_release_checksums = $checksums_before; } @@ -190,7 +199,7 @@ $tgt_workers = scalar(@build_targets) if $tgt_workers > scalar(@build_targets); # mock build already gets a unique --uniqueext (separate chroot), so the only limit needed is # hardware: total concurrent builds across all targets stays <= $cap (default host nproc). The # per-target build-step concurrency is therefore the cap divided across the active targets. -my $cap = $max_parallel > 0 ? $max_parallel : (capture('nproc') || 4); +my $cap = $max_parallel > 0 ? $max_parallel : (capture_command('nproc') || 4); my $per_target_builds = defined($parallel_builds) ? $parallel_builds : int($cap / $tgt_workers); $per_target_builds = 1 if $per_target_builds < 1; print "parallel_targets: " . ($parallel_targets > 0 ? $parallel_targets : "auto($tgt_workers)") . "\n"; @@ -304,7 +313,7 @@ my @collect_roots; if ($scrub_all_chroots) { run_step( step => "Scrub all chroots for target $target", - cmd => "mock -r " . sh_quote($target) . " --scrub=all", + cmd => "mock -r " . shell_quote($target) . " --scrub=all", log => "$log_root/scrub-all-chroots.log", ); } @@ -321,13 +330,13 @@ if (!$skip_build) { my $step_log = "$log_root/$name"; my $step_uniqueext = build_mock_uniqueext($run_id, ++$build_step_seq, $name); my $cmd = join(' ', - 'perl', sh_quote($script), - '--mock-cfg', sh_quote($target), - '--mock-uniqueext', sh_quote($step_uniqueext), - '--result-dir', sh_quote($step_result), - '--log-dir', sh_quote($step_log), + 'perl', shell_quote($script), + '--mock-cfg', shell_quote($target), + '--mock-uniqueext', shell_quote($step_uniqueext), + '--result-dir', shell_quote($step_result), + '--log-dir', shell_quote($step_log), # host-local, run-scoped work dir so /tmp doesn't collide between runs - '--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/$name"), + '--work-dir', shell_quote("/tmp/mockbuild-all-$run_id/$name"), '--build-timestamp', $SOURCE_DATE_EPOCH, ($skip_install ? '--skip-install' : ()), ); @@ -349,12 +358,12 @@ if (!$skip_build) { # forks one mock build per perl package (~7), which -- multiplied by parallel EL targets -- # oversubscribes the host. my $cmd = join(' ', - 'perl', sh_quote($perl_builder), - '--mock-cfg', sh_quote($target), - '--mock-uniqueext', sh_quote($perl_uniqueext), - '--result-dir', sh_quote($perl_result), - '--log-dir', sh_quote($perl_log), - '--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/perl-list6"), + 'perl', shell_quote($perl_builder), + '--mock-cfg', shell_quote($target), + '--mock-uniqueext', shell_quote($perl_uniqueext), + '--result-dir', shell_quote($perl_result), + '--log-dir', shell_quote($perl_log), + '--work-dir', shell_quote("/tmp/mockbuild-all-$run_id/perl-list6"), (($max_build_workers && $max_build_workers >= 1) ? ('--jobs', $max_build_workers) : ()), '--build-timestamp', $SOURCE_DATE_EPOCH, ($skip_install ? '--skip-install' : ()), @@ -371,14 +380,14 @@ if (!$skip_build) { if (!$skip_xcat) { # Own HOME per target (buildrpms.pl uses $HOME/rpmbuild) so parallel targets don't race. my $xcat_home = "/tmp/mockbuild-all-$run_id/xcat-home"; - my $mktree = join(' ', map { sh_quote("$xcat_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS)); - my $cmd = "mkdir -p $mktree && HOME=" . sh_quote($xcat_home) . ' ' . join(' ', - 'perl', sh_quote("$xcat_src/buildrpms.pl"), - '--target', sh_quote($target), + my $mktree = join(' ', map { shell_quote("$xcat_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS)); + my $cmd = "mkdir -p $mktree && HOME=" . shell_quote($xcat_home) . ' ' . join(' ', + 'perl', shell_quote("$xcat_src/buildrpms.pl"), + '--target', shell_quote($target), '--nproc', int($nproc), '--force', '--verbose', - '--xcat_dep_path', sh_quote($repo_root), + '--xcat_dep_path', shell_quote($repo_root), ); push @build_steps, { id => 'xcat', @@ -402,15 +411,15 @@ if (!$skip_build) { my $genesis_home = "/tmp/mockbuild-all-$run_id/genesis-home"; # buildrpms.pl's rpmdev-setuptree only runs during env setup, not per build, so create the # rpmbuild tree ourselves for this per-target HOME (else $HOME/rpmbuild/SOURCES is missing). - my $mktree = join(' ', map { sh_quote("$genesis_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS)); - my $cmd = "mkdir -p $mktree && HOME=" . sh_quote($genesis_home) . ' ' . join(' ', - 'perl', sh_quote("$xcat_src/buildrpms.pl"), + my $mktree = join(' ', map { shell_quote("$genesis_home/rpmbuild/$_") } qw(SOURCES SPECS BUILD BUILDROOT RPMS SRPMS)); + my $cmd = "mkdir -p $mktree && HOME=" . shell_quote($genesis_home) . ' ' . join(' ', + 'perl', shell_quote("$xcat_src/buildrpms.pl"), '--package', 'xCAT-genesis-base', - '--target', sh_quote($target), + '--target', shell_quote($target), '--nproc', int($nproc), '--force', '--verbose', - '--xcat_dep_path', sh_quote($repo_root), + '--xcat_dep_path', shell_quote($repo_root), ); push @build_steps, { id => 'genesis', @@ -537,8 +546,8 @@ if (!$skip_tarball) { my $cmd = join(' ', 'tar', '--sort=name', '--owner=0', '--group=0', "--mtime=\@$SOURCE_DATE_EPOCH", - '-C', sh_quote($run_root), - '-czf', sh_quote($tarball), + '-C', shell_quote($run_root), + '-czf', shell_quote($tarball), 'repo' ); run_step( @@ -549,8 +558,8 @@ if (!$skip_tarball) { my $srpm_cmd = join(' ', 'tar', '--sort=name', '--owner=0', '--group=0', "--mtime=\@$SOURCE_DATE_EPOCH", - '-C', sh_quote($run_root), - '-czf', sh_quote($srpm_tarball), + '-C', shell_quote($run_root), + '-czf', shell_quote($srpm_tarball), 'repo-src' ); run_step( @@ -637,8 +646,8 @@ sub deploy_target { sub createrepo_c_cmd { my ($dir) = @_; return 'createrepo_c --update --database ' - . '--revision ' . sh_quote($SOURCE_DATE_EPOCH) . ' --set-timestamp-to-revision ' - . sh_quote($dir); + . '--revision ' . shell_quote($SOURCE_DATE_EPOCH) . ' --set-timestamp-to-revision ' + . shell_quote($dir); } sub sign_and_index_repo { @@ -647,15 +656,15 @@ sub sign_and_index_repo { if ($gpg_sign && @rpms) { local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; run_simple(qq(rpmsign --define "%_gpg_name $gpg_key_name" --addsign ) - . join(' ', map { sh_quote($_) } @rpms)); + . join(' ', map { shell_quote($_) } @rpms)); } run_simple(createrepo_c_cmd($dir)); if ($gpg_sign) { local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; my $repomd = "$dir/repodata/repomd.xml"; unlink "$repomd.asc" if -f "$repomd.asc"; - run_simple(qq(gpg -a --detach-sign --default-key "$gpg_key_name" ) . sh_quote($repomd)); - run_simple(qq(gpg -a --export "$gpg_key_name" > ) . sh_quote("$repomd.key")); + run_simple(qq(gpg -a --detach-sign --default-key "$gpg_key_name" ) . shell_quote($repomd)); + run_simple(qq(gpg -a --export "$gpg_key_name" > ) . shell_quote("$repomd.key")); } } @@ -769,16 +778,6 @@ Notes: USAGE } -sub print_step { - my ($msg) = @_; - print "\n== $msg ==\n"; -} - -sub require_command { - my ($cmd) = @_; - run_simple("command -v " . sh_quote($cmd) . " >/dev/null 2>&1"); -} - sub run_simple { my ($cmd) = @_; my $rc = system($cmd); @@ -788,27 +787,6 @@ sub run_simple { } } -sub run_argv { - my (@command) = @_; - my $rc = system(@command); - if ($rc != 0) { - my $exit = $rc == -1 ? 255 : ($rc >> 8); - die "Command failed (rc=$exit): @command\n"; - } -} - -sub capture { - my ($cmd) = @_; - my $out = `$cmd`; - my $rc = $?; - if ($rc != 0) { - my $exit = $rc == -1 ? 255 : ($rc >> 8); - die "Command failed (rc=$exit): $cmd\n$out\n"; - } - chomp $out; - return $out; -} - sub run_step { my (%args) = @_; my $step = $args{step} // 'Run command'; @@ -829,12 +807,12 @@ sub run_step { my $full_cmd = $cmd; if ($cwd) { - $full_cmd = "cd " . sh_quote($cwd) . " && $cmd"; + $full_cmd = "cd " . shell_quote($cwd) . " && $cmd"; } if ($log) { my $log_dir = dirname($log); make_path($log_dir) if !-d $log_dir; - $full_cmd .= " > " . sh_quote($log) . " 2>&1"; + $full_cmd .= " > " . shell_quote($log) . " 2>&1"; } my $rc = system($full_cmd); @@ -1006,15 +984,6 @@ sub assert_genesis_release_copied { } } -sub checksum_sets_match { - my ($left, $right) = @_; - return 0 unless keys(%{$left}) == keys(%{$right}); - for my $name (keys %{$left}) { - return 0 unless exists($right->{$name}) && $left->{$name} eq $right->{$name}; - } - return 1; -} - sub collect_rpms { my (%args) = @_; my $roots = $args{roots} // []; @@ -1125,14 +1094,14 @@ sub resolve_mock_cfg { rocky => 'rocky', ); my $candidate = "${os_id}+epel-${rel}-${arch}"; - my $rc = system("mock -r " . sh_quote($candidate) . " --print-root-path >/dev/null 2>&1"); + my $rc = system("mock -r " . shell_quote($candidate) . " --print-root-path >/dev/null 2>&1"); if ($rc == 0) { return $candidate; } if (exists $short_forms{$os_id}) { my $short = $short_forms{$os_id}; $candidate = "${short}+epel-${rel}-${arch}"; - $rc = system("mock -r " . sh_quote($candidate) . " --print-root-path >/dev/null 2>&1"); + $rc = system("mock -r " . shell_quote($candidate) . " --print-root-path >/dev/null 2>&1"); if ($rc == 0) { print "Mock config resolved (short form): $candidate\n"; return $candidate; @@ -1193,7 +1162,7 @@ sub acquire_output_lock { if (mkdir $lock) { $HELD_LOCK = $lock; $LOCK_OWNER_PID = $$; - my $host = capture('uname -n') || 'unknown'; + my $host = capture_command('uname', '-n') || 'unknown'; if (open my $fh, '>', "$lock/owner") { print {$fh} "host=$host\npid=$$\nepoch=" . time() . "\n"; close $fh; @@ -1250,19 +1219,3 @@ sub uniq { my %seen; return grep { defined($_) && !$seen{$_}++ } @_; } - -sub slurp_chomp { - my ($path) = @_; - open my $fh, '<', $path or die "Cannot read $path: $!\n"; - my $line = <$fh>; - close $fh; - chomp $line if defined $line; - return $line // ''; -} - -sub sh_quote { - my ($s) = @_; - $s = '' if !defined $s; - $s =~ s/'/'"'"'/g; - return "'$s'"; -} From ffbf576a5f6e98882a0fbbc904fc98aaa9069647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:26:03 -0300 Subject: [PATCH 51/82] test(genesis): reuse build helpers --- t/genesis_openembedded_consumer.t | 75 ++++++++++++--------------- t/genesis_openembedded_release.t | 65 +++++++++++------------ t/lib/XCAT/GenesisReleaseTest.pm | 86 ++++++------------------------- 3 files changed, 82 insertions(+), 144 deletions(-) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index 36070f2..cdefb70 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -10,21 +10,23 @@ use FindBin; use Test::More; use lib "$FindBin::Bin/../lib"; -use lib "$FindBin::Bin/../genesis-openembedded/lib"; use lib "$FindBin::Bin/lib"; +use XCAT::BuildUtils qw( + capture_command + command_exists + digest_file + read_binary + write_binary +); use XCAT::GenesisRelease qw( architectures deb_package_name rpm_package_name ); use XCAT::GenesisReleaseTest qw( - command_exists - file_sha make_export - read_file run_capture write_checksums - write_file write_release_manifest ); @@ -74,15 +76,15 @@ sub test_rpm_consumer { my $package = "xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; my $source_package = "xCAT-genesis-openembedded-x86_64-$version-$release.src.rpm"; my $output = "$tmp/rpm output"; - my $target = 'test+epel-10-' . capture('uname', '-m'); + my $target = 'test+epel-10-' . capture_command('uname', '-m'); my $run = "$target-consumer"; - my $run_repo = "$output/mockbuild-all/$run/repo/" . capture('uname', '-m'); + my $run_repo = "$output/mockbuild-all/$run/repo/" . capture_command('uname', '-m'); my $source_repo = "$output/mockbuild-all/$run/repo-src"; - my $deploy_repo = "$output/xcat-dep/rh10/" . capture('uname', '-m'); + my $deploy_repo = "$output/xcat-dep/rh10/" . capture_command('uname', '-m'); make_path($run_repo, $source_repo, $deploy_repo); - write_file("$run_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale'); - write_file("$source_repo/xCAT-genesis-openembedded-stale.src.rpm", 'stale'); - write_file("$deploy_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale'); + write_binary("$run_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale'); + write_binary("$source_repo/xCAT-genesis-openembedded-stale.src.rpm", 'stale'); + write_binary("$deploy_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale'); my $dependencies = "$tmp/rpm-dependencies"; my $scratch_repo_root = "$tmp/rpm-repo-root"; @@ -95,18 +97,18 @@ sub test_rpm_consumer { copy("$release_root/rpm/$package", "$dependencies/$name-1.noarch.rpm") or die $!; } - write_file( + write_binary( "$dependencies/xCAT-genesis-openembedded-x86_64-$version-old.noarch.rpm", 'stale OpenEmbedded RPM', ); - write_file( + write_binary( "$dependencies/xCAT-genesis-openembedded-x86_64-$version-old.src.rpm", 'stale OpenEmbedded SRPM', ); my $stub = "$tmp/perl-stub/Parallel/ForkManager.pm"; make_path("$tmp/perl-stub/Parallel"); - write_file( + write_binary( $stub, "package Parallel::ForkManager;\n" . "sub new { bless {}, shift }\n" @@ -137,7 +139,7 @@ sub test_rpm_consumer { ); is($status, 0, 'RPM repository accepts a verified Genesis release'); - is(file_sha("$deploy_repo/$package"), file_sha("$release_root/rpm/$package"), + is(digest_file("$deploy_repo/$package"), digest_file("$release_root/rpm/$package"), 'deployed RPM matches the release'); ok(!-e "$run_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale run RPM is removed'); @@ -145,8 +147,8 @@ sub test_rpm_consumer { 'stale source RPM is removed'); ok(!-e "$deploy_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale deployed RPM is removed'); - is(file_sha("$source_repo/$source_package"), - file_sha("$release_root/srpm/$source_package"), + is(digest_file("$source_repo/$source_package"), + digest_file("$release_root/srpm/$source_package"), 'source RPM matches the release'); ok(-f "$deploy_repo/repodata/repomd.xml", 'RPM repository metadata is generated'); ok(-f "$deploy_repo/xCAT-genesis-base-x86_64-1.noarch.rpm", @@ -155,7 +157,7 @@ sub test_rpm_consumer { 'stale OpenEmbedded RPM is not collected'); ok(!-e "$source_repo/xCAT-genesis-openembedded-x86_64-$version-old.src.rpm", 'stale OpenEmbedded SRPM is not collected'); - like(read_file("$output/mockbuild-all/$run/summary.txt"), qr/^copied_rpms=15$/m, + like(read_binary("$output/mockbuild-all/$run/summary.txt"), qr/^copied_rpms=15$/m, 'release RPM is counted alongside required dependencies'); } @@ -166,7 +168,7 @@ sub test_deb_consumer { my $input = "$apt_root/ubuntu24.04"; my $dummy = "$tmp/dummy-deb"; make_path("$dummy/DEBIAN", $input, "$apt_root/pool/main/noble"); - write_file( + write_binary( "$dummy/DEBIAN/control", "Package: xcat-genesis-base-amd64\nVersion: 1\nArchitecture: all\n" . "Maintainer: xCAT \n" @@ -177,8 +179,8 @@ sub test_deb_consumer { "$tmp/dummy-deb.log", 'dpkg-deb', '--root-owner-group', '--build', $dummy, "$input/xcat-genesis-base-amd64_1_all.deb", ); - write_file("$input/xcat-genesis-openembedded-stale.deb", 'stale'); - write_file("$apt_root/pool/main/noble/xcat-genesis-openembedded-old.deb", 'stale'); + write_binary("$input/xcat-genesis-openembedded-stale.deb", 'stale'); + write_binary("$apt_root/pool/main/noble/xcat-genesis-openembedded-old.deb", 'stale'); local $ENV{SOURCE_DATE_EPOCH} = $epoch; my $log = "$tmp/deb-consumer.log"; @@ -196,23 +198,23 @@ sub test_deb_consumer { my $ppc64el = "$apt_root/dists/noble/main/binary-ppc64el/Packages"; is($status, 0, 'APT repository accepts a verified Genesis release'); - is(file_sha($pool_package), file_sha("$release_root/deb/$package"), + is(digest_file($pool_package), digest_file("$release_root/deb/$package"), 'pooled DEB matches the release'); ok(!-e "$apt_root/pool/main/noble/xcat-genesis-openembedded-old.deb", 'stale pooled DEB is removed'); - like(read_file($amd64), qr/^Package: xcat-genesis-openembedded-x86-64$/m, + like(read_binary($amd64), qr/^Package: xcat-genesis-openembedded-x86-64$/m, 'all-architecture DEB is indexed for amd64'); - like(read_file($ppc64el), qr/^Package: xcat-genesis-openembedded-x86-64$/m, + like(read_binary($ppc64el), qr/^Package: xcat-genesis-openembedded-x86-64$/m, 'all-architecture DEB is indexed for ppc64el'); ok(-f "$apt_root/dists/noble/Release", 'APT Release metadata is generated'); - like(read_file($amd64), qr/^Package: xcat-genesis-base-amd64$/m, + like(read_binary($amd64), qr/^Package: xcat-genesis-base-amd64$/m, 'legacy Genesis DEB remains available'); ok(!-e "$apt_root/pool/main/noble/xcat-genesis-openembedded-stale.deb", 'stale OpenEmbedded DEB is not collected'); my $collision = "$tmp/apt-collision"; make_path("$collision/ubuntu24.04"); - write_file("$collision/ubuntu24.04/$package", 'different'); + write_binary("$collision/ubuntu24.04/$package", 'different'); my $collision_log = "$tmp/deb-collision.log"; my $collision_status = run_capture( $collision_log, @@ -224,19 +226,19 @@ sub test_deb_consumer { 'ubuntu24.04', ); is($collision_status, 0, 'verified release replaces a stale source package'); - is(file_sha("$collision/pool/main/noble/$package"), - file_sha("$release_root/deb/$package"), + is(digest_file("$collision/pool/main/noble/$package"), + digest_file("$release_root/deb/$package"), 'pooled package still matches the verified release'); } sub test_partial_rpm_release { my $release_root = make_package_release("$tmp/rpm-partial", 'rpm', 'x86_64'); my $output = "$tmp/partial-output"; - my $target = 'test+epel-10-' . capture('uname', '-m'); - my $deployed = "$output/xcat-dep/rh10/" . capture('uname', '-m'); + my $target = 'test+epel-10-' . capture_command('uname', '-m'); + my $deployed = "$output/xcat-dep/rh10/" . capture_command('uname', '-m'); my $existing = "$deployed/xCAT-genesis-base-existing.noarch.rpm"; make_path($deployed); - write_file($existing, 'existing release'); + write_binary($existing, 'existing release'); my $log = "$tmp/rpm-partial.log"; my $status = run_capture( @@ -262,7 +264,7 @@ sub test_partial_deb_release { my $pool = "$apt_root/pool/main/noble"; my $existing = "$pool/xcat-genesis-base-existing.deb"; make_path($pool); - write_file($existing, 'existing release'); + write_binary($existing, 'existing release'); my $log = "$tmp/deb-partial.log"; my $status = run_capture( @@ -327,12 +329,3 @@ sub make_package_release { write_checksums($release_root); return $release_root; } - -sub capture { - my (@command) = @_; - open(my $fh, '-|', @command) or die $!; - my $output = <$fh>; - close($fh) or die $!; - chomp($output); - return $output; -} diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 2c3ba85..46dcab3 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -9,8 +9,13 @@ use FindBin; use Test::More; use lib "$FindBin::Bin/../lib"; -use lib "$FindBin::Bin/../genesis-openembedded/lib"; use lib "$FindBin::Bin/lib"; +use XCAT::BuildUtils qw( + command_exists + digest_file + read_binary + write_binary +); use XCAT::GenesisRelease qw( architectures deb_package_name @@ -23,15 +28,11 @@ use XCAT::GenesisRelease qw( verify_release_file ); use XCAT::GenesisReleaseTest qw( - command_exists copy_tree dies_like - file_sha make_export - read_file run_capture write_checksums - write_file write_release_manifest ); @@ -75,12 +76,12 @@ dies_like(sub { validate_export($missing, 'x86_64') }, qr/missing image\.vex\.js 'missing release evidence fails'); my $corrupt = make_export("$tmp/corrupt", 'x86_64'); -write_file("$corrupt/kernel", 'changed'); +write_binary("$corrupt/kernel", 'changed'); dies_like(sub { validate_export($corrupt, 'x86_64') }, qr/Checksum mismatch for kernel/, 'corrupt payload fails'); my $unexpected = make_export("$tmp/unexpected", 'x86_64'); -write_file("$unexpected/extra", 'not part of the export'); +write_binary("$unexpected/extra", 'not part of the export'); write_checksums($unexpected); dies_like(sub { validate_export($unexpected, 'x86_64') }, qr/Unexpected Genesis export file/, 'unlisted export files fail'); @@ -100,9 +101,9 @@ make_path("$release_dir/rpm", "$release_dir/srpm", "$release_dir/deb"); for my $architecture (qw(x86_64 ppc64le)) { my $rpm = rpm_package_name($architecture); my $deb = deb_package_name($architecture); - write_file("$release_dir/rpm/$rpm-$version-$release.noarch.rpm", "rpm $architecture"); - write_file("$release_dir/srpm/$rpm-$version-$release.src.rpm", "srpm $architecture"); - write_file("$release_dir/deb/${deb}_${version}-${release}_all.deb", "deb $architecture"); + write_binary("$release_dir/rpm/$rpm-$version-$release.noarch.rpm", "rpm $architecture"); + write_binary("$release_dir/srpm/$rpm-$version-$release.src.rpm", "srpm $architecture"); + write_binary("$release_dir/deb/${deb}_${version}-${release}_all.deb", "deb $architecture"); } write_release_manifest( $release_dir, $version, $release, $revision, $epoch, @@ -115,7 +116,7 @@ my $qualified_release = "$tmp/qualified-release"; my $qualified_release_name = '1+deb~1'; my $qualified_deb = deb_package_name('x86_64'); make_path("$qualified_release/deb"); -write_file( +write_binary( "$qualified_release/deb/${qualified_deb}_${version}-${qualified_release_name}_all.deb", 'deb x86_64', ); @@ -131,7 +132,7 @@ my $verified_copy = "$tmp/verified-copy.rpm"; copy("$release_dir/$verified_relative", $verified_copy) or die $!; ok(verify_release_file($verified_checksums, $verified_relative, $verified_copy), 'collected package matches the verified release'); -write_file($verified_copy, 'changed after verification'); +write_binary($verified_copy, 'changed after verification'); dies_like( sub { verify_release_file($verified_checksums, $verified_relative, $verified_copy) }, qr/Collected release file checksum mismatch/, @@ -148,9 +149,9 @@ make_path("$complete_release/rpm", "$complete_release/srpm", "$complete_release/ for my $architecture (architectures()) { my $rpm = rpm_package_name($architecture); my $deb = deb_package_name($architecture); - write_file("$complete_release/rpm/$rpm-$version-$release.noarch.rpm", "rpm $architecture"); - write_file("$complete_release/srpm/$rpm-$version-$release.src.rpm", "srpm $architecture"); - write_file("$complete_release/deb/${deb}_${version}-${release}_all.deb", "deb $architecture"); + write_binary("$complete_release/rpm/$rpm-$version-$release.noarch.rpm", "rpm $architecture"); + write_binary("$complete_release/srpm/$rpm-$version-$release.src.rpm", "srpm $architecture"); + write_binary("$complete_release/deb/${deb}_${version}-${release}_all.deb", "deb $architecture"); } write_release_manifest( $complete_release, $version, $release, $revision, $epoch, @@ -163,7 +164,7 @@ my $deb_only_release = "$tmp/deb-only-release"; make_path("$deb_only_release/deb"); for my $architecture (architectures()) { my $deb = deb_package_name($architecture); - write_file("$deb_only_release/deb/${deb}_${version}-${release}_all.deb", "deb $architecture"); + write_binary("$deb_only_release/deb/${deb}_${version}-${release}_all.deb", "deb $architecture"); } write_release_manifest( $deb_only_release, $version, $release, $revision, $epoch, @@ -173,12 +174,12 @@ write_checksums($deb_only_release); my $verify_all_log = "$tmp/verify-all.log"; isnt(run_capture($verify_all_log, $verifier, $deb_only_release), 0, 'all-format verification rejects a single-format release'); -like(read_file($verify_all_log), qr/Release does not contain rpm packages/, +like(read_binary($verify_all_log), qr/Release does not contain rpm packages/, 'all-format failure names the missing format'); my $bad_release = "$tmp/bad-release"; copy_tree($release_dir, $bad_release); -write_file("$bad_release/rpm/stale.rpm", 'stale'); +write_binary("$bad_release/rpm/stale.rpm", 'stale'); write_checksums($bad_release); dies_like(sub { validate_release($bad_release) }, qr/Unexpected Genesis release artifact/, 'stale package fails'); @@ -194,9 +195,9 @@ SKIP: { skip 'git is not installed', 2 unless command_exists('git'); my $source = "$tmp/dirty-xcat-core"; make_path("$source/xCAT-genesis-builder/oe"); - write_file("$source/Version", "$version\n"); - write_file("$source/xCAT-genesis-builder/oe/build", "#!/bin/sh\nexit 99\n"); - write_file("$source/xCAT-genesis-builder/oe/export", "#!/bin/sh\nexit 99\n"); + write_binary("$source/Version", "$version\n"); + write_binary("$source/xCAT-genesis-builder/oe/build", "#!/bin/sh\nexit 99\n"); + write_binary("$source/xCAT-genesis-builder/oe/export", "#!/bin/sh\nexit 99\n"); for my $command ( [ 'git', '-C', $source, 'init', '-q' ], [ 'git', '-C', $source, 'add', '.' ], @@ -206,7 +207,7 @@ SKIP: { die "Cannot prepare test repository\n" if run_capture("$tmp/git-fixture.log", @{$command}); } - write_file("$source/untracked", "not part of the commit\n"); + write_binary("$source/untracked", "not part of the commit\n"); my $log = "$tmp/dirty-source.log"; isnt( run_capture( @@ -216,7 +217,7 @@ SKIP: { 0, 'release builder rejects untracked source files', ); - like(read_file($log), qr/xcat-core checkout is not clean/, + like(read_binary($log), qr/xcat-core checkout is not clean/, 'dirty checkout failure is explicit'); } @@ -277,11 +278,11 @@ sub exercise_packager { $relative = "deb/xcat-genesis-openembedded-x86-64_${version}-${release}_all.deb"; } ok(-f "$first/$relative", "$format binary exists"); - is(file_sha("$first/$relative"), file_sha("$second/$relative"), + is(digest_file("$first/$relative"), digest_file("$second/$relative"), "$format binary is reproducible across umasks"); if ($format eq 'rpm') { ok(-f "$first/$source_relative", 'source RPM exists'); - is(file_sha("$first/$source_relative"), file_sha("$second/$source_relative"), + is(digest_file("$first/$source_relative"), digest_file("$second/$source_relative"), 'source RPM is reproducible across umasks'); } @@ -299,16 +300,16 @@ sub exercise_packager { if ($format eq 'rpm') { is(run_capture($contents_log, 'rpm', '-qpl', "$first/$relative"), 0, 'RPM payload can be listed'); - like(read_file($contents_log), + like(read_binary($contents_log), qr{/opt/xcat/share/xcat/netboot/genesis-openembedded/x86_64/kernel}, 'RPM uses the OpenEmbedded staging namespace'); is(run_capture($contents_log, 'rpm', '-qp', '--scripts', "$first/$relative"), 0, 'RPM script metadata can be read'); - is(read_file($contents_log), '', 'RPM has no package scripts'); + is(read_binary($contents_log), '', 'RPM has no package scripts'); } else { is(run_capture($contents_log, 'dpkg-deb', '-c', "$first/$relative"), 0, 'DEB payload can be listed'); - like(read_file($contents_log), + like(read_binary($contents_log), qr{/opt/xcat/share/xcat/netboot/genesis-openembedded/x86_64/kernel}, 'DEB uses the OpenEmbedded staging namespace'); my $control = "$tmp/deb-control"; @@ -363,8 +364,8 @@ sub exercise_builder_tmpdir { my $source = "$tmp/tmpdir-xcat-core"; my $oe = "$source/xCAT-genesis-builder/oe"; make_path($oe); - write_file("$source/Version", "$version\n"); - write_file( + write_binary("$source/Version", "$version\n"); + write_binary( "$oe/build", <<'BUILD', #!/bin/sh @@ -374,7 +375,7 @@ expected=$XCAT_GENESIS_WORK_DIR/build/tmp mkdir -p "$TMPDIR/deploy" BUILD ); - write_file( + write_binary( "$oe/export", <<'EXPORT', #!/bin/sh @@ -424,7 +425,7 @@ EXPORT ); } is($status, 0, 'release builder isolates the OpenEmbedded tmpdir'); - unlike(read_file($log), qr/Invalid OpenEmbedded deploy directory/, + unlike(read_binary($log), qr/Invalid OpenEmbedded deploy directory/, 'release builder finds the configured deploy directory'); ok(-d "$persistent_work/openembedded/build/tmp", 'release builder preserves the requested work directory'); diff --git a/t/lib/XCAT/GenesisReleaseTest.pm b/t/lib/XCAT/GenesisReleaseTest.pm index a4f83ee..36716ab 100644 --- a/t/lib/XCAT/GenesisReleaseTest.pm +++ b/t/lib/XCAT/GenesisReleaseTest.pm @@ -3,25 +3,23 @@ package XCAT::GenesisReleaseTest; use strict; use warnings; -use Digest::SHA (); use Exporter qw(import); use File::Basename qw(dirname); use File::Copy qw(copy); -use File::Find qw(find); use File::Path qw(make_path); -use File::Spec; use Test::More (); +use XCAT::BuildUtils qw( + digest_manifest + relative_files + write_binary +); our @EXPORT_OK = qw( - command_exists copy_tree dies_like - file_sha make_export - read_file run_capture write_checksums - write_file write_release_manifest ); @@ -39,7 +37,7 @@ sub make_export { "format=xcat-genesis\nversion=1\narchitecture=$architecture\n", ); $content{'fw_jump.elf'} = 'firmware' if $architecture eq 'riscv64'; - write_file("$directory/$_", $content{$_}) for sort keys %content; + write_binary("$directory/$_", $content{$_}) for sort keys %content; write_checksums($directory); return $directory; } @@ -47,7 +45,7 @@ sub make_export { sub write_release_manifest { my ($directory, $xcat_version, $xcat_release, $revision, $epoch, $architectures, $formats) = @_; - write_file( + write_binary( "$directory/release.manifest", "format=xcat-genesis-packages\n" . "version=1\n" @@ -63,66 +61,21 @@ sub write_release_manifest { sub write_checksums { my ($directory) = @_; unlink("$directory/SHA256SUMS") if -e "$directory/SHA256SUMS"; - my @files; - find( - { - no_chdir => 1, - wanted => sub { - return unless -f $_ && !-l $_; - my $relative = File::Spec->abs2rel($File::Find::name, $directory); - $relative =~ tr{\\}{/}; - push(@files, $relative); - }, - }, - $directory, + my @files = relative_files($directory); + write_binary( + "$directory/SHA256SUMS", + digest_manifest($directory, 'sha256', @files), ); - my $content = join('', map { file_sha("$directory/$_") . " $_\n" } sort @files); - write_file("$directory/SHA256SUMS", $content); -} - -sub file_sha { - my ($path) = @_; - open(my $fh, '<:raw', $path) or die $!; - my $digest = Digest::SHA->new(256)->addfile($fh)->hexdigest; - close($fh) or die $!; - return $digest; } sub copy_tree { my ($source, $destination) = @_; make_path($destination); - find( - { - no_chdir => 1, - wanted => sub { - return if $File::Find::name eq $source; - my $relative = File::Spec->abs2rel($File::Find::name, $source); - my $target = "$destination/$relative"; - if (-d $File::Find::name) { - make_path($target); - } elsif (-f $File::Find::name) { - make_path(dirname($target)); - copy($File::Find::name, $target) or die $!; - } - }, - }, - $source, - ); -} - -sub write_file { - my ($path, $content) = @_; - open(my $fh, '>:raw', $path) or die $!; - print {$fh} $content or die $!; - close($fh) or die $!; -} - -sub command_exists { - my ($command) = @_; - for my $directory (File::Spec->path()) { - return 1 if -x "$directory/$command"; + for my $relative (relative_files($source)) { + my $target = "$destination/$relative"; + make_path(dirname($target)); + copy("$source/$relative", $target) or die $!; } - return 0; } sub run_capture { @@ -140,15 +93,6 @@ sub run_capture { return $? >> 8; } -sub read_file { - my ($path) = @_; - open(my $fh, '<:raw', $path) or die $!; - local $/; - my $content = <$fh> // ''; - close($fh) or die $!; - return $content; -} - sub dies_like { my ($code, $pattern, $name) = @_; my $error = ''; From f91f5ff929fbd226cfec256aee6f0ba0d492339f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:27:24 -0300 Subject: [PATCH 52/82] fix(genesis): share copied package verification --- build-apt-repo.sh | 14 ++++------- genesis-openembedded/verify-release | 37 ++++++++++++++++++++++++++--- lib/XCAT/GenesisRelease.pm | 10 ++++---- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/build-apt-repo.sh b/build-apt-repo.sh index 55ef407..83fc300 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -123,8 +123,6 @@ command -v cmp >/dev/null 2>&1 \ if [[ -n "$GENESIS_RELEASE" ]]; then command -v dpkg-deb >/dev/null 2>&1 \ || die "dpkg-deb not found. Install: sudo apt-get install dpkg" - command -v sha256sum >/dev/null 2>&1 \ - || die "sha256sum not found. Install: sudo apt-get install coreutils" GENESIS_CHECKSUMS=$(mktemp "${TMPDIR:-/tmp}/xcat-genesis-checksums.XXXXXX") cp -- "$GENESIS_RELEASE/SHA256SUMS" "$GENESIS_CHECKSUMS" "$GENESIS_VERIFIER" --complete --format deb "$GENESIS_RELEASE" @@ -193,17 +191,15 @@ copy_deb() { copy_genesis_deb() { local source="$1" local directory="$2" - local name destination relative expected actual + local name destination relative name=$(basename "$source") destination="$directory/$name" relative="deb/$name" cp -- "$source" "$destination" - expected=$(awk -v path="$relative" '$2 == path { print $1 }' "$GENESIS_CHECKSUMS") - [[ -n "$expected" ]] || die "Missing verified checksum for $relative" - actual=$(sha256sum "$destination") - actual=${actual%% *} - [[ "$actual" = "$expected" ]] \ - || die "Collected Genesis package checksum mismatch: $destination" + "$GENESIS_VERIFIER" \ + --checksum-file "$GENESIS_CHECKSUMS" \ + --relative-file "$relative" \ + --copied-file "$destination" } for ver in "${SELECTED_VERS[@]}"; do diff --git a/genesis-openembedded/verify-release b/genesis-openembedded/verify-release index da7563a..d91e45d 100755 --- a/genesis-openembedded/verify-release +++ b/genesis-openembedded/verify-release @@ -10,17 +10,44 @@ use lib "$FindBin::Bin/../lib"; use XCAT::BuildUtils qw(capture_command require_command); use XCAT::GenesisRelease qw( deb_package_name + read_checksum_manifest rpm_package_name validate_complete_release validate_release + verify_release_file ); my $format = 'all'; my $complete = 0; +my ($checksum_file, $relative_file, $copied_file); GetOptions( - 'format=s' => \$format, - 'complete!' => \$complete, + 'format=s' => \$format, + 'complete!' => \$complete, + 'checksum-file=s' => \$checksum_file, + 'relative-file=s' => \$relative_file, + 'copied-file=s' => \$copied_file, ) or die usage(); + +my $copy_mode = grep { defined($_) } + ($checksum_file, $relative_file, $copied_file); +if ($copy_mode) { + die usage() + unless defined($checksum_file) + && defined($relative_file) + && defined($copied_file) + && !$complete + && $format eq 'all' + && !@ARGV; + $checksum_file = abs_path($checksum_file) + or die "Cannot resolve checksum file\n"; + $copied_file = abs_path($copied_file) + or die "Cannot resolve copied file\n"; + my $checksums = read_checksum_manifest($checksum_file); + verify_release_file($checksums, $relative_file, $copied_file); + print "Verified copied Genesis package: $copied_file\n"; + exit(0); +} + die usage() unless @ARGV == 1; die "Unsupported verification format: $format\n" unless $format eq 'all' || $format eq 'rpm' || $format eq 'deb'; @@ -103,5 +130,9 @@ sub _verify_deb { } sub usage { - return "Usage: verify-release [--format all|rpm|deb] [--complete] RELEASE_DIRECTORY\n"; + return <<'USAGE'; +Usage: verify-release [--format all|rpm|deb] [--complete] RELEASE_DIRECTORY + verify-release --checksum-file SHA256SUMS --relative-file PATH + --copied-file FILE +USAGE } diff --git a/lib/XCAT/GenesisRelease.pm b/lib/XCAT/GenesisRelease.pm index 0bcc9ae..2dbeaa3 100644 --- a/lib/XCAT/GenesisRelease.pm +++ b/lib/XCAT/GenesisRelease.pm @@ -9,6 +9,7 @@ use XCAT::BuildUtils qw(digest_file read_lines relative_files); our @EXPORT_OK = qw( architectures deb_package_name + read_checksum_manifest read_release_manifest rpm_package_name validated_release_checksums @@ -60,9 +61,8 @@ sub _read_key_values { return \%values; } -sub _read_checksums { - my ($root) = @_; - my $path = "$root/SHA256SUMS"; +sub read_checksum_manifest { + my ($path) = @_; my %checksums; for my $line (read_lines($path)) { die "Invalid checksum entry in $path: $line\n" @@ -80,7 +80,7 @@ sub _read_checksums { sub _verify_checksums { my ($root) = @_; my @files = grep { $_ ne 'SHA256SUMS' } relative_files($root); - my $checksums = _read_checksums($root); + my $checksums = read_checksum_manifest("$root/SHA256SUMS"); my %files = map { $_ => 1 } @files; for my $name (@files) { @@ -224,7 +224,7 @@ sub validate_complete_release { sub validated_release_checksums { my ($directory) = @_; validate_release($directory); - return _read_checksums($directory); + return read_checksum_manifest("$directory/SHA256SUMS"); } sub verify_release_file { From 9228a699cac832616bd9662042f80af34f2d4a63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:28:01 -0300 Subject: [PATCH 53/82] ci(genesis): check shared Perl modules --- .github/workflows/genesis-openembedded.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/genesis-openembedded.yml b/.github/workflows/genesis-openembedded.yml index af934fb..25fc684 100644 --- a/.github/workflows/genesis-openembedded.yml +++ b/.github/workflows/genesis-openembedded.yml @@ -30,7 +30,8 @@ jobs: perl -c genesis-openembedded/build perl -c genesis-openembedded/package perl -c genesis-openembedded/verify-release - perl -c genesis-openembedded/lib/XCAT/GenesisRelease.pm + perl -Ilib -c lib/XCAT/BuildUtils.pm + perl -Ilib -c lib/XCAT/GenesisRelease.pm perl -c mockbuild-all.pl rpmspec -P \ -D 'genesis_arch x86_64' \ @@ -41,7 +42,8 @@ jobs: genesis-openembedded/build \ genesis-openembedded/package \ genesis-openembedded/verify-release \ - genesis-openembedded/lib/XCAT/GenesisRelease.pm \ + lib/XCAT/BuildUtils.pm \ + lib/XCAT/GenesisRelease.pm \ mockbuild-all.pl \ t/genesis_openembedded_release.t \ t/genesis_openembedded_consumer.t \ From 4c2dfcf3e9d78642d4f3ca70a65c5c0b00e59001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:29:22 -0300 Subject: [PATCH 54/82] test(genesis): cover shared verification helpers --- .github/workflows/genesis-openembedded.yml | 2 + t/build_utils.t | 82 ++++++++++++++++++++++ t/genesis_openembedded_consumer.t | 4 +- t/genesis_openembedded_release.t | 30 ++++++++ 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 t/build_utils.t diff --git a/.github/workflows/genesis-openembedded.yml b/.github/workflows/genesis-openembedded.yml index 25fc684..25171d1 100644 --- a/.github/workflows/genesis-openembedded.yml +++ b/.github/workflows/genesis-openembedded.yml @@ -45,6 +45,7 @@ jobs: lib/XCAT/BuildUtils.pm \ lib/XCAT/GenesisRelease.pm \ mockbuild-all.pl \ + t/build_utils.t \ t/genesis_openembedded_release.t \ t/genesis_openembedded_consumer.t \ t/lib/XCAT/GenesisReleaseTest.pm @@ -53,5 +54,6 @@ jobs: - name: Run package tests run: | + prove -v t/build_utils.t prove -v -It/lib t/genesis_openembedded_release.t sudo -E prove -v -It/lib t/genesis_openembedded_consumer.t diff --git a/t/build_utils.t b/t/build_utils.t new file mode 100644 index 0000000..af8aad8 --- /dev/null +++ b/t/build_utils.t @@ -0,0 +1,82 @@ +use strict; +use warnings; + +use File::Path qw(make_path); +use File::Temp qw(tempdir); +use FindBin; +use Test::More; + +use lib "$FindBin::Bin/../lib"; +use XCAT::BuildUtils qw( + capture_command + command_exists + digest_file + digest_manifest + display_quote + hashes_equal + read_binary + read_first_line + read_lines + relative_files + require_command + run_command + shell_quote + write_binary +); + +my $tmp = tempdir(CLEANUP => 1); +write_binary("$tmp/alpha", "abc"); +make_path("$tmp/nested"); +write_binary("$tmp/nested/lines", "first\r\nsecond\n"); + +is(read_binary("$tmp/alpha"), 'abc', 'binary files round trip'); +is_deeply( + [ read_lines("$tmp/nested/lines") ], + [ 'first', 'second' ], + 'line reader accepts CRLF and LF', +); +is(read_first_line("$tmp/nested/lines"), 'first', 'first line is returned'); +is_deeply( + [ relative_files($tmp) ], + [ 'alpha', 'nested/lines' ], + 'regular files are sorted and relative', +); + +my $sha256 = 'ba7816bf8f01cfea414140de5dae2223' + . 'b00361a396177a9cb410ff61f20015ad'; +is(digest_file("$tmp/alpha", 'sha256'), $sha256, 'SHA-256 matches a known value'); +is(digest_file("$tmp/alpha", 'md5'), '900150983cd24fb0d6963f7d28e17f72', + 'MD5 matches a known value'); +is( + digest_manifest($tmp, 'sha256', 'alpha'), + "$sha256 alpha\n", + 'digest manifest uses the release format', +); + +ok(command_exists($^X), 'current Perl interpreter is executable'); +is(capture_command($^X, '-e', 'print "captured\\n"'), 'captured', + 'command output is captured without a shell'); +ok(run_command($^X, '-e', 'exit 0'), 'successful command returns true'); + +eval { run_command($^X, '-e', 'exit 7'); 1 }; +like($@, qr/Command failed \(rc=7\)/, 'command failure reports its exit status'); +eval { require_command("xcat-missing-command-$$"); 1 }; +like($@, qr/Required command not found/, 'missing command is rejected'); +eval { digest_file("$tmp/alpha", 'unknown'); 1 }; +like($@, qr/Unsupported digest algorithm/, 'unknown digest is rejected'); + +is(display_quote('plain/value'), 'plain/value', 'simple display value is unquoted'); +is(display_quote('two words'), q{'two words'}, 'display value with spaces is quoted'); +is(shell_quote("it's"), q{'it'"'"'s'}, 'shell quote escapes apostrophes'); +ok(hashes_equal({ a => 1 }, { a => 1 }), 'equal hashes match'); +ok(!hashes_equal({ a => 1 }, { a => 2 }), 'different hashes do not match'); + +my $link = "$tmp/link"; +if (symlink('alpha', $link)) { + eval { relative_files($tmp); 1 }; + like($@, qr/Symbolic links are not allowed/, 'directory walk rejects symlinks'); +} else { + fail('test filesystem supports symbolic links'); +} + +done_testing(); diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index cdefb70..f25f346 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -59,7 +59,7 @@ SKIP: { } SKIP: { - skip 'APT repository tools are not installed', 12 + skip 'APT repository tools are not installed', 13 unless $^O eq 'linux' && command_exists('bash') && command_exists('dpkg-deb') @@ -198,6 +198,8 @@ sub test_deb_consumer { my $ppc64el = "$apt_root/dists/noble/main/binary-ppc64el/Packages"; is($status, 0, 'APT repository accepts a verified Genesis release'); + like(read_binary($log), qr/Verified copied Genesis package:/, + 'APT repository uses the shared copied-package verifier'); is(digest_file($pool_package), digest_file("$release_root/deb/$package"), 'pooled DEB matches the release'); ok(!-e "$apt_root/pool/main/noble/xcat-genesis-openembedded-old.deb", diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 46dcab3..c2c0afa 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -138,6 +138,36 @@ dies_like( qr/Collected release file checksum mismatch/, 'release changes after verification are rejected', ); +copy("$release_dir/$verified_relative", $verified_copy) or die $!; +my $copied_file_log = "$tmp/copied-file.log"; +is( + run_capture( + $copied_file_log, + $verifier, + '--checksum-file', "$release_dir/SHA256SUMS", + '--relative-file', $verified_relative, + '--copied-file', $verified_copy, + ), + 0, + 'verifier accepts a copied release file', +); +write_binary($verified_copy, 'changed after verification'); +isnt( + run_capture( + $copied_file_log, + $verifier, + '--checksum-file', "$release_dir/SHA256SUMS", + '--relative-file', $verified_relative, + '--copied-file', $verified_copy, + ), + 0, + 'verifier rejects a changed copied file', +); +like( + read_binary($copied_file_log), + qr/Collected release file checksum mismatch/, + 'copied-file failure names the checksum mismatch', +); dies_like( sub { validate_complete_release($release_dir) }, qr/Genesis release is missing supported architectures/, From 542ec87d8fbbeac1ed4f283d0f6cbcb2743ebc3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:33:02 -0300 Subject: [PATCH 55/82] fix(genesis): return the verified checksum snapshot --- lib/XCAT/GenesisRelease.pm | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/XCAT/GenesisRelease.pm b/lib/XCAT/GenesisRelease.pm index 2dbeaa3..b07368f 100644 --- a/lib/XCAT/GenesisRelease.pm +++ b/lib/XCAT/GenesisRelease.pm @@ -152,7 +152,7 @@ sub read_release_manifest { return $values; } -sub validate_release { +sub _validate_release { my ($directory) = @_; die "Invalid Genesis package release: $directory\n" unless -d $directory && !-l $directory; @@ -187,7 +187,7 @@ sub validate_release { } die "Release manifest has no package formats\n" unless @formats; - _verify_checksums($directory); + my $checksums = _verify_checksums($directory); my %expected = ( 'release.manifest' => 1, @@ -209,6 +209,12 @@ sub validate_release { } die "Genesis release is missing: " . join(', ', sort keys %expected) . "\n" if %expected; + return ($manifest, $checksums); +} + +sub validate_release { + my ($directory) = @_; + my ($manifest) = _validate_release($directory); return $manifest; } @@ -223,8 +229,8 @@ sub validate_complete_release { sub validated_release_checksums { my ($directory) = @_; - validate_release($directory); - return read_checksum_manifest("$directory/SHA256SUMS"); + my (undef, $checksums) = _validate_release($directory); + return $checksums; } sub verify_release_file { From fe55231e732e16e82b3b2b431a573af380c782b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:33:29 -0300 Subject: [PATCH 56/82] test(genesis): pin the verified checksum read --- t/genesis_openembedded_release.t | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index c2c0afa..47b44d6 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -126,7 +126,18 @@ write_release_manifest( ); write_checksums($qualified_release); ok(validate_release($qualified_release), 'package filenames accept valid release qualifiers'); -my $verified_checksums = validated_release_checksums($release_dir); +my $checksum_reads = 0; +my $verified_checksums; +{ + no warnings 'redefine'; + my $reader = \&XCAT::GenesisRelease::read_checksum_manifest; + local *XCAT::GenesisRelease::read_checksum_manifest = sub { + $checksum_reads++; + return $reader->(@_); + }; + $verified_checksums = validated_release_checksums($release_dir); +} +is($checksum_reads, 1, 'validated checksums use the verified manifest read'); my $verified_relative = "rpm/xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; my $verified_copy = "$tmp/verified-copy.rpm"; copy("$release_dir/$verified_relative", $verified_copy) or die $!; From 994ba1f9e87f13de7831635d762270364667f817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:36:35 -0300 Subject: [PATCH 57/82] docs(genesis): list shared Perl modules --- BUILD.md | 4 +++- genesis-openembedded/README.md | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/BUILD.md b/BUILD.md index eac380b..fa6c167 100644 --- a/BUILD.md +++ b/BUILD.md @@ -97,7 +97,9 @@ Use these flags to skip specific operations: Install baseline tooling: ```bash -dnf -y install perl perl-Parallel-ForkManager mock createrepo tar rpm-build rpmdevtools dnf-plugins-core wget git +dnf -y install perl perl-File-Slurper perl-IPC-Cmd \ + perl-Parallel-ForkManager mock createrepo tar rpm-build rpmdevtools \ + dnf-plugins-core wget git ``` If you will build xCAT packages (that is, you will **not** use `--skip-xcat`), install xCAT build dependencies: diff --git a/genesis-openembedded/README.md b/genesis-openembedded/README.md index 6d4a21d..0c0efc6 100644 --- a/genesis-openembedded/README.md +++ b/genesis-openembedded/README.md @@ -7,6 +7,10 @@ Run the release builder on a host that can build the OpenEmbedded layer. The xcat-core checkout must be clean. `--xcat-ref` checks that the checkout points to the intended commit; it does not change the checkout for you. +The packaging scripts use `File::Slurper` and `IPC::Cmd`. Install +`libfile-slurper-perl` on Ubuntu. On EL, install `perl-File-Slurper` and +`perl-IPC-Cmd` from EPEL and AppStream. + ```bash ./genesis-openembedded/build \ --xcat-source /path/to/xcat-core \ @@ -74,6 +78,7 @@ belong in release storage, not in Git. Run the package tests on a Linux builder with RPM, DEB, and repository tools: ```bash +prove t/build_utils.t prove -It/lib t/genesis_openembedded_release.t sudo -E prove -It/lib t/genesis_openembedded_consumer.t ``` From d81a4ae248986c21d375c03385e83c07b765235f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 23:44:01 -0300 Subject: [PATCH 58/82] fix(genesis): harden package publication --- genesis-openembedded/package | 9 ++++----- .../rpm/xCAT-genesis-openembedded.spec | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/genesis-openembedded/package b/genesis-openembedded/package index 683ed25..e98b4ab 100755 --- a/genesis-openembedded/package +++ b/genesis-openembedded/package @@ -103,16 +103,15 @@ if ($format eq 'all' || $format eq 'deb') { _build_deb($work, $staging, $source_root, $deb_name); } +chmod(0755, $staging) or die "Cannot set output mode: $!\n"; rename($staging, $output_dir) or die "Cannot publish $output_dir: $!\n"; print "Packaged Genesis $architecture in $output_dir\n"; sub _build_rpm { my ($work, $staging, $source_root, $source_name, $package_name) = @_; _require_gnu_tar(); - my $candidate = "/var/tmp/xcat-genesis-rpmbuild-$revision-$architecture-$version-$release"; - die "RPM work directory already exists: $candidate\n" if -e $candidate || -l $candidate; - mkdir($candidate, 0700) or die "Cannot create RPM work directory $candidate: $!\n"; - $rpm_top = $candidate; + require_command('gzip'); + $rpm_top = tempdir('xcat-genesis-rpmbuild.XXXXXX', DIR => '/var/tmp', CLEANUP => 0); make_path(map { "$rpm_top/$_" } qw(BUILD BUILDROOT RPMS SOURCES SPECS SRPMS)); make_path("$staging/rpm", "$staging/srpm"); @@ -126,7 +125,7 @@ sub _build_rpm { copy($spec, "$rpm_top/SPECS/xCAT-genesis-openembedded.spec") or die "Cannot stage RPM spec: $!\n"; run_command( - 'rpmbuild', '-ba', + 'rpmbuild', '-ba', '--nodeps', '--define', "_topdir $rpm_top", '--define', "genesis_arch $architecture", '--define', "version $version", diff --git a/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec b/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec index 689b25a..bcb6266 100644 --- a/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec +++ b/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec @@ -9,29 +9,35 @@ URL: https://xcat.org/ Source0: %{name}-%{version}.tar.gz BuildArch: noarch AutoReqProv: no +BuildRequires: gzip +BuildRequires: tar %description The OpenEmbedded Genesis image used by xCAT for discovery, inventory, and service actions before a node boots its installed operating system. %prep -%setup -q +rm -rf %{name}-%{version} +tar -xzf ../SOURCES/%{name}-%{version}.tar.gz %build %install -rm -rf %{buildroot} -install -d -m 0755 %{buildroot}/opt/xcat/share/xcat/netboot/genesis-openembedded/%{genesis_arch} -install -m 0644 image/* %{buildroot}/opt/xcat/share/xcat/netboot/genesis-openembedded/%{genesis_arch}/ +cd %{name}-%{version} +rm -rf "$RPM_BUILD_ROOT" +install -d -m 0755 "$RPM_BUILD_ROOT/opt/xcat/share/xcat/netboot/genesis-openembedded/%{genesis_arch}" +install -m 0644 image/* "$RPM_BUILD_ROOT/opt/xcat/share/xcat/netboot/genesis-openembedded/%{genesis_arch}/" # Keep this path stable across RPM build hosts. %%{_docdir} differs on SUSE. -install -d -m 0755 %{buildroot}/usr/share/doc/%{name} -install -m 0644 xcat-core-revision %{buildroot}/usr/share/doc/%{name}/ +install -d -m 0755 "$RPM_BUILD_ROOT/usr/share/doc/%{name}" +install -m 0644 xcat-core-revision "$RPM_BUILD_ROOT/usr/share/doc/%{name}/" %files +%defattr(-,root,root,-) %dir /opt/xcat %dir /opt/xcat/share %dir /opt/xcat/share/xcat %dir /opt/xcat/share/xcat/netboot %dir /opt/xcat/share/xcat/netboot/genesis-openembedded /opt/xcat/share/xcat/netboot/genesis-openembedded/%{genesis_arch} +%dir /usr/share/doc/%{name} %doc /usr/share/doc/%{name}/xcat-core-revision From 690863fea9c6c0dc4b6296ef0b3441fe17b1775a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 23:44:06 -0300 Subject: [PATCH 59/82] test(genesis): cover package publication safeguards --- t/genesis_openembedded_release.t | 40 +++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index 47b44d6..dadffdb 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -3,7 +3,7 @@ use warnings; use Cwd qw(abs_path); use File::Copy qw(copy); -use File::Path qw(make_path); +use File::Path qw(make_path remove_tree); use File::Temp qw(tempdir); use FindBin; use Test::More; @@ -263,7 +263,7 @@ SKIP: { } SKIP: { - skip 'rpmbuild and rpm are not installed', 12 + skip 'rpmbuild and rpm are not installed', 18 unless command_exists('rpmbuild') && command_exists('rpm'); exercise_packager('rpm'); } @@ -276,7 +276,7 @@ SKIP: { } SKIP: { - skip 'dpkg-deb is not installed', 10 unless command_exists('dpkg-deb'); + skip 'dpkg-deb is not installed', 12 unless command_exists('dpkg-deb'); exercise_packager('deb'); } @@ -302,6 +302,14 @@ sub exercise_packager { '--source-date-epoch', $epoch, '--format', $format, ); + my ($legacy_rpm_top, $legacy_rpm_work); + if ($format eq 'rpm') { + $legacy_rpm_top = + "/var/tmp/xcat-genesis-rpmbuild-$revision-x86_64-$version-$release"; + $legacy_rpm_work = "$legacy_rpm_top/BUILD/stale"; + make_path("$legacy_rpm_top/BUILD"); + write_binary($legacy_rpm_work, 'stale work'); + } my $original_umask = umask(0022); is(system(@command, '--output-dir', $first), 0, @@ -310,6 +318,14 @@ sub exercise_packager { is(system(@command, '--output-dir', $second), 0, "$format package rebuilds with umask 0002"); umask($original_umask); + is(sprintf('%04o', (stat($first))[2] & 0x0fff), '0755', + "$format output is readable and searchable"); + is(sprintf('%04o', (stat($second))[2] & 0x0fff), '0755', + "$format rebuilt output is readable and searchable"); + if (defined($legacy_rpm_work)) { + ok(-e $legacy_rpm_work, 'RPM package ignores the legacy fixed work path'); + remove_tree($legacy_rpm_top); + } my ($relative, $source_relative); if ($format eq 'rpm') { @@ -344,6 +360,24 @@ sub exercise_packager { like(read_binary($contents_log), qr{/opt/xcat/share/xcat/netboot/genesis-openembedded/x86_64/kernel}, 'RPM uses the OpenEmbedded staging namespace'); + like(read_binary($contents_log), + qr{^/usr/share/doc/xCAT-genesis-openembedded-x86_64/?$}m, + 'RPM owns its documentation directory'); + my $ownership_log = "$tmp/rpm-ownership.log"; + is( + run_capture( + $ownership_log, 'rpm', '-qp', + '--qf', '[%{FILEUSERNAME}:%{FILEGROUPNAME}\n]', + "$first/$relative", + ), + 0, + 'RPM file ownership can be read', + ); + is_deeply( + [ grep { $_ ne 'root:root' } split(/\n/, read_binary($ownership_log)) ], + [], + 'RPM owns every payload path as root', + ); is(run_capture($contents_log, 'rpm', '-qp', '--scripts', "$first/$relative"), 0, 'RPM script metadata can be read'); is(read_binary($contents_log), '', 'RPM has no package scripts'); From d5cf7abf2e819d90eb37ff8b31ed5035ad2905ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 23:44:11 -0300 Subject: [PATCH 60/82] fix(build): preserve legacy repository behavior --- mockbuild-all.pl | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 8711969..b7b6634 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -9,6 +9,7 @@ use File::Copy qw(copy); use File::Find qw(find); use File::Glob qw(bsd_glob); use File::Path qw(make_path); +use File::Temp qw(tempfile); use FindBin; use Getopt::Long qw(GetOptions); use Parallel::ForkManager; @@ -18,7 +19,7 @@ use XCAT::BuildUtils qw( capture_command hashes_equal print_step - read_first_line + read_lines require_command run_command shell_quote @@ -105,7 +106,8 @@ $repo_root = abs_path($repo_root); my $SOURCE_DATE_EPOCH; $SOURCE_DATE_EPOCH = $build_timestamp if defined $build_timestamp; if (!$SOURCE_DATE_EPOCH && -f "$repo_root/Gitepoch") { - $SOURCE_DATE_EPOCH = read_first_line("$repo_root/Gitepoch"); + my @gitepoch = read_lines("$repo_root/Gitepoch"); + $SOURCE_DATE_EPOCH = $gitepoch[0] // ''; } unless ($SOURCE_DATE_EPOCH && $SOURCE_DATE_EPOCH =~ /^\d+$/) { $SOURCE_DATE_EPOCH = `git -C \Q$repo_root\E log -1 --format=%ct HEAD 2>/dev/null`; @@ -297,7 +299,6 @@ print "skip_xcat_dep: $skip_xcat_dep\n"; print "skip_perl: $skip_perl\n"; print "skip_xcat: $skip_xcat\n"; print "skip_genesis: $skip_genesis\n"; -print "genesis_release: " . ($genesis_release || '(none)') . "\n"; print "skip_install: $skip_install\n"; print "skip_createrepo: $skip_createrepo\n"; print "skip_tarball: $skip_tarball\n"; @@ -620,13 +621,7 @@ sub deploy_target { for my $rpm (bsd_glob("$src/*.rpm")) { next if $rpm =~ /\.src\.rpm$/; my $destination = "$dest/" . basename($rpm); - my $temporary = "$destination.$$.new"; - die "Deployment staging file already exists: $temporary\n" - if -e $temporary || -l $temporary; - copy($rpm, $temporary) - or die "Failed to stage $rpm -> $temporary: $!\n"; - rename($temporary, $destination) - or die "Failed to publish $temporary -> $destination: $!\n"; + publish_file($rpm, $destination); } if ($genesis_release) { my @keep = map { basename($_) } genesis_release_files('rpm'); @@ -640,6 +635,33 @@ sub deploy_target { print "Deployed rh$rel/$arch: $n rpms\n"; } +sub publish_file { + my ($source, $destination) = @_; + my ($temporary_fh, $temporary) = tempfile( + '.xcat-deploy.XXXXXX', + DIR => dirname($destination), + UNLINK => 0, + ); + close($temporary_fh) or die "Cannot close deployment staging file: $!\n"; + + my $mode = (stat($source))[2]; + die "Cannot read mode from $source: $!\n" unless defined($mode); + my $published = eval { + copy($source, $temporary) + or die "Failed to stage $source -> $temporary: $!\n"; + chmod($mode & 0x0fff, $temporary) + or die "Failed to set mode on $temporary: $!\n"; + rename($temporary, $destination) + or die "Failed to publish $temporary -> $destination: $!\n"; + 1; + }; + return if $published; + + my $error = $@ || "Failed to publish $source\n"; + unlink($temporary) if -e $temporary || -l $temporary; + die $error; +} + # createrepo_c command with upstream-matching, deterministic metadata. The tool's # defaults emit primary/filelists/other as *.xml.zst plus *.sqlite.bz2 (--database), # exactly the upstream shape; --set-timestamp-to-revision pins repomd to SOURCE_DATE_EPOCH. From 4d46b8f7cab79b24695ba0c0f05a7e75cdf33452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 23:44:11 -0300 Subject: [PATCH 61/82] test(build): cover repository fallback safeguards --- t/genesis_openembedded_consumer.t | 160 +++++++++++++++++++++++------- t/lib/XCAT/GenesisReleaseTest.pm | 18 ++++ 2 files changed, 140 insertions(+), 38 deletions(-) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index f25f346..55e9db2 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -26,6 +26,7 @@ use XCAT::GenesisRelease qw( use XCAT::GenesisReleaseTest qw( make_export run_capture + write_forkmanager_stub write_checksums write_release_manifest ); @@ -48,24 +49,26 @@ if ($ENV{XCAT_GENESIS_CI}) { } SKIP: { - skip 'RPM repository tools require a root Linux builder', 13 + skip 'RPM repository tools require a root Linux builder', 19 unless $^O eq 'linux' && $> == 0 && command_exists('rpmbuild') && command_exists('rpm') && command_exists('createrepo_c'); test_rpm_consumer(); + test_legacy_rpm_consumer(); test_partial_rpm_release(); } SKIP: { - skip 'APT repository tools are not installed', 13 + skip 'APT repository tools are not installed', 17 unless $^O eq 'linux' && command_exists('bash') && command_exists('dpkg-deb') && command_exists('apt-ftparchive') && command_exists('gpg'); test_deb_consumer(); + test_legacy_deb_consumer(); test_partial_deb_release(); } @@ -88,15 +91,8 @@ sub test_rpm_consumer { my $dependencies = "$tmp/rpm-dependencies"; my $scratch_repo_root = "$tmp/rpm-repo-root"; - make_path($dependencies, $scratch_repo_root); - for my $name (qw( - ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi - perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB - xCAT-genesis-base-x86_64 - )) { - copy("$release_root/rpm/$package", "$dependencies/$name-1.noarch.rpm") - or die $!; - } + make_rpm_dependencies($dependencies, "$release_root/rpm/$package"); + make_path($scratch_repo_root); write_binary( "$dependencies/xCAT-genesis-openembedded-x86_64-$version-old.noarch.rpm", 'stale OpenEmbedded RPM', @@ -106,20 +102,9 @@ sub test_rpm_consumer { 'stale OpenEmbedded SRPM', ); - my $stub = "$tmp/perl-stub/Parallel/ForkManager.pm"; - make_path("$tmp/perl-stub/Parallel"); - write_binary( - $stub, - "package Parallel::ForkManager;\n" - . "sub new { bless {}, shift }\n" - . "sub run_on_finish { \$_[0]->{callback} = \$_[1] }\n" - . "sub start { 0 }\n" - . "sub finish { my (\$self, \$exit) = \@_; " - . "\$self->{callback}->(\$\$, \$exit, undef, 0, 0) if \$self->{callback}; 0 }\n" - . "sub wait_all_children { 0 }\n1;\n", - ); - - my @perl_lib = ("$tmp/perl-stub"); + my @perl_lib; + push(@perl_lib, write_forkmanager_stub("$tmp/perl-stub")) + unless eval { require Parallel::ForkManager; 1 }; push(@perl_lib, $ENV{PERL5LIB}) if defined($ENV{PERL5LIB}) && $ENV{PERL5LIB} ne ''; local $ENV{PERL5LIB} = join(':', @perl_lib); @@ -141,6 +126,15 @@ sub test_rpm_consumer { is($status, 0, 'RPM repository accepts a verified Genesis release'); is(digest_file("$deploy_repo/$package"), digest_file("$release_root/rpm/$package"), 'deployed RPM matches the release'); + is( + sprintf('%04o', (stat("$deploy_repo/$package"))[2] & 0x0fff), + sprintf('%04o', (stat("$release_root/rpm/$package"))[2] & 0x0fff), + 'deployed RPM keeps the release file mode', + ); + opendir(my $deploy_dh, $deploy_repo) or die $!; + my @staging_files = grep { /^\.xcat-deploy\./ } readdir($deploy_dh); + closedir($deploy_dh) or die $!; + is_deeply(\@staging_files, [], 'RPM deployment leaves no staging files'); ok(!-e "$run_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale run RPM is removed'); ok(!-e "$source_repo/xCAT-genesis-openembedded-stale.src.rpm", @@ -166,19 +160,8 @@ sub test_deb_consumer { my $package = "xcat-genesis-openembedded-x86-64_${version}-${release}_all.deb"; my $apt_root = "$tmp/apt"; my $input = "$apt_root/ubuntu24.04"; - my $dummy = "$tmp/dummy-deb"; - make_path("$dummy/DEBIAN", $input, "$apt_root/pool/main/noble"); - write_binary( - "$dummy/DEBIAN/control", - "Package: xcat-genesis-base-amd64\nVersion: 1\nArchitecture: all\n" - . "Maintainer: xCAT \n" - . "Description: repository test package\n", - ); - die "Cannot build test DEB\n" - if run_capture( - "$tmp/dummy-deb.log", 'dpkg-deb', '--root-owner-group', '--build', - $dummy, "$input/xcat-genesis-base-amd64_1_all.deb", - ); + make_path($input, "$apt_root/pool/main/noble"); + make_legacy_deb("$tmp/dummy-deb", "$input/xcat-genesis-base-amd64_1_all.deb"); write_binary("$input/xcat-genesis-openembedded-stale.deb", 'stale'); write_binary("$apt_root/pool/main/noble/xcat-genesis-openembedded-old.deb", 'stale'); @@ -233,6 +216,76 @@ sub test_deb_consumer { 'pooled package still matches the verified release'); } +sub test_legacy_rpm_consumer { + my $release_root = make_package_release("$tmp/rpm-legacy", 'rpm', 'x86_64'); + my $package = "xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; + my $dependencies = "$tmp/rpm-legacy-dependencies"; + my $output = "$tmp/rpm-legacy-output"; + my $scratch_repo_root = "$tmp/rpm-legacy-repo-root"; + my $target = 'test+epel-10-' . capture_command('uname', '-m'); + my $deployed = "$output/xcat-dep/rh10/" . capture_command('uname', '-m'); + make_rpm_dependencies($dependencies, "$release_root/rpm/$package"); + make_path($scratch_repo_root); + write_binary("$scratch_repo_root/Gitepoch", ''); + + my @perl_lib; + push(@perl_lib, write_forkmanager_stub("$tmp/perl-legacy-stub")) + unless eval { require Parallel::ForkManager; 1 }; + push(@perl_lib, $ENV{PERL5LIB}) + if defined($ENV{PERL5LIB}) && $ENV{PERL5LIB} ne ''; + local $ENV{PERL5LIB} = join(':', @perl_lib); + + my $log = "$tmp/rpm-legacy-consumer.log"; + my $status = run_capture( + $log, + $^X, $rpm_consumer, + '--repo-root', $scratch_repo_root, + '--output', $output, + '--target', $target, + '--run-id', 'legacy', + '--skip-build', '--skip-xcat', '--skip-xcat-dep', '--skip-perl', + '--skip-createrepo', '--skip-tarball', + '--collect-dir', $dependencies, + ); + + is($status, 0, 'RPM legacy repository accepts an empty Gitepoch'); + ok(-f "$deployed/xCAT-genesis-base-x86_64-1.noarch.rpm", + 'RPM legacy path still deploys the existing Genesis package'); + is_deeply( + [ glob("$deployed/xCAT-genesis-openembedded-*.rpm") ], + [], + 'RPM legacy path does not add OpenEmbedded packages', + ); +} + +sub test_legacy_deb_consumer { + my $apt_root = "$tmp/apt-legacy"; + my $input = "$apt_root/ubuntu24.04"; + make_path($input); + make_legacy_deb( + "$tmp/legacy-dummy-deb", + "$input/xcat-genesis-base-amd64_1_all.deb", + ); + + local $ENV{SOURCE_DATE_EPOCH} = $epoch; + my $log = "$tmp/deb-legacy-consumer.log"; + my $status = run_capture( + $log, + 'bash', $deb_consumer, + '--repo-root', $repo_root, + '--apt-dir', $apt_root, + '--skip-sign', + 'ubuntu24.04', + ); + my $packages = "$apt_root/dists/noble/main/binary-amd64/Packages"; + + is($status, 0, 'APT repository keeps working without a Genesis release'); + like(read_binary($packages), qr/^Package: xcat-genesis-base-amd64$/m, + 'APT legacy path still indexes the existing Genesis package'); + unlike(read_binary($packages), qr/^Package: xcat-genesis-openembedded-/m, + 'APT legacy path does not add OpenEmbedded packages'); +} + sub test_partial_rpm_release { my $release_root = make_package_release("$tmp/rpm-partial", 'rpm', 'x86_64'); my $output = "$tmp/partial-output"; @@ -257,6 +310,8 @@ sub test_partial_rpm_release { ); isnt($status, 0, 'RPM repository rejects a partial Genesis release'); + like(read_binary($log), qr/Genesis release is missing supported architectures/, + 'RPM partial-release failure names the missing architectures'); ok(-f $existing, 'partial release does not remove the deployed package'); } @@ -280,6 +335,8 @@ sub test_partial_deb_release { ); isnt($status, 0, 'APT repository rejects a partial Genesis release'); + like(read_binary($log), qr/Genesis release is missing supported architectures/, + 'APT partial-release failure names the missing architectures'); ok(-f $existing, 'partial DEB release does not remove the deployed package'); } @@ -331,3 +388,30 @@ sub make_package_release { write_checksums($release_root); return $release_root; } + +sub make_rpm_dependencies { + my ($directory, $package) = @_; + make_path($directory); + for my $name (qw( + ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi + perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB + xCAT-genesis-base-x86_64 + )) { + copy($package, "$directory/$name-1.noarch.rpm") or die $!; + } +} + +sub make_legacy_deb { + my ($root, $output) = @_; + make_path("$root/DEBIAN"); + write_binary( + "$root/DEBIAN/control", + "Package: xcat-genesis-base-amd64\nVersion: 1\nArchitecture: all\n" + . "Maintainer: xCAT \n" + . "Description: repository test package\n", + ); + die "Cannot build legacy test DEB\n" + if run_capture( + "$root.log", 'dpkg-deb', '--root-owner-group', '--build', $root, $output, + ); +} diff --git a/t/lib/XCAT/GenesisReleaseTest.pm b/t/lib/XCAT/GenesisReleaseTest.pm index 36716ab..7660784 100644 --- a/t/lib/XCAT/GenesisReleaseTest.pm +++ b/t/lib/XCAT/GenesisReleaseTest.pm @@ -19,6 +19,7 @@ our @EXPORT_OK = qw( dies_like make_export run_capture + write_forkmanager_stub write_checksums write_release_manifest ); @@ -78,6 +79,23 @@ sub copy_tree { } } +sub write_forkmanager_stub { + my ($root) = @_; + my $stub = "$root/Parallel/ForkManager.pm"; + make_path("$root/Parallel"); + write_binary( + $stub, + "package Parallel::ForkManager;\n" + . "sub new { bless {}, shift }\n" + . "sub run_on_finish { \$_[0]->{callback} = \$_[1] }\n" + . "sub start { 0 }\n" + . "sub finish { my (\$self, \$exit) = \@_; " + . "\$self->{callback}->(\$\$, \$exit, undef, 0, 0) if \$self->{callback}; 0 }\n" + . "sub wait_all_children { 0 }\n1;\n", + ); + return $root; +} + sub run_capture { my ($log, @command) = @_; my $pid = fork(); From 72cba4a4a556702c3609b43655d2e95d319ec239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sat, 22 Aug 2026 23:45:42 -0300 Subject: [PATCH 62/82] docs(genesis): state the release trust boundary --- genesis-openembedded/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/genesis-openembedded/README.md b/genesis-openembedded/README.md index 0c0efc6..7ac34cd 100644 --- a/genesis-openembedded/README.md +++ b/genesis-openembedded/README.md @@ -51,6 +51,9 @@ Validate the directory before publishing it: ./genesis-openembedded/verify-release --complete /path/to/xcat-genesis-release ``` +The checksum file detects incomplete or changed output. It does not authenticate +the release, so only accept a directory produced by a trusted build host. + Pass that same directory to the repository builders: ```bash From e13f2fc2074a53e958c91bc97581e81f05e32651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sun, 23 Aug 2026 00:10:03 -0300 Subject: [PATCH 63/82] fix(genesis): support custom RPM build layouts --- genesis-openembedded/package | 3 ++- genesis-openembedded/rpm/xCAT-genesis-openembedded.spec | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/genesis-openembedded/package b/genesis-openembedded/package index e98b4ab..9229229 100755 --- a/genesis-openembedded/package +++ b/genesis-openembedded/package @@ -32,7 +32,8 @@ my $format = 'all'; my $rpm_top; END { - remove_tree($rpm_top) if defined($rpm_top) && -d $rpm_top && !-l $rpm_top; + eval { remove_tree($rpm_top) } + if defined($rpm_top) && -d $rpm_top && !-l $rpm_top; } GetOptions( diff --git a/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec b/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec index bcb6266..965203c 100644 --- a/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec +++ b/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec @@ -18,7 +18,7 @@ service actions before a node boots its installed operating system. %prep rm -rf %{name}-%{version} -tar -xzf ../SOURCES/%{name}-%{version}.tar.gz +tar -xzf "$RPM_SOURCE_DIR/%{name}-%{version}.tar.gz" %build From 64afd0c935aabfcb8c0b686f88b2e81fe4e52725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Sun, 23 Aug 2026 00:10:03 -0300 Subject: [PATCH 64/82] test(genesis): isolate legacy workspace fixture --- t/genesis_openembedded_release.t | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t index dadffdb..276061b 100644 --- a/t/genesis_openembedded_release.t +++ b/t/genesis_openembedded_release.t @@ -292,20 +292,21 @@ sub exercise_packager { my ($format) = @_; my $first = "$tmp/$format-first"; my $second = "$tmp/$format-second"; + my $package_revision = sprintf('%032x%08x', time, $$); my @command = ( $packager, '--architecture', 'x86_64', '--export-dir', $export, '--version', $version, '--release', $release, - '--revision', $revision, + '--revision', $package_revision, '--source-date-epoch', $epoch, '--format', $format, ); my ($legacy_rpm_top, $legacy_rpm_work); if ($format eq 'rpm') { $legacy_rpm_top = - "/var/tmp/xcat-genesis-rpmbuild-$revision-x86_64-$version-$release"; + "/var/tmp/xcat-genesis-rpmbuild-$package_revision-x86_64-$version-$release"; $legacy_rpm_work = "$legacy_rpm_top/BUILD/stale"; make_path("$legacy_rpm_top/BUILD"); write_binary($legacy_rpm_work, 'stale work'); @@ -347,7 +348,7 @@ sub exercise_packager { make_path($release_root); copy_tree($first, $release_root); write_release_manifest( - $release_root, $version, $release, $revision, $epoch, 'x86_64', $format, + $release_root, $version, $release, $package_revision, $epoch, 'x86_64', $format, ); write_checksums($release_root); ok(validate_release($release_root), "$format release layout passes"); From 6ed0fa2ba055e69c8f8f617df92804468a887bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:07:04 -0300 Subject: [PATCH 65/82] fix(mockbuild): keep the empty-collection guards ahead of the Genesis release The Genesis release is installed from a verified directory instead of being built here, so counting it before the "did anything build?" checks made those checks unreachable with --genesis-release: a run whose builders had all failed carried seven release packages past the guard, ran createrepo, and copied them into the deployable per-EL tree before assert_required_deps stopped it, naming the missing dependencies rather than the failed builds. Assert on what the run collected first, then add the release. The source-RPM warning was masked the same way and moves with it. --- mockbuild-all.pl | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index b7b6634..0949f2c 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -492,13 +492,17 @@ my ($copied, $skipped_src, $missing_roots) = collect_rpms( dry_run => $dry_run, ); -$copied += install_genesis_release_packages('rpm', $repo_dir) - if $genesis_release && !$dry_run; - +# Assert on what this run BUILT, before the Genesis release is added: the release is +# installed from a verified directory rather than built here, so counting it first would +# let a run whose builders all failed reach createrepo and the deployable tree, and fail +# much later in assert_required_deps naming packages instead of the failed builds. if (!$dry_run && $copied == 0) { die "No binary RPMs were collected. Check build logs and collection roots.\n"; } +$copied += install_genesis_release_packages('rpm', $repo_dir) + if $genesis_release && !$dry_run; + # Ensure the OS-dependent xCAT-genesis-base rpm (built by the genesis step above) # lands in the dep repo even when the full xCAT core is built elsewhere (--skip-xcat). if (!$skip_genesis && !$dry_run) { @@ -520,16 +524,16 @@ my ($copied_srpms, $skipped_non_src, $missing_srpm_roots) = collect_srpms( dry_run => $dry_run, ); +if (!$dry_run && $copied_srpms == 0) { + print "WARN: No source RPMs were collected. SRPM repo and tarball may be empty.\n"; +} + $copied_srpms += install_genesis_release_packages('srpm', $srpm_repo_dir) if $genesis_release && !$dry_run; assert_genesis_release_copied($repo_dir, $srpm_repo_dir) if $genesis_release && !$dry_run; -if (!$dry_run && $copied_srpms == 0) { - print "WARN: No source RPMs were collected. SRPM repo and tarball may be empty.\n"; -} - if (!$skip_createrepo) { run_step( step => 'Run createrepo', From 7e76d523a4b1ca728c530bc66a8d69fcd3bed1a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:08:19 -0300 Subject: [PATCH 66/82] test(genesis): cover the empty-collection guard with a Genesis release A run with --genesis-release whose builders produced nothing must stop at the collection guard rather than reach the deployable repository, so pin the exit status, the message and the untouched deployment directory. --- t/genesis_openembedded_consumer.t | 35 ++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index 55e9db2..3ed5b4e 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -49,7 +49,7 @@ if ($ENV{XCAT_GENESIS_CI}) { } SKIP: { - skip 'RPM repository tools require a root Linux builder', 19 + skip 'RPM repository tools require a root Linux builder', 22 unless $^O eq 'linux' && $> == 0 && command_exists('rpmbuild') @@ -58,6 +58,7 @@ SKIP: { test_rpm_consumer(); test_legacy_rpm_consumer(); test_partial_rpm_release(); + test_failed_build_release(); } SKIP: { @@ -340,6 +341,38 @@ sub test_partial_deb_release { ok(-f $existing, 'partial DEB release does not remove the deployed package'); } +sub test_failed_build_release { + my $release_root = make_package_release("$tmp/rpm-empty", 'rpm'); + my $package = "xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; + my $output = "$tmp/empty-output"; + my $target = 'test+epel-10-' . capture_command('uname', '-m'); + my $deployed = "$output/xcat-dep/rh10/" . capture_command('uname', '-m'); + my $scratch_repo_root = "$tmp/rpm-empty-root"; + my $collected = "$tmp/rpm-empty-collect"; + make_path($deployed, $scratch_repo_root, $collected); + + my $log = "$tmp/rpm-empty.log"; + my $status = run_capture( + $log, + $^X, $rpm_consumer, + '--repo-root', $scratch_repo_root, + '--output', $output, + '--target', $target, + '--run-id', 'empty', + '--build-timestamp', $epoch, + '--skip-build', '--skip-xcat', '--skip-xcat-dep', '--skip-perl', + '--skip-createrepo', '--skip-tarball', + '--genesis-release', $release_root, + '--collect-dir', $collected, + ); + + isnt($status, 0, 'a run that built nothing fails even with a Genesis release'); + like(read_binary($log), qr/No binary RPMs were collected/, + 'the failure names the empty collection, not the missing dependencies'); + ok(!-e "$deployed/$package", + 'a run that built nothing publishes no release package'); +} + sub make_package_release { my ($root, $format, @requested_architectures) = @_; @requested_architectures = architectures() unless @requested_architectures; From 7ed58177a1300b7d179eed804e66df2f762094af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:08:37 -0300 Subject: [PATCH 67/82] fix(mockbuild): preview the Genesis release packages in dry runs Collection drops every xCAT-genesis-openembedded package once --genesis-release is given, and the install that puts the verified ones back was skipped in dry runs. A dry run therefore described a repository with no Genesis packages while the real run published the whole set. Report the release packages a real run would install and count them, so the dry-run summary matches what the run would do. --- mockbuild-all.pl | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 0949f2c..b733f8e 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -500,8 +500,11 @@ if (!$dry_run && $copied == 0) { die "No binary RPMs were collected. Check build logs and collection roots.\n"; } -$copied += install_genesis_release_packages('rpm', $repo_dir) - if $genesis_release && !$dry_run; +if ($genesis_release) { + $copied += $dry_run + ? preview_genesis_release_packages('rpm', $repo_dir) + : install_genesis_release_packages('rpm', $repo_dir); +} # Ensure the OS-dependent xCAT-genesis-base rpm (built by the genesis step above) # lands in the dep repo even when the full xCAT core is built elsewhere (--skip-xcat). @@ -528,8 +531,11 @@ if (!$dry_run && $copied_srpms == 0) { print "WARN: No source RPMs were collected. SRPM repo and tarball may be empty.\n"; } -$copied_srpms += install_genesis_release_packages('srpm', $srpm_repo_dir) - if $genesis_release && !$dry_run; +if ($genesis_release) { + $copied_srpms += $dry_run + ? preview_genesis_release_packages('srpm', $srpm_repo_dir) + : install_genesis_release_packages('srpm', $srpm_repo_dir); +} assert_genesis_release_copied($repo_dir, $srpm_repo_dir) if $genesis_release && !$dry_run; @@ -980,6 +986,21 @@ sub install_genesis_release_packages { return $copied; } +# Dry runs copy nothing, but they must still report what a real run would publish: +# collect_rpms and collect_srpms drop every xCAT-genesis-openembedded package whenever +# --genesis-release is given, so without this preview a dry run describes a repository +# with no Genesis packages at all while the real run installs the whole set. +sub preview_genesis_release_packages { + my ($prefix, $destination_root) = @_; + my @files = genesis_release_files($prefix); + die "Genesis release has no $prefix packages\n" unless @files; + for my $relative (@files) { + print "DRY-RUN install Genesis release package: $genesis_release/$relative" + . " -> $destination_root/" . basename($relative) . "\n"; + } + return scalar(@files); +} + sub genesis_release_files { my ($prefix) = @_; my @files = sort grep { From d7b39389b8d17587d023046b4358a82975726e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:09:02 -0300 Subject: [PATCH 68/82] test(genesis): cover the dry-run Genesis package preview Pin that a dry run with --genesis-release reports the release packages, counts them alongside the collected dependencies, and still installs nothing. --- t/genesis_openembedded_consumer.t | 39 ++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index 3ed5b4e..4881013 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -49,7 +49,7 @@ if ($ENV{XCAT_GENESIS_CI}) { } SKIP: { - skip 'RPM repository tools require a root Linux builder', 22 + skip 'RPM repository tools require a root Linux builder', 26 unless $^O eq 'linux' && $> == 0 && command_exists('rpmbuild') @@ -59,6 +59,7 @@ SKIP: { test_legacy_rpm_consumer(); test_partial_rpm_release(); test_failed_build_release(); + test_dry_run_release(); } SKIP: { @@ -373,6 +374,42 @@ sub test_failed_build_release { 'a run that built nothing publishes no release package'); } +sub test_dry_run_release { + my $release_root = make_package_release("$tmp/rpm-dry", 'rpm'); + my $package = "xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; + my $output = "$tmp/dry-output"; + my $target = 'test+epel-10-' . capture_command('uname', '-m'); + my $run_repo = "$output/mockbuild-all/$target-dry/repo/" . capture_command('uname', '-m'); + my $dependencies = "$tmp/rpm-dry-dependencies"; + my $scratch_repo_root = "$tmp/rpm-dry-root"; + make_rpm_dependencies($dependencies, "$release_root/rpm/$package"); + make_path($scratch_repo_root); + + my $log = "$tmp/rpm-dry.log"; + my $status = run_capture( + $log, + $^X, $rpm_consumer, + '--repo-root', $scratch_repo_root, + '--output', $output, + '--target', $target, + '--run-id', 'dry', + '--build-timestamp', $epoch, + '--skip-build', '--skip-xcat', '--skip-xcat-dep', '--skip-perl', + '--skip-createrepo', '--skip-tarball', + '--genesis-release', $release_root, + '--collect-dir', $dependencies, + '--dry-run', + ); + my $printed = read_binary($log); + + is($status, 0, 'a dry run accepts a verified Genesis release'); + like($printed, qr/^DRY-RUN install Genesis release package: .*\Q$package\E$/m, + 'the dry run reports the release packages it would install'); + like($printed, qr/^Collected binary RPMs: 15$/m, + 'the dry run counts the release packages a real run installs'); + ok(!-e "$run_repo/$package", 'the dry run installs nothing'); +} + sub make_package_release { my ($root, $format, @requested_architectures) = @_; @requested_architectures = architectures() unless @requested_architectures; From 5a097c353205d0f2d61c2c505096422414bbdd95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:09:21 -0300 Subject: [PATCH 69/82] build(apt-repo): require cmp only where the Genesis release needs it cmp guards the release manifest against changing during verification, which is a --genesis-release concern; a legacy build never reaches it. Check for it with the other release prerequisites instead of failing a plain build over a tool it does not use. --- build-apt-repo.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-apt-repo.sh b/build-apt-repo.sh index 83fc300..12a66cc 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -118,11 +118,11 @@ command -v apt-ftparchive >/dev/null 2>&1 \ || die "apt-ftparchive not found. Install: sudo apt-get install apt-utils" command -v gpg >/dev/null 2>&1 \ || die "gpg not found. Install: sudo apt-get install gnupg" -command -v cmp >/dev/null 2>&1 \ - || die "cmp not found. Install: sudo apt-get install diffutils" if [[ -n "$GENESIS_RELEASE" ]]; then command -v dpkg-deb >/dev/null 2>&1 \ || die "dpkg-deb not found. Install: sudo apt-get install dpkg" + command -v cmp >/dev/null 2>&1 \ + || die "cmp not found. Install: sudo apt-get install diffutils" GENESIS_CHECKSUMS=$(mktemp "${TMPDIR:-/tmp}/xcat-genesis-checksums.XXXXXX") cp -- "$GENESIS_RELEASE/SHA256SUMS" "$GENESIS_CHECKSUMS" "$GENESIS_VERIFIER" --complete --format deb "$GENESIS_RELEASE" From 2fe5f186a1897258ac5a1e3d5f0af364b50c82ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:09:47 -0300 Subject: [PATCH 70/82] build(genesis): drop the _buildtime define no rpm 4 honours %_buildtime is not an rpm macro: built with it alone and no SOURCE_DATE_EPOCH, rpm 4.14.3, 4.16.1.3 and 4.19.1.1 all stamped the current time. The build time and build host the release verifier checks come from SOURCE_DATE_EPOCH with use_source_date_epoch_as_buildtime, and from _buildhost -- both honoured on every one of those versions and on rpm 6.0.2, where the package also builds and verifies unchanged. --- genesis-openembedded/package | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/genesis-openembedded/package b/genesis-openembedded/package index 9229229..937e380 100755 --- a/genesis-openembedded/package +++ b/genesis-openembedded/package @@ -131,7 +131,10 @@ sub _build_rpm { '--define', "genesis_arch $architecture", '--define', "version $version", '--define', "release $release", - '--define', "_buildtime $source_date_epoch", + # SOURCE_DATE_EPOCH is exported above; rpm turns it into the build time only + # when asked, and takes the build host from _buildhost. Both are honoured from + # rpm 4.14 through 6, which is what makes the identity verify-release expects + # reproducible on every builder. '--define', '_buildhost xcat-release', '--define', 'use_source_date_epoch_as_buildtime 1', '--define', 'clamp_mtime_to_source_date_epoch 1', From 5b226fdad4d887842ee731abc52063bef52533c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:10:04 -0300 Subject: [PATCH 71/82] docs(genesis): explain the release re-checksum and the verified rpm versions The two checksum passes around the verifier read as a duplicated walk; say what the second one is for. Record which rpm versions reproduce the package identity verify-release requires, so a build host outside that set is a known question rather than a guess. --- genesis-openembedded/README.md | 5 +++++ mockbuild-all.pl | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/genesis-openembedded/README.md b/genesis-openembedded/README.md index 7ac34cd..ffdd194 100644 --- a/genesis-openembedded/README.md +++ b/genesis-openembedded/README.md @@ -54,6 +54,11 @@ Validate the directory before publishing it: The checksum file detects incomplete or changed output. It does not authenticate the release, so only accept a directory produced by a trusted build host. +`verify-release` also checks the identity of each package, including a fixed +build host and a build time taken from the source epoch. Those are reproduced +by rpm 4.14.3, 4.16.1.3, 4.19.1.1 and 6.0.2, so an EL8 or later builder -- and +a current Fedora one -- produces a release the verifier accepts. + Pass that same directory to the repository builders: ```bash diff --git a/mockbuild-all.pl b/mockbuild-all.pl index b733f8e..f266dfa 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -163,6 +163,10 @@ if ($genesis_release ne '') { unless -d $genesis_release; my $verifier = "$script_dir/genesis-openembedded/verify-release"; die "Genesis release verifier not found: $verifier\n" unless -x $verifier; + # Checksum, verify, checksum again. The verifier reads the tree it validates, so a + # release rewritten together with its SHA256SUMS while the verifier runs would satisfy + # both the verifier and any single pass taken afterwards; comparing the pass taken + # before with the one taken after is what closes that window. my $checksums_before = validated_release_checksums($genesis_release); run_command($^X, $verifier, '--complete', '--format', 'rpm', $genesis_release); my $checksums_after = validated_release_checksums($genesis_release); From 8ab9b909a15c5a5c9df3a73215e249786cad6783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:57:25 -0300 Subject: [PATCH 72/82] fix(apt-repo): copy each Genesis release deb once per suite, reflinked Every selected suite receives the whole release, so publication copied one image per architecture per codename -- hundreds of megabytes each, several gigabytes on the build host and again in the published tree. Use --reflink=auto: filesystems that can share extents copy-on-write stop paying for the duplication, and the pool entry stays a file of its own everywhere, which a hardlink would not -- that would leave the published package and the verified release sharing an inode, where a write through either path changes the other. --- build-apt-repo.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build-apt-repo.sh b/build-apt-repo.sh index 12a66cc..d40d91f 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -195,7 +195,13 @@ copy_genesis_deb() { name=$(basename "$source") destination="$directory/$name" relative="deb/$name" - cp -- "$source" "$destination" + # Every selected suite receives the whole release, so a plain copy spends hundreds of + # megabytes per suite. --reflink=auto lets a filesystem that can share extents + # copy-on-write avoid that, while still giving the pool a file of its own: a link would + # leave the published package and the verified release sharing one inode, where a write + # through either path changes what the other holds. + cp --reflink=auto -- "$source" "$destination" 2>/dev/null \ + || cp -- "$source" "$destination" "$GENESIS_VERIFIER" \ --checksum-file "$GENESIS_CHECKSUMS" \ --relative-file "$relative" \ From 7b9dcaeff7ca0eaaea59c34533fa5f15e3bdb3fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:45:23 -0300 Subject: [PATCH 73/82] test(genesis): pin the pooled Genesis DEB as an independent file The pooled package carries the release bytes, which the existing digest check covers, but it must not be the release file itself; assert the identity is distinct so a link cannot come back unnoticed. --- t/genesis_openembedded_consumer.t | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index 4881013..db34048 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -198,6 +198,12 @@ sub test_deb_consumer { 'legacy Genesis DEB remains available'); ok(!-e "$apt_root/pool/main/noble/xcat-genesis-openembedded-stale.deb", 'stale OpenEmbedded DEB is not collected'); + # The published package must be a file of its own: sharing an inode with the release + # would make a later write through either path change what the other one holds. + my @pooled = stat($pool_package); + my @released = stat("$release_root/deb/$package"); + isnt("$pooled[0]:$pooled[1]", "$released[0]:$released[1]", + 'pooled DEB is published independently of the release file'); my $collision = "$tmp/apt-collision"; make_path("$collision/ubuntu24.04"); From b0c0345b68831b172d921e230e94b562f14493b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:43:38 -0300 Subject: [PATCH 74/82] fix(apt-repo): re-verify the pooled Genesis packages before indexing apt-ftparchive indexes and the signature covers whatever is in the pool at that moment, while the packages were checked when they were copied. Verify them again against the release manifest immediately before the indexes are generated, so a package that changed in between cannot be published as a verified one. --- build-apt-repo.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/build-apt-repo.sh b/build-apt-repo.sh index d40d91f..542a179 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -228,6 +228,25 @@ for ver in "${SELECTED_VERS[@]}"; do fi done +# The pool is indexed and the metadata is signed from what is on disk now, not from what +# was copied earlier, so check the packages again here: anything that changed between the +# copy and this point would otherwise be published and signed as verified. +if [[ -n "$GENESIS_RELEASE" && $DRY_RUN -eq 0 ]]; then + step "Re-verifying pooled Genesis packages" + for ver in "${SELECTED_VERS[@]}"; do + codename="${CODENAME_MAP[$ver]}" + for deb in "$GENESIS_RELEASE"/deb/*.deb; do + name=$(basename "$deb") + pooled="$APT_DIR/pool/main/$codename/$name" + "$GENESIS_VERIFIER" \ + --checksum-file "$GENESIS_CHECKSUMS" \ + --relative-file "deb/$name" \ + --copied-file "$pooled" + echo "Re-verified pooled Genesis package: $pooled" + done + done +fi + step "Generating Packages indexes" for ver in "${SELECTED_VERS[@]}"; do From 896d250d6248141dbf05b0b3835da78ec516e423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:45:32 -0300 Subject: [PATCH 75/82] test(genesis): cover the pre-index re-verification of pooled packages Pin that every pooled Genesis package is checked against the release manifest again before apt-ftparchive reads the pool, so the check cannot be dropped without a failing test. --- t/genesis_openembedded_consumer.t | 2 ++ 1 file changed, 2 insertions(+) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index db34048..b99ab13 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -204,6 +204,8 @@ sub test_deb_consumer { my @released = stat("$release_root/deb/$package"); isnt("$pooled[0]:$pooled[1]", "$released[0]:$released[1]", 'pooled DEB is published independently of the release file'); + like(read_binary($log), qr/^Re-verified pooled Genesis package: \Q$pool_package\E$/m, + 'pooled DEB is verified again before the indexes are generated'); my $collision = "$tmp/apt-collision"; make_path("$collision/ubuntu24.04"); From 07debd3ee843df234248c1b8a8e0d70d96d42b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:43:56 -0300 Subject: [PATCH 76/82] fix(mockbuild): stage each invocation into empty repository directories The collection guard counts what this run copied, but the staging repositories were never cleared: packages left by an earlier run with the same --run-id sat there unseen by collection, were indexed by createrepo and published by deploy_target, where the name-only dependency assertion accepted them. A run whose builders all failed could therefore ship a previous run's packages. Empty the staging repositories before collection, so everything they hold afterwards comes from this invocation. --- mockbuild-all.pl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index f266dfa..1792d37 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -284,6 +284,12 @@ die "Missing perl builder script: $perl_builder\n" if (!$dry_run) { make_path($build_root, $log_root, $repo_dir, $srpm_repo_dir); + # The staging repositories hold what THIS invocation produces. A reused --run-id, or a + # rerun after a failed invocation, otherwise leaves an earlier run's packages in them, + # where collection never sees them, createrepo indexes them and deploy_target publishes + # them as this run's output. + reset_staging_repo($repo_dir); + reset_staging_repo($srpm_repo_dir); } print_step("Configuration"); @@ -957,6 +963,21 @@ sub assert_required_deps { print "[deps] required set present in $dir: @req\n"; } +sub reset_staging_repo { + my ($directory) = @_; + return unless -d $directory; + opendir(my $dh, $directory) or die "Cannot read $directory: $!\n"; + my @stale = grep { /\.rpm\z/ } readdir($dh); + closedir($dh) or die "Cannot close $directory: $!\n"; + return unless @stale; + print "Removing " . scalar(@stale) . " package(s) left in $directory by an earlier run\n"; + for my $name (@stale) { + unlink("$directory/$name") + or die "Cannot remove stale package $directory/$name: $!\n"; + } + return; +} + sub remove_genesis_packages { my ($directory, $source, $keep_names) = @_; return unless -d $directory; From 5ab387fb7e4d6922c17a09e25db76fa2cf8f13df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:45:43 -0300 Subject: [PATCH 77/82] test(genesis): cover the staging repository reset A package left in the staging repository by an earlier run is invisible to collection but visible to createrepo and deploy. Pin that the run clears it, so the empty-collection guard cannot be satisfied by an earlier invocation's output. --- t/genesis_openembedded_consumer.t | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index b99ab13..0174230 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -49,7 +49,7 @@ if ($ENV{XCAT_GENESIS_CI}) { } SKIP: { - skip 'RPM repository tools require a root Linux builder', 26 + skip 'RPM repository tools require a root Linux builder', 27 unless $^O eq 'linux' && $> == 0 && command_exists('rpmbuild') @@ -358,7 +358,10 @@ sub test_failed_build_release { my $deployed = "$output/xcat-dep/rh10/" . capture_command('uname', '-m'); my $scratch_repo_root = "$tmp/rpm-empty-root"; my $collected = "$tmp/rpm-empty-collect"; - make_path($deployed, $scratch_repo_root, $collected); + my $run_repo = "$output/mockbuild-all/$target-empty/repo/" . capture_command('uname', '-m'); + my $stale = "$run_repo/ipmitool-xcat-0-stale.noarch.rpm"; + make_path($deployed, $scratch_repo_root, $collected, $run_repo); + write_binary($stale, 'package left by an earlier run'); my $log = "$tmp/rpm-empty.log"; my $status = run_capture( @@ -380,6 +383,8 @@ sub test_failed_build_release { 'the failure names the empty collection, not the missing dependencies'); ok(!-e "$deployed/$package", 'a run that built nothing publishes no release package'); + ok(!-e $stale, + 'a package left by an earlier run is cleared from the staging repository'); } sub test_dry_run_release { From 54be7520065bdb0783434587018ee4408caab197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:50:33 -0300 Subject: [PATCH 78/82] fix(mockbuild): fail when every build step of the invocation failed Tolerating individual dep-builder failures is deliberate: some packages are el- or arch-pinned and are expected to fail on some targets. Tolerating all of them is not -- it means the builder is unusable, the invocation produced nothing, and whatever the run publishes came from somewhere other than this build. Count the failures on both the serial and the parallel path and stop when they account for every attempted step, before collection can take an earlier run's artifacts for this one's. The rule itself lives in BuildUtils, where it can be exercised without a builder. --- lib/XCAT/BuildUtils.pm | 9 +++++++++ mockbuild-all.pl | 19 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/XCAT/BuildUtils.pm b/lib/XCAT/BuildUtils.pm index c8fa7b0..c64a518 100644 --- a/lib/XCAT/BuildUtils.pm +++ b/lib/XCAT/BuildUtils.pm @@ -17,6 +17,7 @@ our @EXPORT_OK = qw( digest_file digest_manifest display_quote + every_step_failed hashes_equal print_step read_binary @@ -147,6 +148,14 @@ sub digest_manifest { ); } +# True when a set of steps was attempted and none of them survived. Callers tolerate +# individual failures; losing every step means the builder itself did not work. +sub every_step_failed { + my ($attempted, $failures) = @_; + return 0 unless $attempted; + return $failures >= $attempted ? 1 : 0; +} + sub hashes_equal { my ($left, $right) = @_; return 0 unless keys(%{$left}) == keys(%{$right}); diff --git a/mockbuild-all.pl b/mockbuild-all.pl index 1792d37..bf0a7b0 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -17,6 +17,7 @@ use POSIX qw(strftime); use lib "$FindBin::Bin/lib"; use XCAT::BuildUtils qw( capture_command + every_step_failed hashes_equal print_step read_lines @@ -875,10 +876,14 @@ sub run_build_steps_parallel { # src.rpm). We collect whatever built and assert the REQUIRED set later (assert_required_deps), # matching the historical build behaviour. if ($dry_run || $max_processes <= 1 || @{$steps} == 1) { + my $serial_failures = 0; for my $step (@{$steps}) { my $ok = eval { run_step(%{$step}); 1 }; - warn "WARN: build step failed (tolerated): $step->{step}\n" . ($@ // '') unless $ok; + next if $ok; + $serial_failures++; + warn "WARN: build step failed (tolerated): $step->{step}\n" . ($@ // ''); } + assert_build_progress(scalar(@{$steps}), $serial_failures); return; } @@ -938,6 +943,18 @@ sub run_build_steps_parallel { warn "WARN: some build steps failed (tolerated; required deps asserted after deploy):\n " . join("\n ", @lines) . "\n"; } + + assert_build_progress(scalar(@{$steps}), scalar(keys %failed)); +} + +# Individual failures are tolerated because some packages are el- or arch-pinned. ALL of them +# failing is a different thing: the builder itself is unusable (no mock, a broken chroot, no +# network), this invocation produced nothing, and every package the run would go on to publish +# would come from somewhere other than this build. +sub assert_build_progress { + my ($attempted, $failures) = @_; + return unless every_step_failed($attempted, $failures); + die "FATAL: every build step failed ($failures/$attempted). Check the build logs.\n"; } # have_rpm: is there a non-src rpm named -... under $dir? From 6d586f7611113dc0c888fec38b46e92802ca7958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:50:43 -0300 Subject: [PATCH 79/82] test(build): cover the all-steps-failed rule Pin the rule that decides whether an invocation produced anything: every attempted step failing is a total failure, one survivor is not, and a run with no steps to attempt is unaffected. --- t/build_utils.t | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/t/build_utils.t b/t/build_utils.t index af8aad8..4cee6d6 100644 --- a/t/build_utils.t +++ b/t/build_utils.t @@ -13,6 +13,7 @@ use XCAT::BuildUtils qw( digest_file digest_manifest display_quote + every_step_failed hashes_equal read_binary read_first_line @@ -68,6 +69,10 @@ like($@, qr/Unsupported digest algorithm/, 'unknown digest is rejected'); is(display_quote('plain/value'), 'plain/value', 'simple display value is unquoted'); is(display_quote('two words'), q{'two words'}, 'display value with spaces is quoted'); is(shell_quote("it's"), q{'it'"'"'s'}, 'shell quote escapes apostrophes'); +ok(every_step_failed(6, 6), 'every attempted step failing is reported'); +ok(!every_step_failed(6, 5), 'one surviving step is not a total failure'); +ok(!every_step_failed(0, 0), 'attempting no steps is not a failure'); + ok(hashes_equal({ a => 1 }, { a => 1 }), 'equal hashes match'); ok(!hashes_equal({ a => 1 }, { a => 2 }), 'different hashes do not match'); From 13ac168e8c7c5be9f0d4bd92497f39c74513669b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:17:53 -0300 Subject: [PATCH 80/82] fix(mockbuild): build into an empty result tree The staging repositories are cleared per invocation, but the builder results are not: a reused --run-id leaves the previous run's packages under build-results, where a step that fails this time is collected from the last time it succeeded. Individual failures are tolerated by design, so this can mix two invocations in one signed repository. Start the result tree empty whenever the run builds. --skip-build collects the repository-level build-output tree instead and keeps what is there. --- mockbuild-all.pl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mockbuild-all.pl b/mockbuild-all.pl index bf0a7b0..608cc47 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -8,7 +8,7 @@ use File::Basename qw(dirname basename); use File::Copy qw(copy); use File::Find qw(find); use File::Glob qw(bsd_glob); -use File::Path qw(make_path); +use File::Path qw(make_path remove_tree); use File::Temp qw(tempfile); use FindBin; use Getopt::Long qw(GetOptions); @@ -291,6 +291,14 @@ if (!$dry_run) { # them as this run's output. reset_staging_repo($repo_dir); reset_staging_repo($srpm_repo_dir); + # Same for the builder results when this run is going to build them: a reused --run-id + # otherwise leaves a previous run's packages there, and a step that fails this time is + # collected from the last time it succeeded. --skip-build deliberately collects earlier + # output, from the repository-level build-output tree, and must keep what is there. + if (!$skip_build) { + remove_tree($build_root); + make_path($build_root); + } } print_step("Configuration"); From 4527841b135cdb696ad432119995cd905da2d312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:18:53 -0300 Subject: [PATCH 81/82] fix(apt-repo): hold the APT directory for the whole publication Everything from the pool wipe to the signature is one transaction over a shared tree, and the Genesis packages are verified inside it, so a second writer between that verification and apt-ftparchive would be indexed and signed unchecked -- or, arriving later, would leave clients with checksum failures against metadata this run signed. Take the tree for the duration with an atomic mkdir lock, the same NFS-safe scheme mockbuild-all.pl uses for its output, released on exit and overridable with --force-unlock when a killed run left one behind. --- build-apt-repo.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/build-apt-repo.sh b/build-apt-repo.sh index 542a179..5fbfe5c 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -16,11 +16,17 @@ DRY_RUN=0 GENESIS_RELEASE="" GENESIS_CHECKSUMS="" GENESIS_VERIFIER="" +FORCE_UNLOCK=0 +HELD_LOCK="" cleanup() { if [[ -n "$GENESIS_CHECKSUMS" ]]; then rm -f -- "$GENESIS_CHECKSUMS" fi + if [[ -n "$HELD_LOCK" ]]; then + rm -f -- "$HELD_LOCK/owner" + rmdir -- "$HELD_LOCK" 2>/dev/null || true + fi } trap cleanup EXIT @@ -57,6 +63,7 @@ Options: --skip-sign Skip GPG signing (for testing) --genesis-release PATH Add a verified OpenEmbedded Genesis DEB release to each selected suite --dry-run Print planned actions without executing + --force-unlock Remove a stale /.lock before acquiring it -h, --help Show this help Examples: @@ -84,6 +91,7 @@ while [[ $# -gt 0 ]]; do --skip-sign) SKIP_SIGN=1; shift ;; --genesis-release) GENESIS_RELEASE="$2"; shift 2 ;; --dry-run) DRY_RUN=1; shift ;; + --force-unlock) FORCE_UNLOCK=1; shift ;; -h|--help) usage; exit 0 ;; -*) die "Unknown option: $1" ;; *) SELECTED_VERS+=("$1"); SUBSET=1; shift ;; @@ -93,6 +101,30 @@ done REPO_ROOT="$(cd "$REPO_ROOT" && pwd)" APT_DIR="${APT_DIR:-$REPO_ROOT/repos/apt}" +# Everything from the pool wipe to the signature is one transaction over a shared tree, and +# the packages are verified inside it: a second writer between the verification and +# apt-ftparchive would be indexed and signed unchecked. Hold the tree for the whole run, with +# an atomic mkdir (reliable over NFS, unlike flock) like mockbuild-all.pl does for its output. +acquire_apt_lock() { + local lock="$APT_DIR/.lock" + [[ $DRY_RUN -eq 0 ]] || return 0 + mkdir -p -- "$APT_DIR" + if [[ $FORCE_UNLOCK -eq 1 && -d "$lock" ]]; then + echo "force-unlock: removing stale lock $lock" + rm -f -- "$lock/owner" + rmdir -- "$lock" 2>/dev/null || true + fi + if mkdir -- "$lock" 2>/dev/null; then + HELD_LOCK="$lock" + printf 'host=%s\npid=%s\nepoch=%s\n' "$(uname -n)" "$$" "$(date +%s)" > "$lock/owner" + return 0 + fi + [[ -d "$lock" ]] || die "Cannot create lock $lock" + die "APT directory $APT_DIR is locked ($lock): $(tr '\n' ' ' < "$lock/owner" 2>/dev/null) +another build-apt-repo.sh run owns it; use a different --apt-dir or --force-unlock if stale." +} +acquire_apt_lock + if [[ -n "$GENESIS_RELEASE" ]]; then [[ -d "$GENESIS_RELEASE" ]] || die "Genesis release directory not found: $GENESIS_RELEASE" GENESIS_RELEASE="$(cd "$GENESIS_RELEASE" && pwd)" From c644148fe0fa9d8864305bff98196b8455bd9bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Ferr=C3=A3o?= <2031761+viniciusferrao@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:19:18 -0300 Subject: [PATCH 82/82] test(genesis): cover the APT lock and the preserved build results Pin that a locked APT directory is refused with a message naming it, that --force-unlock takes over and the lock is released at the end, and that a run which skips building keeps the build results it exists to collect. --- t/genesis_openembedded_consumer.t | 43 +++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t index 0174230..391238f 100644 --- a/t/genesis_openembedded_consumer.t +++ b/t/genesis_openembedded_consumer.t @@ -49,7 +49,7 @@ if ($ENV{XCAT_GENESIS_CI}) { } SKIP: { - skip 'RPM repository tools require a root Linux builder', 27 + skip 'RPM repository tools require a root Linux builder', 28 unless $^O eq 'linux' && $> == 0 && command_exists('rpmbuild') @@ -72,6 +72,7 @@ SKIP: { test_deb_consumer(); test_legacy_deb_consumer(); test_partial_deb_release(); + test_apt_lock(); } done_testing(); @@ -359,9 +360,12 @@ sub test_failed_build_release { my $scratch_repo_root = "$tmp/rpm-empty-root"; my $collected = "$tmp/rpm-empty-collect"; my $run_repo = "$output/mockbuild-all/$target-empty/repo/" . capture_command('uname', '-m'); + my $results = "$output/mockbuild-all/$target-empty/build-results/ipmitool-xcat"; my $stale = "$run_repo/ipmitool-xcat-0-stale.noarch.rpm"; - make_path($deployed, $scratch_repo_root, $collected, $run_repo); + my $kept = "$results/ipmitool-xcat-0-earlier.noarch.rpm"; + make_path($deployed, $scratch_repo_root, $collected, $run_repo, $results); write_binary($stale, 'package left by an earlier run'); + write_binary($kept, 'build output an earlier run produced'); my $log = "$tmp/rpm-empty.log"; my $status = run_capture( @@ -385,6 +389,8 @@ sub test_failed_build_release { 'a run that built nothing publishes no release package'); ok(!-e $stale, 'a package left by an earlier run is cleared from the staging repository'); + ok(-e $kept, + 'a run that skips building keeps the build results it collects from'); } sub test_dry_run_release { @@ -423,6 +429,39 @@ sub test_dry_run_release { ok(!-e "$run_repo/$package", 'the dry run installs nothing'); } +sub test_apt_lock { + my $apt_root = "$tmp/apt-lock"; + my $input = "$apt_root/ubuntu24.04"; + make_path($input, "$apt_root/.lock"); + make_legacy_deb("$tmp/lock-deb", "$input/xcat-genesis-base-amd64_1_all.deb"); + + my $locked_log = "$tmp/deb-locked.log"; + my $locked_status = run_capture( + $locked_log, + 'bash', $deb_consumer, + '--repo-root', $repo_root, + '--apt-dir', $apt_root, + '--skip-sign', + 'ubuntu24.04', + ); + isnt($locked_status, 0, 'a locked APT directory is not published into'); + like(read_binary($locked_log), qr/\Q$apt_root\E is locked/, + 'the refusal names the directory another run owns'); + + my $forced_log = "$tmp/deb-forced.log"; + my $forced_status = run_capture( + $forced_log, + 'bash', $deb_consumer, + '--repo-root', $repo_root, + '--apt-dir', $apt_root, + '--skip-sign', + '--force-unlock', + 'ubuntu24.04', + ); + is($forced_status, 0, '--force-unlock takes over a stale lock'); + ok(!-d "$apt_root/.lock", 'the lock is released when the run finishes'); +} + sub make_package_release { my ($root, $format, @requested_architectures) = @_; @requested_architectures = architectures() unless @requested_architectures;