diff --git a/.github/workflows/genesis-openembedded.yml b/.github/workflows/genesis-openembedded.yml new file mode 100644 index 0000000..25171d1 --- /dev/null +++ b/.github/workflows/genesis-openembedded.yml @@ -0,0 +1,59 @@ +name: Genesis OpenEmbedded packages + +on: + pull_request: + push: + branches: + - master + +permissions: + contents: read + +jobs: + package-tests: + runs-on: ubuntu-24.04 + env: + XCAT_GENESIS_CI: 1 + steps: + - uses: actions/checkout@v4 + + - name: Install build tools + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + apt-utils createrepo-c dpkg-dev gnupg \ + libfile-slurper-perl \ + libparallel-forkmanager-perl libperl-critic-perl rpm shellcheck + + - name: Run static checks + run: | + perl -c genesis-openembedded/build + perl -c genesis-openembedded/package + perl -c genesis-openembedded/verify-release + perl -Ilib -c lib/XCAT/BuildUtils.pm + perl -Ilib -c lib/XCAT/GenesisRelease.pm + perl -c mockbuild-all.pl + rpmspec -P \ + -D 'genesis_arch x86_64' \ + -D 'version 2.19.0' \ + -D 'release snap202608210726' \ + genesis-openembedded/rpm/xCAT-genesis-openembedded.spec >/dev/null + perlcritic \ + genesis-openembedded/build \ + genesis-openembedded/package \ + genesis-openembedded/verify-release \ + lib/XCAT/BuildUtils.pm \ + lib/XCAT/GenesisRelease.pm \ + mockbuild-all.pl \ + t/build_utils.t \ + t/genesis_openembedded_release.t \ + t/genesis_openembedded_consumer.t \ + t/lib/XCAT/GenesisReleaseTest.pm + bash -n build-apt-repo.sh + shellcheck build-apt-repo.sh + + - name: Run package tests + run: | + prove -v t/build_utils.t + prove -v -It/lib t/genesis_openembedded_release.t + sudo -E prove -v -It/lib t/genesis_openembedded_consumer.t diff --git a/.gitignore b/.gitignore index 0622450..ed2cb46 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ Gitepoch Gitinfo Release +build-output/ diff --git a/BUILD.md b/BUILD.md index b969225..75d4750 100644 --- a/BUILD.md +++ b/BUILD.md @@ -112,6 +112,11 @@ Use these flags to skip specific operations: - Skips `createrepo --update`. - `--skip-tarball` - Skips tarball creation for both binary and SRPM repos. +- `--skip-genesis` + - Skips the existing per-EL Genesis image build. +- `--genesis-release ` + - Adds a verified OpenEmbedded Genesis RPM release alongside the existing + per-EL Genesis packages. - `--scrub-all-chroots` - Runs `mock -r --scrub=all` before build and collection. - `--keep-buildroots` @@ -138,7 +143,9 @@ Use these flags to skip specific operations: Install baseline tooling: ```bash -dnf -y install perl perl-Parallel-ForkManager mock createrepo tar rpm-build rpmdevtools dnf-plugins-core wget git +dnf -y install perl perl-File-Slurper perl-IPC-Cmd \ + perl-Parallel-ForkManager mock createrepo tar rpm-build rpmdevtools \ + dnf-plugins-core wget git ``` If you will build the `xCAT-genesis-base` package (that is, you will **not** use `--skip-genesis`), install xCAT build dependencies: @@ -202,6 +209,39 @@ Notes: `/buildrpms.pl`). - `` is optional; when omitted it is timestamp-based. +# Add an OpenEmbedded Genesis Release + +Build the Genesis package release first, following +[`genesis-openembedded/README.md`](genesis-openembedded/README.md). Then pass the +result to the regular repository build: + +```bash +cd +perl ./mockbuild-all.pl \ + --repo-root \ + --xcat-source \ + --genesis-release /path/to/xcat-genesis-release +``` + +The release is checked before any package is collected. Its RPMs and SRPMs are +copied into each generated EL repository. Stale OpenEmbedded Genesis packages +are removed from the output first, while the existing per-EL Genesis package +remains available. + +Repository publication requires a release containing every supported Genesis +architecture. The packages are `noarch`, and every management-node repository +receives the full set of target images. + +The release checksums cover the unsigned input packages. If repository signing +is enabled, `rpmsign` changes the deployed RPM bytes after collection. + +The OpenEmbedded packages use their own names and install under +`/opt/xcat/share/xcat/netboot/genesis-openembedded/`. Publishing them does not +replace the Genesis packages used by current xcat-core releases. Activation is +a separate xcat-core change. + +Omit `--genesis-release` to keep using the existing Genesis builder. + # Common Build Modes xcat-dep repo: diff --git a/build-apt-repo.sh b/build-apt-repo.sh index 7a2b8ca..5fbfe5c 100755 --- a/build-apt-repo.sh +++ b/build-apt-repo.sh @@ -4,7 +4,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" if [ -z "${SOURCE_DATE_EPOCH:-}" ] && [ -f "$SCRIPT_DIR/Gitepoch" ]; then - export SOURCE_DATE_EPOCH=$(cat "$SCRIPT_DIR/Gitepoch") + SOURCE_DATE_EPOCH=$(cat "$SCRIPT_DIR/Gitepoch") + export SOURCE_DATE_EPOCH fi REPO_ROOT="$SCRIPT_DIR" @@ -12,6 +13,22 @@ APT_DIR="" GPG_KEY_ID="xcat@megware.com" SKIP_SIGN=0 DRY_RUN=0 +GENESIS_RELEASE="" +GENESIS_CHECKSUMS="" +GENESIS_VERIFIER="" +FORCE_UNLOCK=0 +HELD_LOCK="" + +cleanup() { + if [[ -n "$GENESIS_CHECKSUMS" ]]; then + rm -f -- "$GENESIS_CHECKSUMS" + fi + if [[ -n "$HELD_LOCK" ]]; then + rm -f -- "$HELD_LOCK/owner" + rmdir -- "$HELD_LOCK" 2>/dev/null || true + fi +} +trap cleanup EXIT declare -A CODENAME_MAP=( [ubuntu22.04]=jammy @@ -44,7 +61,9 @@ Options: --apt-dir PATH APT output directory (default: /repos/apt) --gpg-key-id ID GPG key ID for signing (default: xcat@megware.com) --skip-sign Skip GPG signing (for testing) + --genesis-release PATH Add a verified OpenEmbedded Genesis DEB release to each selected suite --dry-run Print planned actions without executing + --force-unlock Remove a stale /.lock before acquiring it -h, --help Show this help Examples: @@ -70,7 +89,9 @@ while [[ $# -gt 0 ]]; do --apt-dir) APT_DIR="$2"; shift 2 ;; --gpg-key-id) GPG_KEY_ID="$2"; shift 2 ;; --skip-sign) SKIP_SIGN=1; shift ;; + --genesis-release) GENESIS_RELEASE="$2"; shift 2 ;; --dry-run) DRY_RUN=1; shift ;; + --force-unlock) FORCE_UNLOCK=1; shift ;; -h|--help) usage; exit 0 ;; -*) die "Unknown option: $1" ;; *) SELECTED_VERS+=("$1"); SUBSET=1; shift ;; @@ -80,6 +101,37 @@ done REPO_ROOT="$(cd "$REPO_ROOT" && pwd)" APT_DIR="${APT_DIR:-$REPO_ROOT/repos/apt}" +# Everything from the pool wipe to the signature is one transaction over a shared tree, and +# the packages are verified inside it: a second writer between the verification and +# apt-ftparchive would be indexed and signed unchecked. Hold the tree for the whole run, with +# an atomic mkdir (reliable over NFS, unlike flock) like mockbuild-all.pl does for its output. +acquire_apt_lock() { + local lock="$APT_DIR/.lock" + [[ $DRY_RUN -eq 0 ]] || return 0 + mkdir -p -- "$APT_DIR" + if [[ $FORCE_UNLOCK -eq 1 && -d "$lock" ]]; then + echo "force-unlock: removing stale lock $lock" + rm -f -- "$lock/owner" + rmdir -- "$lock" 2>/dev/null || true + fi + if mkdir -- "$lock" 2>/dev/null; then + HELD_LOCK="$lock" + printf 'host=%s\npid=%s\nepoch=%s\n' "$(uname -n)" "$$" "$(date +%s)" > "$lock/owner" + return 0 + fi + [[ -d "$lock" ]] || die "Cannot create lock $lock" + die "APT directory $APT_DIR is locked ($lock): $(tr '\n' ' ' < "$lock/owner" 2>/dev/null) +another build-apt-repo.sh run owns it; use a different --apt-dir or --force-unlock if stale." +} +acquire_apt_lock + +if [[ -n "$GENESIS_RELEASE" ]]; then + [[ -d "$GENESIS_RELEASE" ]] || die "Genesis release directory not found: $GENESIS_RELEASE" + GENESIS_RELEASE="$(cd "$GENESIS_RELEASE" && pwd)" + GENESIS_VERIFIER="$SCRIPT_DIR/genesis-openembedded/verify-release" + [[ -x "$GENESIS_VERIFIER" ]] || die "Genesis release verifier not found: $GENESIS_VERIFIER" +fi + # Default to all known versions when no DIST arg was given; otherwise validate # each requested version against CODENAME_MAP. if [[ ${#SELECTED_VERS[@]} -eq 0 ]]; then @@ -98,6 +150,18 @@ command -v apt-ftparchive >/dev/null 2>&1 \ || die "apt-ftparchive not found. Install: sudo apt-get install apt-utils" command -v gpg >/dev/null 2>&1 \ || die "gpg not found. Install: sudo apt-get install gnupg" +if [[ -n "$GENESIS_RELEASE" ]]; then + command -v dpkg-deb >/dev/null 2>&1 \ + || die "dpkg-deb not found. Install: sudo apt-get install dpkg" + command -v cmp >/dev/null 2>&1 \ + || die "cmp not found. Install: sudo apt-get install diffutils" + GENESIS_CHECKSUMS=$(mktemp "${TMPDIR:-/tmp}/xcat-genesis-checksums.XXXXXX") + cp -- "$GENESIS_RELEASE/SHA256SUMS" "$GENESIS_CHECKSUMS" + "$GENESIS_VERIFIER" --complete --format deb "$GENESIS_RELEASE" + cmp -s "$GENESIS_CHECKSUMS" "$GENESIS_RELEASE/SHA256SUMS" \ + || die "Genesis release changed during verification" + echo "Genesis release: $GENESIS_RELEASE" +fi if [[ $SKIP_SIGN -eq 0 ]]; then if ! gpg --list-secret-keys "$GPG_KEY_ID" >/dev/null 2>&1; then @@ -144,6 +208,38 @@ done step "Populating pool" +copy_deb() { + local source="$1" + local destination + destination="$2/$(basename "$source")" + if [[ -e "$destination" ]]; then + cmp -s "$source" "$destination" \ + || die "Package collision with different content: $destination" + return + fi + ln "$source" "$destination" 2>/dev/null || cp "$source" "$destination" +} + +copy_genesis_deb() { + local source="$1" + local directory="$2" + local name destination relative + name=$(basename "$source") + destination="$directory/$name" + relative="deb/$name" + # Every selected suite receives the whole release, so a plain copy spends hundreds of + # megabytes per suite. --reflink=auto lets a filesystem that can share extents + # copy-on-write avoid that, while still giving the pool a file of its own: a link would + # leave the published package and the verified release sharing one inode, where a write + # through either path changes what the other holds. + cp --reflink=auto -- "$source" "$destination" 2>/dev/null \ + || cp -- "$source" "$destination" + "$GENESIS_VERIFIER" \ + --checksum-file "$GENESIS_CHECKSUMS" \ + --relative-file "$relative" \ + --copied-file "$destination" +} + for ver in "${SELECTED_VERS[@]}"; do codename="${CODENAME_MAP[$ver]}" src="$APT_DIR/$ver" @@ -151,11 +247,38 @@ for ver in "${SELECTED_VERS[@]}"; do echo "$ver -> pool/main/$codename/" if [[ $DRY_RUN -eq 0 ]]; then for deb in "$src"/*.deb; do - ln "$deb" "$dst/" 2>/dev/null || cp "$deb" "$dst/" + if [[ -n "$GENESIS_RELEASE" && ${deb##*/} == xcat-genesis-openembedded-*.deb ]]; then + continue + fi + copy_deb "$deb" "$dst" done + if [[ -n "$GENESIS_RELEASE" ]]; then + for deb in "$GENESIS_RELEASE"/deb/*.deb; do + copy_genesis_deb "$deb" "$dst" + done + fi fi done +# The pool is indexed and the metadata is signed from what is on disk now, not from what +# was copied earlier, so check the packages again here: anything that changed between the +# copy and this point would otherwise be published and signed as verified. +if [[ -n "$GENESIS_RELEASE" && $DRY_RUN -eq 0 ]]; then + step "Re-verifying pooled Genesis packages" + for ver in "${SELECTED_VERS[@]}"; do + codename="${CODENAME_MAP[$ver]}" + for deb in "$GENESIS_RELEASE"/deb/*.deb; do + name=$(basename "$deb") + pooled="$APT_DIR/pool/main/$codename/$name" + "$GENESIS_VERIFIER" \ + --checksum-file "$GENESIS_CHECKSUMS" \ + --relative-file "deb/$name" \ + --copied-file "$pooled" + echo "Re-verified pooled Genesis package: $pooled" + done + done +fi + step "Generating Packages indexes" for ver in "${SELECTED_VERS[@]}"; do diff --git a/genesis-openembedded/README.md b/genesis-openembedded/README.md new file mode 100644 index 0000000..ffdd194 --- /dev/null +++ b/genesis-openembedded/README.md @@ -0,0 +1,92 @@ +# OpenEmbedded Genesis packages + +xCAT builds the Genesis image in `xcat-core`. This directory turns those build +outputs into packages and feeds them into the xcat-dep repositories. + +Run the release builder on a host that can build the OpenEmbedded layer. The +xcat-core checkout must be clean. `--xcat-ref` checks that the checkout points +to the intended commit; it does not change the checkout for you. + +The packaging scripts use `File::Slurper` and `IPC::Cmd`. Install +`libfile-slurper-perl` on Ubuntu. On EL, install `perl-File-Slurper` and +`perl-IPC-Cmd` from EPEL and AppStream. + +```bash +./genesis-openembedded/build \ + --xcat-source /path/to/xcat-core \ + --xcat-ref \ + --all \ + --work-dir /path/to/oe-work \ + --output-dir /path/to/xcat-genesis-release +``` + +The default format is `all`, which produces RPM, SRPM, and DEB packages. Use +`--format rpm` or `--format deb` when only one package family is needed. The +supported image architectures are `x86`, `x86_64`, `ppc64`, `ppc64le`, +`armv7hf`, `aarch64`, and `riscv64`. + +Use `--architecture` for development builds. Repository publication requires a +complete release built with `--all`. + +Each package installs one exact-architecture export under +`/opt/xcat/share/xcat/netboot/genesis-openembedded//`. The package +and install namespaces are separate from the old Genesis packages, so both +generations can be published and installed without replacing one another. The +packages are `noarch` or `all` because they are installed on the management +node, not run on the target node. + +`--work-dir` keeps the OpenEmbedded downloads and build state between release +builds. Without it, the builder uses a temporary directory and removes it when +the command finishes. + +The release directory contains: + +- `rpm/`, `srpm/`, and `deb/` package directories +- `release.manifest`, including the xcat-core commit +- `SHA256SUMS` + +Validate the directory before publishing it: + +```bash +./genesis-openembedded/verify-release --complete /path/to/xcat-genesis-release +``` + +The checksum file detects incomplete or changed output. It does not authenticate +the release, so only accept a directory produced by a trusted build host. + +`verify-release` also checks the identity of each package, including a fixed +build host and a build time taken from the source epoch. Those are reproduced +by rpm 4.14.3, 4.16.1.3, 4.19.1.1 and 6.0.2, so an EL8 or later builder -- and +a current Fedora one -- produces a release the verifier accepts. + +Pass that same directory to the repository builders: + +```bash +perl ./mockbuild-all.pl \ + --genesis-release /path/to/xcat-genesis-release \ + [other build options] + +./build-apt-repo.sh \ + --genesis-release /path/to/xcat-genesis-release \ + [DIST ...] +``` + +The RPM builder keeps its old per-EL Genesis build and adds the new packages. +The APT builder does the same for each selected suite. Both consumers require +all seven architectures and verify package identities and checksums before +collecting packages. Every management-node repository receives every target +image so it can provision nodes of another architecture. + +Without `--genesis-release`, both builders keep their existing behavior. The +new packages do not provide, replace, or obsolete the old package names. A +separate xcat-core change will select the OpenEmbedded package and install +namespace after the repositories carry it. Generated images and packages +belong in release storage, not in Git. + +Run the package tests on a Linux builder with RPM, DEB, and repository tools: + +```bash +prove t/build_utils.t +prove -It/lib t/genesis_openembedded_release.t +sudo -E prove -It/lib t/genesis_openembedded_consumer.t +``` diff --git a/genesis-openembedded/build b/genesis-openembedded/build new file mode 100755 index 0000000..41fa3ba --- /dev/null +++ b/genesis-openembedded/build @@ -0,0 +1,205 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use Cwd qw(abs_path); +use File::Basename qw(basename dirname); +use File::Copy qw(copy); +use File::Path qw(make_path); +use File::Temp qw(tempdir); +use FindBin; +use Getopt::Long qw(GetOptions); +use POSIX qw(strftime); +use lib "$FindBin::Bin/../lib"; +use XCAT::BuildUtils qw( + capture_command + digest_manifest + read_first_line + relative_files + run_command + write_binary +); +use XCAT::GenesisRelease qw( + architectures + deb_package_name + rpm_package_name + validate_architecture + validate_release +); + +my $repo_root = abs_path("$FindBin::Bin/.."); +my $xcat_source = "$repo_root/../xcat-core"; +my $xcat_ref = ''; +my $output_dir = ''; +my $work_dir = ''; +my $format = 'all'; +my $all = 0; +my @requested_architectures; + +GetOptions( + 'xcat-source=s' => \$xcat_source, + 'xcat-ref=s' => \$xcat_ref, + 'output-dir=s' => \$output_dir, + 'work-dir=s' => \$work_dir, + 'format=s' => \$format, + 'architecture=s@' => \@requested_architectures, + 'all!' => \$all, +) or die usage(); + +die "Unsupported package format: $format\n" + unless $format eq 'all' || $format eq 'rpm' || $format eq 'deb'; +die "Use either --all or --architecture\n" if $all && @requested_architectures; +@requested_architectures = architectures() if $all; +@requested_architectures = ('x86_64') unless @requested_architectures; + +my %seen; +@requested_architectures = grep { !$seen{$_}++ } @requested_architectures; +validate_architecture($_) for @requested_architectures; +my %requested = map { $_ => 1 } @requested_architectures; +@requested_architectures = grep { $requested{$_} } architectures(); +umask(0022); + +$xcat_source = abs_path($xcat_source) or die "Cannot resolve xcat-core source\n"; +for my $path (qw(Version xCAT-genesis-builder/oe/build xCAT-genesis-builder/oe/export)) { + die "xcat-core source is missing $path\n" unless -f "$xcat_source/$path"; +} +die "xcat-core checkout is not clean\n" + if capture_command('git', '-C', $xcat_source, 'status', '--porcelain') ne ''; + +my $revision = capture_command('git', '-C', $xcat_source, 'rev-parse', 'HEAD'); +die "Invalid xcat-core revision: $revision\n" unless $revision =~ /^[0-9a-f]{40}$/; +if ($xcat_ref ne '') { + die "Invalid xcat-core ref: $xcat_ref\n" unless $xcat_ref =~ /^[A-Za-z0-9][A-Za-z0-9._\/-]*$/; + my $expected = capture_command( + 'git', '-C', $xcat_source, 'rev-parse', '--verify', "$xcat_ref^{commit}", + ); + die "xcat-core HEAD $revision does not match $xcat_ref ($expected)\n" + unless $revision eq $expected; +} + +my $version = read_first_line("$xcat_source/Version"); +die "Invalid xCAT version: $version\n" unless $version =~ /^\d+(?:\.\d+){1,3}$/; +my $source_date_epoch = capture_command( + 'git', '-C', $xcat_source, 'show', '-s', '--format=%ct', 'HEAD', +); +die "Invalid xcat-core commit time\n" unless $source_date_epoch =~ /^\d+$/; +my $release = strftime('snap%Y%m%d%H%M', gmtime($source_date_epoch)); + +if ($output_dir eq '') { + $output_dir = "$repo_root/build-output/genesis-openembedded/$version-$release"; +} +my $output_parent = dirname($output_dir); +make_path($output_parent) unless -d $output_parent; +$output_parent = abs_path($output_parent) or die "Cannot resolve output parent\n"; +$output_dir = "$output_parent/" . basename($output_dir); +die "Output already exists: $output_dir\n" if -e $output_dir || -l $output_dir; + +my $work = tempdir('xcat-genesis-release.XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $staging = tempdir('.xcat-genesis-release.XXXXXX', DIR => $output_parent, CLEANUP => 1); +my $oe_work; +if ($work_dir ne '') { + make_path($work_dir) unless -d $work_dir; + $work_dir = abs_path($work_dir) or die "Cannot resolve work directory\n"; + $oe_work = "$work_dir/openembedded"; +} else { + $oe_work = "$work/openembedded"; +} +my $oe_tmp = "$oe_work/build/tmp"; +local $ENV{XCAT_GENESIS_WORK_DIR} = $oe_work; +{ + local $ENV{TMPDIR} = $oe_tmp; + make_path($oe_tmp); + run_command("$xcat_source/xCAT-genesis-builder/oe/build", @requested_architectures); +} +my $effective_deploy = "$oe_tmp/deploy"; +die "Invalid OpenEmbedded deploy directory: $effective_deploy\n" + unless -d $effective_deploy && !-l $effective_deploy; + +for my $architecture (@requested_architectures) { + my $export = "$work/exports/$architecture"; + make_path(dirname($export)); + run_command( + "$xcat_source/xCAT-genesis-builder/oe/export", + $architecture, $effective_deploy, $export, + ); + + my $packages = "$work/packages/$architecture"; + run_command( + "$FindBin::Bin/package", + '--architecture', $architecture, + '--export-dir', $export, + '--output-dir', $packages, + '--version', $version, + '--release', $release, + '--revision', $revision, + '--source-date-epoch', $source_date_epoch, + '--format', $format, + ); + _collect_packages($packages, $staging, $architecture); +} + +my @formats = $format eq 'all' ? qw(deb rpm) : ($format); +write_binary( + "$staging/release.manifest", + "format=xcat-genesis-packages\n" + . "version=1\n" + . "xcat_version=$version\n" + . "xcat_release=$release\n" + . "xcat_revision=$revision\n" + . "source_date_epoch=$source_date_epoch\n" + . "architectures=" . join(',', @requested_architectures) . "\n" + . "formats=" . join(',', @formats) . "\n", +); +my @release_files = grep { $_ ne 'SHA256SUMS' } relative_files($staging); +write_binary( + "$staging/SHA256SUMS", + digest_manifest($staging, 'sha256', @release_files), +); +validate_release($staging); +my @verify_args = ('--format', $format); +push(@verify_args, '--complete') if $all; +run_command("$FindBin::Bin/verify-release", @verify_args, $staging); + +chmod(0755, $staging) or die "Cannot make release directory readable: $!\n"; +rename($staging, $output_dir) or die "Cannot publish $output_dir: $!\n"; +print "Built Genesis package release in $output_dir\n"; + +sub _collect_packages { + my ($source, $destination, $architecture) = @_; + if ($format eq 'all' || $format eq 'rpm') { + my $name = rpm_package_name($architecture); + _collect_one( + "$source/rpm/$name-$version-$release.noarch.rpm", + "$destination/rpm", + ); + _collect_one( + "$source/srpm/$name-$version-$release.src.rpm", + "$destination/srpm", + ); + } + if ($format eq 'all' || $format eq 'deb') { + my $name = deb_package_name($architecture); + _collect_one( + "$source/deb/${name}_${version}-${release}_all.deb", + "$destination/deb", + ); + } +} + +sub _collect_one { + my ($source, $directory) = @_; + die "Missing package artifact: $source\n" unless -f $source && !-l $source; + make_path($directory) unless -d $directory; + my $destination = "$directory/" . basename($source); + die "Duplicate package artifact: $destination\n" if -e $destination || -l $destination; + copy($source, $destination) or die "Cannot collect $source: $!\n"; +} + +sub usage { + return <<'USAGE'; +Usage: build [--xcat-source DIR] [--xcat-ref REF] [--output-dir DIR] + [--work-dir DIR] + [--format all|rpm|deb] [--architecture ARCH ... | --all] +USAGE +} diff --git a/genesis-openembedded/package b/genesis-openembedded/package new file mode 100755 index 0000000..937e380 --- /dev/null +++ b/genesis-openembedded/package @@ -0,0 +1,246 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use Cwd qw(abs_path); +use File::Basename qw(basename dirname); +use File::Copy qw(copy); +use File::Path qw(make_path remove_tree); +use File::Temp qw(tempdir); +use FindBin; +use Getopt::Long qw(GetOptions); +use lib "$FindBin::Bin/../lib"; +use XCAT::BuildUtils qw( + capture_command + digest_manifest + relative_files + require_command + run_command + write_binary +); +use XCAT::GenesisRelease qw( + deb_package_name + rpm_package_name + validate_architecture + validate_export +); + +my ($architecture, $export_dir, $output_dir, $version, $release, $revision); +my $source_date_epoch; +my $format = 'all'; +my $rpm_top; + +END { + eval { remove_tree($rpm_top) } + if defined($rpm_top) && -d $rpm_top && !-l $rpm_top; +} + +GetOptions( + 'architecture=s' => \$architecture, + 'export-dir=s' => \$export_dir, + 'output-dir=s' => \$output_dir, + 'version=s' => \$version, + 'release=s' => \$release, + 'revision=s' => \$revision, + 'source-date-epoch=i' => \$source_date_epoch, + 'format=s' => \$format, +) or die usage(); + +for my $required ( + [ architecture => $architecture ], + [ 'export-dir' => $export_dir ], + [ 'output-dir' => $output_dir ], + [ version => $version ], + [ release => $release ], + [ revision => $revision ], + [ 'source-date-epoch' => $source_date_epoch ], + ) +{ + die "Missing --$required->[0]\n" unless defined($required->[1]) && $required->[1] ne ''; +} + +validate_architecture($architecture); +die "Invalid version: $version\n" unless $version =~ /^\d+(?:\.\d+){1,3}$/; +die "Invalid release: $release\n" unless $release =~ /^[A-Za-z0-9][A-Za-z0-9.+~]*$/; +die "Invalid xcat-core revision: $revision\n" unless $revision =~ /^[0-9a-f]{40}$/; +die "Invalid source date epoch\n" unless $source_date_epoch =~ /^\d+$/; +die "Unsupported package format: $format\n" + unless $format eq 'all' || $format eq 'rpm' || $format eq 'deb'; +umask(0022); + +$export_dir = abs_path($export_dir) or die "Cannot resolve export directory\n"; +validate_export($export_dir, $architecture); + +my $output_parent = dirname($output_dir); +make_path($output_parent) unless -d $output_parent; +$output_parent = abs_path($output_parent) or die "Cannot resolve output parent\n"; +$output_dir = "$output_parent/" . basename($output_dir); +die "Output already exists: $output_dir\n" if -e $output_dir || -l $output_dir; +chdir($output_parent) or die "Cannot use output parent $output_parent: $!\n"; + +my $work = tempdir('xcat-genesis-package.XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $staging = tempdir('.xcat-genesis-output.XXXXXX', DIR => $output_parent, CLEANUP => 1); +my $rpm_name = rpm_package_name($architecture); +my $deb_name = deb_package_name($architecture); +my $source_name = "$rpm_name-$version"; +my $source_root = "$work/$source_name"; +make_path("$source_root/image"); + +for my $entry (_flat_files($export_dir)) { + copy("$export_dir/$entry", "$source_root/image/$entry") + or die "Cannot copy export file $entry: $!\n"; +} +write_binary("$source_root/xcat-core-revision", "$revision\n"); + +local $ENV{SOURCE_DATE_EPOCH} = $source_date_epoch; + +if ($format eq 'all' || $format eq 'rpm') { + require_command('rpmbuild'); + _build_rpm($work, $staging, $source_root, $source_name, $rpm_name); +} +if ($format eq 'all' || $format eq 'deb') { + require_command('dpkg-deb'); + _build_deb($work, $staging, $source_root, $deb_name); +} + +chmod(0755, $staging) or die "Cannot set output mode: $!\n"; +rename($staging, $output_dir) or die "Cannot publish $output_dir: $!\n"; +print "Packaged Genesis $architecture in $output_dir\n"; + +sub _build_rpm { + my ($work, $staging, $source_root, $source_name, $package_name) = @_; + _require_gnu_tar(); + require_command('gzip'); + $rpm_top = tempdir('xcat-genesis-rpmbuild.XXXXXX', DIR => '/var/tmp', CLEANUP => 0); + make_path(map { "$rpm_top/$_" } qw(BUILD BUILDROOT RPMS SOURCES SPECS SRPMS)); + make_path("$staging/rpm", "$staging/srpm"); + + my $archive = "$rpm_top/SOURCES/$source_name.tar.gz"; + run_command( + 'tar', '--sort=name', '--owner=0', '--group=0', '--numeric-owner', + "--mtime=\@$source_date_epoch", '--use-compress-program=gzip -n', + '-cf', $archive, '-C', dirname($source_root), basename($source_root), + ); + my $spec = "$FindBin::Bin/rpm/xCAT-genesis-openembedded.spec"; + copy($spec, "$rpm_top/SPECS/xCAT-genesis-openembedded.spec") + or die "Cannot stage RPM spec: $!\n"; + run_command( + 'rpmbuild', '-ba', '--nodeps', + '--define', "_topdir $rpm_top", + '--define', "genesis_arch $architecture", + '--define', "version $version", + '--define', "release $release", + # SOURCE_DATE_EPOCH is exported above; rpm turns it into the build time only + # when asked, and takes the build host from _buildhost. Both are honoured from + # rpm 4.14 through 6, which is what makes the identity verify-release expects + # reproducible on every builder. + '--define', '_buildhost xcat-release', + '--define', 'use_source_date_epoch_as_buildtime 1', + '--define', 'clamp_mtime_to_source_date_epoch 1', + '--define', '_binary_payload w9.gzdio', + '--define', '_source_payload w9.gzdio', + "$rpm_top/SPECS/xCAT-genesis-openembedded.spec", + ); + + my $binary = "$rpm_top/RPMS/noarch/$package_name-$version-$release.noarch.rpm"; + my $source = "$rpm_top/SRPMS/$package_name-$version-$release.src.rpm"; + die "RPM build did not produce $binary\n" unless -f $binary; + die "RPM build did not produce $source\n" unless -f $source; + copy($binary, "$staging/rpm/" . basename($binary)) or die "Cannot collect RPM: $!\n"; + copy($source, "$staging/srpm/" . basename($source)) or die "Cannot collect SRPM: $!\n"; + remove_tree($rpm_top); + undef($rpm_top); +} + +sub _build_deb { + my ($work, $staging, $source_root, $package_name) = @_; + my $root = "$work/deb-root"; + my $image = "$root/opt/xcat/share/xcat/netboot/genesis-openembedded/$architecture"; + my $doc = "$root/usr/share/doc/$package_name"; + make_path("$root/DEBIAN", $image, $doc, "$staging/deb"); + + for my $entry (_flat_files("$source_root/image")) { + copy("$source_root/image/$entry", "$image/$entry") + or die "Cannot stage DEB image file $entry: $!\n"; + chmod(0644, "$image/$entry") or die "Cannot set mode on $entry: $!\n"; + } + copy("$source_root/xcat-core-revision", "$doc/xcat-core-revision") + or die "Cannot stage DEB revision: $!\n"; + chmod(0644, "$doc/xcat-core-revision") or die "Cannot set revision mode: $!\n"; + + my $installed_kib = _tree_bytes($root); + $installed_kib = int(($installed_kib + 1023) / 1024); + write_binary( + "$root/DEBIAN/control", + "Package: $package_name\n" + . "Version: $version-$release\n" + . "Section: admin\nPriority: optional\nArchitecture: all\n" + . "Installed-Size: $installed_kib\n" + . "Maintainer: xCAT \n" + . "Description: xCAT Genesis netboot image\n" + . " The OpenEmbedded Genesis image used for node discovery, inventory,\n" + . " and service actions.\n", + ); + _write_deb_md5sums($root); + _normalize_mtime($root, $source_date_epoch); + + my $deb = "$staging/deb/${package_name}_${version}-${release}_all.deb"; + run_command('dpkg-deb', '--root-owner-group', '-Zgzip', '-z9', '--build', $root, $deb); + die "DEB build did not produce $deb\n" unless -f $deb; +} + +sub _write_deb_md5sums { + my ($root) = @_; + my @files = grep { $_ !~ m{\ADEBIAN/} } relative_files($root); + write_binary( + "$root/DEBIAN/md5sums", + digest_manifest($root, 'md5', @files), + ); +} + +sub _normalize_mtime { + my ($root, $epoch) = @_; + require File::Find; + File::Find::find( + { + no_chdir => 1, + wanted => sub { + utime($epoch, $epoch, $File::Find::name) + or die "Cannot set timestamp on $File::Find::name: $!\n"; + }, + }, + $root, + ); +} + +sub _flat_files { + my ($directory) = @_; + my @entries = relative_files($directory); + for my $entry (@entries) { + die "Expected a flat directory: $directory\n" if $entry =~ m{/}; + } + return @entries; +} + +sub _tree_bytes { + my ($directory) = @_; + my $bytes = 0; + $bytes += -s "$directory/$_" for relative_files($directory); + return $bytes; +} + +sub _require_gnu_tar { + require_command('tar'); + my $version_line = capture_command('tar', '--version'); + die "GNU tar is required to build Genesis source packages\n" + unless $version_line =~ /^tar \(GNU tar\)/; +} + +sub usage { + return <<'USAGE'; +Usage: package --architecture ARCH --export-dir DIR --output-dir DIR + --version VERSION --release RELEASE --revision COMMIT + --source-date-epoch EPOCH [--format all|rpm|deb] +USAGE +} diff --git a/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec b/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec new file mode 100644 index 0000000..965203c --- /dev/null +++ b/genesis-openembedded/rpm/xCAT-genesis-openembedded.spec @@ -0,0 +1,43 @@ +%global genesis_arch %{?genesis_arch}%{!?genesis_arch:x86_64} + +Name: xCAT-genesis-openembedded-%{genesis_arch} +Version: %{?version}%{!?version:0} +Release: %{?release}%{!?release:1} +Summary: xCAT OpenEmbedded Genesis netboot image +License: Various +URL: https://xcat.org/ +Source0: %{name}-%{version}.tar.gz +BuildArch: noarch +AutoReqProv: no +BuildRequires: gzip +BuildRequires: tar + +%description +The OpenEmbedded Genesis image used by xCAT for discovery, inventory, and +service actions before a node boots its installed operating system. + +%prep +rm -rf %{name}-%{version} +tar -xzf "$RPM_SOURCE_DIR/%{name}-%{version}.tar.gz" + +%build + +%install +cd %{name}-%{version} +rm -rf "$RPM_BUILD_ROOT" +install -d -m 0755 "$RPM_BUILD_ROOT/opt/xcat/share/xcat/netboot/genesis-openembedded/%{genesis_arch}" +install -m 0644 image/* "$RPM_BUILD_ROOT/opt/xcat/share/xcat/netboot/genesis-openembedded/%{genesis_arch}/" +# Keep this path stable across RPM build hosts. %%{_docdir} differs on SUSE. +install -d -m 0755 "$RPM_BUILD_ROOT/usr/share/doc/%{name}" +install -m 0644 xcat-core-revision "$RPM_BUILD_ROOT/usr/share/doc/%{name}/" + +%files +%defattr(-,root,root,-) +%dir /opt/xcat +%dir /opt/xcat/share +%dir /opt/xcat/share/xcat +%dir /opt/xcat/share/xcat/netboot +%dir /opt/xcat/share/xcat/netboot/genesis-openembedded +/opt/xcat/share/xcat/netboot/genesis-openembedded/%{genesis_arch} +%dir /usr/share/doc/%{name} +%doc /usr/share/doc/%{name}/xcat-core-revision diff --git a/genesis-openembedded/verify-release b/genesis-openembedded/verify-release new file mode 100755 index 0000000..d91e45d --- /dev/null +++ b/genesis-openembedded/verify-release @@ -0,0 +1,138 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use Cwd qw(abs_path); +use FindBin; +use Getopt::Long qw(GetOptions); +use lib "$FindBin::Bin/../lib"; +use XCAT::BuildUtils qw(capture_command require_command); +use XCAT::GenesisRelease qw( + deb_package_name + read_checksum_manifest + rpm_package_name + validate_complete_release + validate_release + verify_release_file +); + +my $format = 'all'; +my $complete = 0; +my ($checksum_file, $relative_file, $copied_file); +GetOptions( + 'format=s' => \$format, + 'complete!' => \$complete, + 'checksum-file=s' => \$checksum_file, + 'relative-file=s' => \$relative_file, + 'copied-file=s' => \$copied_file, +) or die usage(); + +my $copy_mode = grep { defined($_) } + ($checksum_file, $relative_file, $copied_file); +if ($copy_mode) { + die usage() + unless defined($checksum_file) + && defined($relative_file) + && defined($copied_file) + && !$complete + && $format eq 'all' + && !@ARGV; + $checksum_file = abs_path($checksum_file) + or die "Cannot resolve checksum file\n"; + $copied_file = abs_path($copied_file) + or die "Cannot resolve copied file\n"; + my $checksums = read_checksum_manifest($checksum_file); + verify_release_file($checksums, $relative_file, $copied_file); + print "Verified copied Genesis package: $copied_file\n"; + exit(0); +} + +die usage() unless @ARGV == 1; +die "Unsupported verification format: $format\n" + unless $format eq 'all' || $format eq 'rpm' || $format eq 'deb'; + +my $directory = abs_path($ARGV[0]) or die "Cannot resolve release directory\n"; +my $manifest = $complete + ? validate_complete_release($directory) + : validate_release($directory); +my %available = map { $_ => 1 } split(/,/, $manifest->{formats}); +my @formats = $format eq 'all' ? qw(rpm deb) : ($format); +for my $required (@formats) { + die "Release does not contain $required packages\n" unless $available{$required}; +} + +my @architectures = split(/,/, $manifest->{architectures}); +for my $architecture (@architectures) { + if (grep { $_ eq 'rpm' } @formats) { + _verify_rpm($directory, $manifest, $architecture); + } + if (grep { $_ eq 'deb' } @formats) { + _verify_deb($directory, $manifest, $architecture); + } +} + +print "Verified Genesis package release: $directory\n"; + +sub _verify_rpm { + my ($directory, $manifest, $architecture) = @_; + require_command('rpm'); + my $name = rpm_package_name($architecture); + my $source_name = "$name-$manifest->{xcat_version}-$manifest->{xcat_release}.src.rpm"; + my @packages = ( + [ "$directory/rpm/$name-$manifest->{xcat_version}-$manifest->{xcat_release}.noarch.rpm", + 'noarch', $source_name ], + [ "$directory/srpm/$source_name", 'noarch', '(none)' ], + ); + for my $package (@packages) { + my ($path, $expected_arch, $expected_source) = @{$package}; + my $metadata = capture_command( + 'rpm', '-qp', '--qf', + "%{NAME}\t%{VERSION}\t%{RELEASE}\t%{ARCH}\t%{EPOCHNUM}\t%{SOURCERPM}" + . "\t%{BUILDHOST}\t%{BUILDTIME}\n", + $path, + ); + my $expected = join( + "\t", $name, $manifest->{xcat_version}, $manifest->{xcat_release}, + $expected_arch, '0', $expected_source, 'xcat-release', + $manifest->{source_date_epoch}, + ); + die "Unexpected RPM identity for $path: $metadata\n" unless $metadata eq $expected; + for my $relationship (qw(conflicts obsoletes)) { + my $value = capture_command('rpm', '-qp', "--$relationship", $path); + die "Unexpected RPM $relationship for $path: $value\n" if $value ne ''; + } + my $provides = capture_command('rpm', '-qp', '--provides', $path); + die "Legacy RPM relationship in $path: $provides\n" + if $provides =~ /xCAT-genesis-base/; + } +} + +sub _verify_deb { + my ($directory, $manifest, $architecture) = @_; + require_command('dpkg-deb'); + my $name = deb_package_name($architecture); + my $path = "$directory/deb/${name}_$manifest->{xcat_version}-$manifest->{xcat_release}_all.deb"; + my $metadata = join( + "\n", + map { capture_command('dpkg-deb', '-f', $path, $_) } + qw(Package Version Architecture), + ); + my $expected = join( + "\n", $name, "$manifest->{xcat_version}-$manifest->{xcat_release}", + 'all', + ); + die "Unexpected DEB identity for $path: $metadata\n" unless $metadata eq $expected; + for my $field (qw(Breaks Conflicts Provides Replaces)) { + my $value = capture_command('dpkg-deb', '-f', $path, $field); + die "Unexpected DEB $field for $path: $value\n" if $value ne ''; + } +} + +sub usage { + return <<'USAGE'; +Usage: verify-release [--format all|rpm|deb] [--complete] RELEASE_DIRECTORY + verify-release --checksum-file SHA256SUMS --relative-file PATH + --copied-file FILE +USAGE +} diff --git a/lib/XCAT/BuildUtils.pm b/lib/XCAT/BuildUtils.pm new file mode 100644 index 0000000..c64a518 --- /dev/null +++ b/lib/XCAT/BuildUtils.pm @@ -0,0 +1,169 @@ +package XCAT::BuildUtils; + +use strict; +use warnings; + +use Digest::MD5 (); +use Digest::SHA (); +use Exporter qw(import); +use File::Find qw(find); +use File::Slurper qw(read_binary write_binary); +use File::Spec; +use IPC::Cmd qw(can_run); + +our @EXPORT_OK = qw( + capture_command + command_exists + digest_file + digest_manifest + display_quote + every_step_failed + hashes_equal + print_step + read_binary + read_first_line + read_lines + relative_files + require_command + run_command + shell_quote + write_binary +); + +sub command_exists { + my ($command) = @_; + return defined(can_run($command)); +} + +sub require_command { + my ($command) = @_; + return can_run($command) + // die "Required command not found: $command\n"; +} + +sub capture_command { + my (@command) = @_; + open(my $fh, '-|', @command) or die "Cannot run $command[0]: $!\n"; + local $/; + my $output = <$fh> // ''; + close($fh) or die "Command failed: $command[0]\n"; + $output =~ s/\s+\z//; + return $output; +} + +sub run_command { + my (@command) = @_; + print '+ ', join(' ', map { display_quote($_) } @command), "\n"; + my $status = system(@command); + return 1 if $status == 0; + + my $exit = $status == -1 + ? 255 + : ($status & 127) ? 128 + ($status & 127) : $status >> 8; + die "Command failed (rc=$exit): " + . join(' ', map { display_quote($_) } @command) . "\n"; +} + +sub display_quote { + my ($value) = @_; + return $value if $value =~ /^[A-Za-z0-9_.,+\/:=@~-]+$/; + return shell_quote($value); +} + +sub shell_quote { + my ($value) = @_; + $value = '' unless defined($value); + $value =~ s/'/'"'"'/g; + return "'$value'"; +} + +sub print_step { + my ($message) = @_; + print "\n== $message ==\n"; +} + +sub read_lines { + my ($path) = @_; + my $content = read_binary($path); + return () if $content eq ''; + + my @lines = split(/\n/, $content, -1); + pop(@lines) if @lines && $lines[-1] eq ''; + s/\r\z// for @lines; + return @lines; +} + +sub read_first_line { + my ($path) = @_; + my @lines = read_lines($path); + die "Empty file: $path\n" unless @lines; + return $lines[0]; +} + +sub relative_files { + my ($root) = @_; + die "Invalid directory: $root\n" unless -d $root && !-l $root; + + my $absolute = File::Spec->rel2abs($root); + my @files; + find( + { + no_chdir => 1, + wanted => sub { + my $path = $File::Find::name; + return if $path eq $absolute; + die "Symbolic links are not allowed: $path\n" if -l $path; + return if -d $path; + die "Non-regular entry: $path\n" unless -f $path; + my $relative = File::Spec->abs2rel($path, $absolute); + $relative =~ tr{\\}{/}; + push(@files, $relative); + }, + }, + $absolute, + ); + my @sorted = sort @files; + return @sorted; +} + +sub digest_file { + my ($path, $algorithm) = @_; + $algorithm //= 'sha256'; + + my $digest = $algorithm eq 'sha256' ? Digest::SHA->new(256) + : $algorithm eq 'md5' ? Digest::MD5->new + : die "Unsupported digest algorithm: $algorithm\n"; + open(my $fh, '<:raw', $path) or die "Cannot read $path: $!\n"; + my $value = $digest->addfile($fh)->hexdigest; + close($fh) or die "Cannot close $path: $!\n"; + return $value; +} + +sub digest_manifest { + my ($root, $algorithm, @files) = @_; + return join( + '', + map { digest_file("$root/$_", $algorithm) . " $_\n" } + sort @files, + ); +} + +# True when a set of steps was attempted and none of them survived. Callers tolerate +# individual failures; losing every step means the builder itself did not work. +sub every_step_failed { + my ($attempted, $failures) = @_; + return 0 unless $attempted; + return $failures >= $attempted ? 1 : 0; +} + +sub hashes_equal { + my ($left, $right) = @_; + return 0 unless keys(%{$left}) == keys(%{$right}); + for my $name (keys %{$left}) { + return 0 unless exists($right->{$name}) + && $left->{$name} eq $right->{$name}; + } + return 1; +} + +1; diff --git a/lib/XCAT/GenesisRelease.pm b/lib/XCAT/GenesisRelease.pm new file mode 100644 index 0000000..b07368f --- /dev/null +++ b/lib/XCAT/GenesisRelease.pm @@ -0,0 +1,247 @@ +package XCAT::GenesisRelease; + +use strict; +use warnings; + +use Exporter qw(import); +use XCAT::BuildUtils qw(digest_file read_lines relative_files); + +our @EXPORT_OK = qw( + architectures + deb_package_name + read_checksum_manifest + read_release_manifest + rpm_package_name + validated_release_checksums + validate_architecture + validate_complete_release + validate_export + validate_release + verify_release_file +); + +my @ARCHITECTURES = qw(x86 x86_64 ppc64 ppc64le armv7hf aarch64 riscv64); +my %ARCHITECTURE = map { $_ => 1 } @ARCHITECTURES; + +sub architectures { + return @ARCHITECTURES; +} + +sub validate_architecture { + my ($architecture) = @_; + die "Unsupported Genesis architecture: $architecture\n" + unless defined($architecture) && $ARCHITECTURE{$architecture}; + return $architecture; +} + +sub rpm_package_name { + my ($architecture) = @_; + validate_architecture($architecture); + return "xCAT-genesis-openembedded-$architecture"; +} + +sub deb_package_name { + my ($architecture) = @_; + validate_architecture($architecture); + $architecture =~ tr/_/-/; + return "xcat-genesis-openembedded-$architecture"; +} + +sub _read_key_values { + my ($path, $allowed) = @_; + my %values; + for my $line (read_lines($path)) { + die "Invalid manifest entry in $path: $line\n" + unless $line =~ /^([a-z][a-z0-9_]*)=([A-Za-z0-9][A-Za-z0-9.,_+~-]*)$/; + my ($key, $value) = ($1, $2); + die "Unknown manifest key in $path: $key\n" unless $allowed->{$key}; + die "Duplicate manifest key in $path: $key\n" if exists($values{$key}); + $values{$key} = $value; + } + return \%values; +} + +sub read_checksum_manifest { + my ($path) = @_; + my %checksums; + for my $line (read_lines($path)) { + die "Invalid checksum entry in $path: $line\n" + unless $line =~ /^([0-9a-f]{64}) ([A-Za-z0-9][A-Za-z0-9._\/+~-]*)$/; + my ($digest, $name) = ($1, $2); + die "Unsafe checksum path in $path: $name\n" + if $name =~ m{(?:\A|/)\.\.(?:/|\z)} || $name =~ m{//}; + die "Duplicate checksum entry in $path: $name\n" + if exists($checksums{$name}); + $checksums{$name} = $digest; + } + return \%checksums; +} + +sub _verify_checksums { + my ($root) = @_; + my @files = grep { $_ ne 'SHA256SUMS' } relative_files($root); + my $checksums = read_checksum_manifest("$root/SHA256SUMS"); + + my %files = map { $_ => 1 } @files; + for my $name (@files) { + die "Missing checksum for $name\n" unless exists($checksums->{$name}); + my $digest = digest_file("$root/$name", 'sha256'); + die "Checksum mismatch for $name\n" unless $digest eq $checksums->{$name}; + } + for my $name (keys %{$checksums}) { + die "Checksum names a missing file: $name\n" unless $files{$name}; + } + return $checksums; +} + +sub validate_export { + my ($directory, $architecture) = @_; + validate_architecture($architecture); + die "Invalid Genesis export: $directory\n" unless -d $directory && !-l $directory; + + my %required = map { $_ => 1 } qw( + SHA256SUMS + image.manifest + image.spdx.json + image.vex.json + initramfs.cpio.gz + kernel + license.manifest + xcat-genesis.manifest + ); + $required{'fw_jump.elf'} = 1 if $architecture eq 'riscv64'; + + my @files = relative_files($directory); + my %files = map { $_ => 1 } @files; + for my $name (sort keys %required) { + die "Genesis export is missing $name\n" unless $files{$name}; + } + for my $name (@files) { + die "Unexpected Genesis export file: $name\n" unless $required{$name}; + } + + my $manifest = _read_key_values( + "$directory/xcat-genesis.manifest", + { map { $_ => 1 } qw(format version architecture) }, + ); + die "Unsupported Genesis export format\n" + unless ($manifest->{format} // '') eq 'xcat-genesis'; + die "Unsupported Genesis export version\n" + unless ($manifest->{version} // '') eq '1'; + die "Genesis export architecture mismatch\n" + unless ($manifest->{architecture} // '') eq $architecture; + + _verify_checksums($directory); + return 1; +} + +sub read_release_manifest { + my ($directory) = @_; + my $values = _read_key_values( + "$directory/release.manifest", + { + map { $_ => 1 } qw( + format version xcat_version xcat_release xcat_revision + source_date_epoch architectures formats + ) + }, + ); + for my $key (qw(format version xcat_version xcat_release xcat_revision source_date_epoch architectures formats)) { + die "Release manifest is missing $key\n" unless exists($values->{$key}); + } + return $values; +} + +sub _validate_release { + my ($directory) = @_; + die "Invalid Genesis package release: $directory\n" + unless -d $directory && !-l $directory; + + my $manifest = read_release_manifest($directory); + die "Unsupported Genesis package release format\n" + unless $manifest->{format} eq 'xcat-genesis-packages'; + die "Unsupported Genesis package release version\n" + unless $manifest->{version} eq '1'; + die "Invalid xCAT version in release manifest\n" + unless $manifest->{xcat_version} =~ /^\d+(?:\.\d+){1,3}$/; + die "Invalid xCAT release in release manifest\n" + unless $manifest->{xcat_release} =~ /^[A-Za-z0-9][A-Za-z0-9.+~]*$/; + die "Invalid xCAT revision in release manifest\n" + unless $manifest->{xcat_revision} =~ /^[0-9a-f]{40}$/; + die "Invalid source epoch in release manifest\n" + unless $manifest->{source_date_epoch} =~ /^\d+$/; + + my @architectures = split(/,/, $manifest->{architectures}); + my %seen_arch; + for my $architecture (@architectures) { + validate_architecture($architecture); + die "Duplicate release architecture: $architecture\n" if $seen_arch{$architecture}++; + } + die "Release manifest has no architectures\n" unless @architectures; + + my @formats = split(/,/, $manifest->{formats}); + my %seen_format; + for my $format (@formats) { + die "Unsupported package format: $format\n" unless $format eq 'rpm' || $format eq 'deb'; + die "Duplicate package format: $format\n" if $seen_format{$format}++; + } + die "Release manifest has no package formats\n" unless @formats; + + my $checksums = _verify_checksums($directory); + + my %expected = ( + 'release.manifest' => 1, + ); + for my $architecture (@architectures) { + my $rpm = rpm_package_name($architecture); + my $deb = deb_package_name($architecture); + if ($seen_format{rpm}) { + $expected{"rpm/$rpm-$manifest->{xcat_version}-$manifest->{xcat_release}.noarch.rpm"} = 1; + $expected{"srpm/$rpm-$manifest->{xcat_version}-$manifest->{xcat_release}.src.rpm"} = 1; + } + if ($seen_format{deb}) { + $expected{"deb/${deb}_$manifest->{xcat_version}-$manifest->{xcat_release}_all.deb"} = 1; + } + } + for my $file (grep { $_ ne 'SHA256SUMS' } relative_files($directory)) { + die "Unexpected Genesis release artifact: $file\n" unless $expected{$file}; + delete($expected{$file}); + } + die "Genesis release is missing: " . join(', ', sort keys %expected) . "\n" + if %expected; + return ($manifest, $checksums); +} + +sub validate_release { + my ($directory) = @_; + my ($manifest) = _validate_release($directory); + return $manifest; +} + +sub validate_complete_release { + my ($directory) = @_; + my $manifest = validate_release($directory); + my %present = map { $_ => 1 } split(/,/, $manifest->{architectures}); + my @missing = grep { !$present{$_} } @ARCHITECTURES; + die "Genesis release is missing supported architectures: @missing\n" if @missing; + return $manifest; +} + +sub validated_release_checksums { + my ($directory) = @_; + my (undef, $checksums) = _validate_release($directory); + return $checksums; +} + +sub verify_release_file { + my ($checksums, $relative, $path) = @_; + die "Missing verified checksum for $relative\n" + unless ref($checksums) eq 'HASH' && exists($checksums->{$relative}); + die "Invalid collected release file: $path\n" unless -f $path && !-l $path; + my $digest = digest_file($path, 'sha256'); + die "Collected release file checksum mismatch: $path\n" + unless $digest eq $checksums->{$relative}; + return 1; +} + +1; diff --git a/mockbuild-all.pl b/mockbuild-all.pl index e544ef6..e38c730 100755 --- a/mockbuild-all.pl +++ b/mockbuild-all.pl @@ -7,18 +7,34 @@ use Cwd qw(abs_path cwd); use File::Basename qw(dirname basename); use File::Copy qw(copy); use File::Find qw(find); +use File::Glob qw(bsd_glob); use File::Path qw(make_path remove_tree); -use File::Temp qw(tempdir); +use File::Temp qw(tempdir tempfile); use Getopt::Long qw(GetOptions); use Parallel::ForkManager; use POSIX qw(strftime); use FindBin qw($RealBin); -use lib $RealBin; +use lib $RealBin, "$RealBin/lib"; use MockBuildUtils qw(sh_quote print_step version_matches required_pkgs read_manifest verify_repo_packages verify_repo_signature verify_rpm_signatures rpm_version rpm_release rpm_sigmd5 restamp_release_line cross_copy_genesis finalize_xcat_dep bump_dep_release_suffix build_mock_uniqueext rpmkeys_checksig_problem); +# print_step and sh_quote come from MockBuildUtils above; XCAT::BuildUtils carries the same +# print_step, so it is deliberately NOT imported here (one definition, no redefinition warning). +use XCAT::BuildUtils qw( + capture_command + every_step_failed + hashes_equal + read_lines + require_command + run_command + shell_quote +); +use XCAT::GenesisRelease qw( + validated_release_checksums + verify_release_file +); # --- Mount-namespace isolation: guard the host cgroup against mock teardown propagation ---------- # mock mounts /sys/fs/cgroup into every build chroot. On these systemd build hosts every mount is @@ -88,6 +104,8 @@ my $skip_perl = 0; my $skip_genesis = 0; my $skip_createrepo = 0; my $skip_tarball = 0; +my $genesis_release = ''; +my $genesis_release_checksums; my $scrub_all_chroots = 0; my $keep_buildroots = 0; # keep per-step mock chroots after build (default: --scrub=chroot each) my $dry_run = 0; @@ -141,6 +159,7 @@ GetOptions( 'skip-genesis!' => \$skip_genesis, 'skip-createrepo!' => \$skip_createrepo, 'skip-tarball!' => \$skip_tarball, + 'genesis-release=s' => \$genesis_release, 'scrub-all-chroots!' => \$scrub_all_chroots, 'keep-buildroots!' => \$keep_buildroots, 'collect-dir=s@' => \@extra_collect_dirs, @@ -161,7 +180,8 @@ $repo_root = abs_path($repo_root); my $SOURCE_DATE_EPOCH; $SOURCE_DATE_EPOCH = $build_timestamp if defined $build_timestamp; if (!$SOURCE_DATE_EPOCH && -f "$repo_root/Gitepoch") { - $SOURCE_DATE_EPOCH = slurp_chomp("$repo_root/Gitepoch"); + my @gitepoch = read_lines("$repo_root/Gitepoch"); + $SOURCE_DATE_EPOCH = $gitepoch[0] // ''; } unless ($SOURCE_DATE_EPOCH && $SOURCE_DATE_EPOCH =~ /^\d+$/) { $SOURCE_DATE_EPOCH = `git -C \Q$repo_root\E log -1 --format=%ct HEAD 2>/dev/null`; @@ -281,7 +301,7 @@ acquire_output_lock($output_base, $force_unlock); $xcat_src = resolve_xcat_source($xcat_src, $repo_root); -my $arch = capture('uname -m'); +my $arch = capture_command('uname', '-m'); my %os = read_os_release('/etc/os-release'); my $os_id = $os{ID} // ''; my $version_id = $os{VERSION_ID} // ''; @@ -298,6 +318,25 @@ require_command('mock') if $scrub_all_chroots; require_command('rpmsign') if $gpg_sign; require_command('gpg') if $gpg_sign; +if ($genesis_release ne '') { + $genesis_release = abs_path($genesis_release) + or die "Cannot resolve --genesis-release directory\n"; + die "Genesis release directory not found: $genesis_release\n" + unless -d $genesis_release; + my $verifier = "$script_dir/genesis-openembedded/verify-release"; + die "Genesis release verifier not found: $verifier\n" unless -x $verifier; + # Checksum, verify, checksum again. The verifier reads the tree it validates, so a + # release rewritten together with its SHA256SUMS while the verifier runs would satisfy + # both the verifier and any single pass taken afterwards; comparing the pass taken + # before with the one taken after is what closes that window. + my $checksums_before = validated_release_checksums($genesis_release); + run_command($^X, $verifier, '--complete', '--format', 'rpm', $genesis_release); + my $checksums_after = validated_release_checksums($genesis_release); + die "Genesis release changed during verification\n" + unless hashes_equal($checksums_before, $checksums_after); + $genesis_release_checksums = $checksums_before; +} + # An explicit --target builds just that target; otherwise build the current host # arch across rh8/rh9/rh10 into a deployable per-EL xcat-dep repo. This script builds # ONLY the host arch (uname -m) -- the other arch is produced on its own build host. @@ -317,6 +356,7 @@ print "lock: $output_base/.lock (held)\n"; print "gpg_sign: $gpg_sign\n"; print "gpg_key_name: $gpg_key_name\n" if $gpg_sign; print "gpg_home: " . ($gpg_home ne '' ? $gpg_home : '(default keyring)') . "\n" if $gpg_sign; +print "genesis_release: " . ($genesis_release || '(legacy builder)') . "\n"; # Build (and deploy) EL targets concurrently. Each target is fully isolated -- distinct run_id # (target-folded), mock --uniqueext, /tmp work dir, xcat_src/dist/, and deploy dir @@ -327,7 +367,7 @@ $tgt_workers = scalar(@build_targets) if $tgt_workers > scalar(@build_targets); # mock build already gets a unique --uniqueext (separate chroot), so the only limit needed is # hardware: total concurrent builds across all targets stays <= $cap (default host nproc). The # per-target build-step concurrency is therefore the cap divided across the active targets. -my $cap = $max_parallel > 0 ? $max_parallel : (capture('nproc') || 4); +my $cap = $max_parallel > 0 ? $max_parallel : (capture_command('nproc') || 4); my $per_target_builds = defined($parallel_builds) ? $parallel_builds : int($cap / $tgt_workers); $per_target_builds = 1 if $per_target_builds < 1; print "parallel_targets: " . ($parallel_targets > 0 ? $parallel_targets : "auto($tgt_workers)") . "\n"; @@ -425,6 +465,20 @@ die "Missing perl builder script: $perl_builder\n" if (!$dry_run) { make_path($build_root, $log_root, $repo_dir, $srpm_repo_dir); + # The staging repositories hold what THIS invocation produces. A reused --run-id, or a + # rerun after a failed invocation, otherwise leaves an earlier run's packages in them, + # where collection never sees them, createrepo indexes them and deploy_target publishes + # them as this run's output. + reset_staging_repo($repo_dir); + reset_staging_repo($srpm_repo_dir); + # Same for the builder results when this run is going to build them: a reused --run-id + # otherwise leaves a previous run's packages there, and a step that fails this time is + # collected from the last time it succeeded. --skip-build deliberately collects earlier + # output, from the repository-level build-output tree, and must keep what is there. + if (!$skip_build) { + remove_tree($build_root); + make_path($build_root); + } } print_step("Configuration"); @@ -458,7 +512,7 @@ my @collect_roots; if ($scrub_all_chroots) { run_step( step => "Scrub all chroots for target $target", - cmd => "mock -r " . sh_quote($target) . " --scrub=all", + cmd => "mock -r " . shell_quote($target) . " --scrub=all", log => "$log_root/scrub-all-chroots.log", ); } @@ -476,13 +530,13 @@ if (!$skip_build) { my $step_log = "$log_root/$name"; my $step_uniqueext = build_mock_uniqueext($run_id, ++$build_step_seq, $name); my $cmd = join(' ', - 'perl', sh_quote($script), - '--mock-cfg', sh_quote($target), - '--mock-uniqueext', sh_quote($step_uniqueext), - '--result-dir', sh_quote($step_result), - '--log-dir', sh_quote($step_log), + 'perl', shell_quote($script), + '--mock-cfg', shell_quote($target), + '--mock-uniqueext', shell_quote($step_uniqueext), + '--result-dir', shell_quote($step_result), + '--log-dir', shell_quote($step_log), # host-local, run-scoped work dir so /tmp doesn't collide between runs - '--work-dir', sh_quote("/tmp/mockbuild-all-$run_id/$name"), + '--work-dir', shell_quote("/tmp/mockbuild-all-$run_id/$name"), '--build-timestamp', $SOURCE_DATE_EPOCH, # goconserver generates its spec at build time (from an upstream clone), so the # in-tree spec Release bump above cannot reach it. Hand the CD suffix down so its @@ -571,11 +625,11 @@ if (!$skip_build) { my $cmd = "mkdir -p $mktree && $genesis_scrub && HOME=" . sh_quote($genesis_home) . ' ' . join(' ', 'perl', sh_quote("$xcat_src/buildrpms.pl"), '--package', 'xCAT-genesis-base', - '--target', sh_quote($target), + '--target', shell_quote($target), '--nproc', int($nproc), '--force', '--verbose', - '--xcat_dep_path', sh_quote($repo_root), + '--xcat_dep_path', shell_quote($repo_root), ); push @build_steps, { id => 'genesis', @@ -654,6 +708,11 @@ push @collect_roots, @extra_collect_dirs; @collect_roots = uniq(@collect_roots); my @srpm_collect_roots = uniq(@collect_roots); +if ($genesis_release && !$dry_run) { + remove_genesis_packages($repo_dir, 0); + remove_genesis_packages($srpm_repo_dir, 1); +} + print_step('Collect RPM artifacts'); print "collection roots:\n"; print " $_\n" for @collect_roots; @@ -664,15 +723,26 @@ my ($copied, $skipped_src, $missing_roots) = collect_rpms( dry_run => $dry_run, ); +# Assert on what this run BUILT, before the Genesis release is added: the release is +# installed from a verified directory rather than built here, so counting it first would +# let a run whose builders all failed reach createrepo and the deployable tree, and fail +# much later in the repo gate (verify_target_repo), naming missing packages instead of the +# failed builds. if (!$dry_run && $copied == 0) { die "No binary RPMs were collected. Check build logs and collection roots.\n"; } +if ($genesis_release) { + $copied += $dry_run + ? preview_genesis_release_packages('rpm', $repo_dir) + : install_genesis_release_packages('rpm', $repo_dir); +} + # Ensure the OS-dependent xCAT-genesis-base rpm (built by the genesis step above) # lands in the dep repo -- pull it individually out of the xcat-core dist tree (the # rest of that tree, the full xCAT core, is built + published by the xcat-core pipeline). if (!$skip_genesis && !$dry_run) { - for my $g (glob("$xcat_rpms_dir/xCAT-genesis-base-*.rpm")) { + for my $g (bsd_glob("$xcat_rpms_dir/xCAT-genesis-base-*.rpm")) { next if $g =~ /\.src\.rpm$/; copy($g, "$repo_dir/" . basename($g)) or die "Failed to copy genesis-base $g -> $repo_dir: $!\n"; @@ -715,6 +785,15 @@ if (!$dry_run && $copied_srpms == 0) { print "WARN: No source RPMs were collected. SRPM repo and tarball may be empty.\n"; } +if ($genesis_release) { + $copied_srpms += $dry_run + ? preview_genesis_release_packages('srpm', $srpm_repo_dir) + : install_genesis_release_packages('srpm', $srpm_repo_dir); +} + +assert_genesis_release_copied($repo_dir, $srpm_repo_dir) + if $genesis_release && !$dry_run; + if (!$skip_createrepo) { run_step( step => 'Run createrepo', @@ -732,8 +811,8 @@ if (!$skip_tarball) { my $cmd = join(' ', 'tar', '--sort=name', '--owner=0', '--group=0', "--mtime=\@$SOURCE_DATE_EPOCH", - '-C', sh_quote($run_root), - '-czf', sh_quote($tarball), + '-C', shell_quote($run_root), + '-czf', shell_quote($tarball), 'repo' ); run_step( @@ -744,8 +823,8 @@ if (!$skip_tarball) { my $srpm_cmd = join(' ', 'tar', '--sort=name', '--owner=0', '--group=0', "--mtime=\@$SOURCE_DATE_EPOCH", - '-C', sh_quote($run_root), - '-czf', sh_quote($srpm_tarball), + '-C', shell_quote($run_root), + '-czf', shell_quote($srpm_tarball), 'repo-src' ); run_step( @@ -818,10 +897,18 @@ sub deploy_target { remove_tree($stage) if -d $stage; make_path($stage); my $ok = eval { - for my $rpm (glob("$src/*.rpm")) { + for my $rpm (bsd_glob("$src/*.rpm")) { next if $rpm =~ /\.src\.rpm$/; - copy($rpm, "$stage/" . basename($rpm)) - or die "Failed to copy $rpm -> $stage: $!\n"; + publish_file($rpm, "$stage/" . basename($rpm)); + } + # --genesis-release: drop any stale OpenEmbedded Genesis rpm the collection carried over, + # then verify the ones from THIS release still match their checksums. Both run on the STAGE + # and BEFORE sign_and_index_repo, because rpmsign rewrites the rpm bytes the release + # checksums cover. + if ($genesis_release) { + my @keep = map { basename($_) } genesis_release_files('rpm'); + remove_genesis_packages($stage, 0, \@keep); + verify_genesis_release_packages('rpm', $stage); } sign_and_index_repo($stage); write_dep_repo_metadata($stage, $rel); @@ -853,23 +940,50 @@ sub deploy_target { } remove_tree($old) if -d $old; - my $n = scalar(grep { !/\.src\.rpm$/ } glob("$dest/*.rpm")); + my $n = scalar(grep { !/\.src\.rpm$/ } bsd_glob("$dest/*.rpm")); print "Deployed rh$rel/$arch: $n rpms\n"; } +sub publish_file { + my ($source, $destination) = @_; + my ($temporary_fh, $temporary) = tempfile( + '.xcat-deploy.XXXXXX', + DIR => dirname($destination), + UNLINK => 0, + ); + close($temporary_fh) or die "Cannot close deployment staging file: $!\n"; + + my $mode = (stat($source))[2]; + die "Cannot read mode from $source: $!\n" unless defined($mode); + my $published = eval { + copy($source, $temporary) + or die "Failed to stage $source -> $temporary: $!\n"; + chmod($mode & 0x0fff, $temporary) + or die "Failed to set mode on $temporary: $!\n"; + rename($temporary, $destination) + or die "Failed to publish $temporary -> $destination: $!\n"; + 1; + }; + return if $published; + + my $error = $@ || "Failed to publish $source\n"; + unlink($temporary) if -e $temporary || -l $temporary; + die $error; +} + # createrepo_c command with upstream-matching, deterministic metadata. The tool's # defaults emit primary/filelists/other as *.xml.zst plus *.sqlite.bz2 (--database), # exactly the upstream shape; --set-timestamp-to-revision pins repomd to SOURCE_DATE_EPOCH. sub createrepo_c_cmd { my ($dir) = @_; return 'createrepo_c --update --database ' - . '--revision ' . sh_quote($SOURCE_DATE_EPOCH) . ' --set-timestamp-to-revision ' - . sh_quote($dir); + . '--revision ' . shell_quote($SOURCE_DATE_EPOCH) . ' --set-timestamp-to-revision ' + . shell_quote($dir); } sub sign_and_index_repo { my ($dir) = @_; - my @rpms = grep { !/\.src\.rpm$/ } glob("$dir/*.rpm"); + my @rpms = grep { !/\.src\.rpm$/ } bsd_glob("$dir/*.rpm"); if ($gpg_sign && @rpms) { local $ENV{GNUPGHOME} = $gpg_home if $gpg_home; run_simple("rpmsign --define " . sh_quote("%_gpg_name $gpg_key_name") . " --addsign " @@ -1014,9 +1128,11 @@ Options: --skip-build Skip all build steps and only collect/create repo/tarballs --skip-xcat-dep Skip xcat-dep mockbuild.pl package steps --skip-perl Skip perl package build step - --skip-genesis Skip the xCAT-genesis-base build step + --skip-genesis Skip the existing per-EL Genesis image build --skip-createrepo Skip createrepo --skip-tarball Skip binary/SRPM tarball creation + --genesis-release PATH Add a verified OpenEmbedded Genesis RPM release alongside the + existing per-EL Genesis packages --scrub-all-chroots Run mock -r --scrub=all before build/collect --collect-dir PATH Additional directory to scan recursively for RPMs (repeatable) --dry-run Print planned commands without executing @@ -1033,12 +1149,6 @@ Notes: USAGE } - -sub require_command { - my ($cmd) = @_; - run_simple("command -v " . sh_quote($cmd) . " >/dev/null 2>&1"); -} - sub run_simple { my ($cmd) = @_; my $rc = system($cmd); @@ -1048,18 +1158,6 @@ sub run_simple { } } -sub capture { - my ($cmd) = @_; - my $out = `$cmd`; - my $rc = $?; - if ($rc != 0) { - my $exit = $rc == -1 ? 255 : ($rc >> 8); - die "Command failed (rc=$exit): $cmd\n$out\n"; - } - chomp $out; - return $out; -} - sub run_step { my (%args) = @_; my $step = $args{step} // 'Run command'; @@ -1080,12 +1178,12 @@ sub run_step { my $full_cmd = $cmd; if ($cwd) { - $full_cmd = "cd " . sh_quote($cwd) . " && $cmd"; + $full_cmd = "cd " . shell_quote($cwd) . " && $cmd"; } if ($log) { my $log_dir = dirname($log); make_path($log_dir) if !-d $log_dir; - $full_cmd .= " > " . sh_quote($log) . " 2>&1"; + $full_cmd .= " > " . shell_quote($log) . " 2>&1"; } my $rc = system($full_cmd); @@ -1140,6 +1238,7 @@ sub run_build_steps_parallel { warn "ERROR: build step failed: $step->{step}\n" . ($@ // ''); push @failed, (defined($step->{id}) && $step->{id} ne '' ? $step->{id} : $step->{step}); } + assert_build_progress(scalar(@{$steps}), scalar(@failed)); return @failed; } @@ -1197,9 +1296,22 @@ sub run_build_steps_parallel { } warn "ERROR: build step(s) failed:\n " . join("\n ", @lines) . "\n"; } + + assert_build_progress(scalar(@{$steps}), scalar(keys %failed)); return sort keys %failed; } +# The caller enforces zero tolerance per required package (verify_target_repo). ALL steps +# failing is a different thing: the builder itself is unusable (no mock, a broken chroot, no +# network), this invocation produced nothing, and every package the run would go on to publish +# would come from somewhere other than this build -- say that, instead of naming the missing +# packages later. +sub assert_build_progress { + my ($attempted, $failures) = @_; + return unless every_step_failed($attempted, $failures); + die "FATAL: every build step failed ($failures/$attempted). Check the build logs.\n"; +} + @@ -1449,6 +1561,99 @@ sub derive_target_from_repo_path { return undef; } +sub reset_staging_repo { + my ($directory) = @_; + return unless -d $directory; + opendir(my $dh, $directory) or die "Cannot read $directory: $!\n"; + my @stale = grep { /\.rpm\z/ } readdir($dh); + closedir($dh) or die "Cannot close $directory: $!\n"; + return unless @stale; + print "Removing " . scalar(@stale) . " package(s) left in $directory by an earlier run\n"; + for my $name (@stale) { + unlink("$directory/$name") + or die "Cannot remove stale package $directory/$name: $!\n"; + } + return; +} + +sub remove_genesis_packages { + my ($directory, $source, $keep_names) = @_; + return unless -d $directory; + my %keep = map { $_ => 1 } @{ $keep_names // [] }; + opendir(my $dh, $directory) or die "Cannot read $directory: $!\n"; + my @names = grep { /^xCAT-genesis-openembedded-.*\.rpm\z/ } readdir($dh); + closedir($dh) or die "Cannot close $directory: $!\n"; + for my $name (@names) { + next if $keep{$name}; + my $path = "$directory/$name"; + next if $source && $path !~ /\.src\.rpm\z/; + next if !$source && $path =~ /\.src\.rpm\z/; + unlink($path) or die "Cannot remove stale Genesis package $path: $!\n"; + } +} + +sub install_genesis_release_packages { + my ($prefix, $destination_root) = @_; + remove_genesis_packages($destination_root, $prefix eq 'srpm'); + + my $copied = 0; + for my $relative (genesis_release_files($prefix)) { + my $source = "$genesis_release/$relative"; + my $destination = "$destination_root/" . basename($relative); + copy($source, $destination) + or die "Cannot install Genesis release package $source: $!\n"; + verify_release_file($genesis_release_checksums, $relative, $destination); + $copied++; + } + die "Genesis release has no $prefix packages\n" unless $copied; + return $copied; +} + +# Dry runs copy nothing, but they must still report what a real run would publish: +# collect_rpms and collect_srpms drop every xCAT-genesis-openembedded package whenever +# --genesis-release is given, so without this preview a dry run describes a repository +# with no Genesis packages at all while the real run installs the whole set. +sub preview_genesis_release_packages { + my ($prefix, $destination_root) = @_; + my @files = genesis_release_files($prefix); + die "Genesis release has no $prefix packages\n" unless @files; + for my $relative (@files) { + print "DRY-RUN install Genesis release package: $genesis_release/$relative" + . " -> $destination_root/" . basename($relative) . "\n"; + } + return scalar(@files); +} + +sub genesis_release_files { + my ($prefix) = @_; + my @files = sort grep { + /^\Q$prefix\E\/xCAT-genesis-openembedded-[^\/]+\.rpm\z/ + } keys %{$genesis_release_checksums}; + return @files; +} + +sub verify_genesis_release_packages { + my ($prefix, $destination_root) = @_; + my @files = genesis_release_files($prefix); + die "Genesis release has no $prefix packages\n" unless @files; + for my $relative (@files) { + my $destination = "$destination_root/" . basename($relative); + verify_release_file($genesis_release_checksums, $relative, $destination); + } + return scalar(@files); +} + +sub assert_genesis_release_copied { + my ($binary_directory, $source_directory) = @_; + for my $entry ( + [ 'rpm', $binary_directory ], + [ 'srpm', $source_directory ], + ) { + my ($prefix, $destination_root) = @{$entry}; + verify_genesis_release_packages($prefix, $destination_root); + } +} + sub collect_rpms { my (%args) = @_; my $roots = $args{roots} // []; @@ -1483,6 +1688,8 @@ sub collect_rpms { next; } my $base = basename($rpm); + next if $genesis_release + && $base =~ /^xCAT-genesis-openembedded-/; next if $seen{$base}++; if ($is_dry) { print "DRY-RUN copy: $rpm -> $dest/$base\n"; @@ -1532,6 +1739,8 @@ sub collect_srpms { next; } my $base = basename($rpm); + next if $genesis_release + && $base =~ /^xCAT-genesis-openembedded-/; next if $seen{$base}++; if ($is_dry) { print "DRY-RUN copy source: $rpm -> $dest/$base\n"; @@ -1600,7 +1809,7 @@ sub acquire_output_lock { if (mkdir $lock) { $HELD_LOCK = $lock; $LOCK_OWNER_PID = $$; - my $host = capture('uname -n') || 'unknown'; + my $host = capture_command('uname', '-n') || 'unknown'; if (open my $fh, '>', "$lock/owner") { print {$fh} "host=$host\npid=$$\nepoch=" . time() . "\n"; close $fh; diff --git a/t/build_utils.t b/t/build_utils.t new file mode 100644 index 0000000..4cee6d6 --- /dev/null +++ b/t/build_utils.t @@ -0,0 +1,87 @@ +use strict; +use warnings; + +use File::Path qw(make_path); +use File::Temp qw(tempdir); +use FindBin; +use Test::More; + +use lib "$FindBin::Bin/../lib"; +use XCAT::BuildUtils qw( + capture_command + command_exists + digest_file + digest_manifest + display_quote + every_step_failed + hashes_equal + read_binary + read_first_line + read_lines + relative_files + require_command + run_command + shell_quote + write_binary +); + +my $tmp = tempdir(CLEANUP => 1); +write_binary("$tmp/alpha", "abc"); +make_path("$tmp/nested"); +write_binary("$tmp/nested/lines", "first\r\nsecond\n"); + +is(read_binary("$tmp/alpha"), 'abc', 'binary files round trip'); +is_deeply( + [ read_lines("$tmp/nested/lines") ], + [ 'first', 'second' ], + 'line reader accepts CRLF and LF', +); +is(read_first_line("$tmp/nested/lines"), 'first', 'first line is returned'); +is_deeply( + [ relative_files($tmp) ], + [ 'alpha', 'nested/lines' ], + 'regular files are sorted and relative', +); + +my $sha256 = 'ba7816bf8f01cfea414140de5dae2223' + . 'b00361a396177a9cb410ff61f20015ad'; +is(digest_file("$tmp/alpha", 'sha256'), $sha256, 'SHA-256 matches a known value'); +is(digest_file("$tmp/alpha", 'md5'), '900150983cd24fb0d6963f7d28e17f72', + 'MD5 matches a known value'); +is( + digest_manifest($tmp, 'sha256', 'alpha'), + "$sha256 alpha\n", + 'digest manifest uses the release format', +); + +ok(command_exists($^X), 'current Perl interpreter is executable'); +is(capture_command($^X, '-e', 'print "captured\\n"'), 'captured', + 'command output is captured without a shell'); +ok(run_command($^X, '-e', 'exit 0'), 'successful command returns true'); + +eval { run_command($^X, '-e', 'exit 7'); 1 }; +like($@, qr/Command failed \(rc=7\)/, 'command failure reports its exit status'); +eval { require_command("xcat-missing-command-$$"); 1 }; +like($@, qr/Required command not found/, 'missing command is rejected'); +eval { digest_file("$tmp/alpha", 'unknown'); 1 }; +like($@, qr/Unsupported digest algorithm/, 'unknown digest is rejected'); + +is(display_quote('plain/value'), 'plain/value', 'simple display value is unquoted'); +is(display_quote('two words'), q{'two words'}, 'display value with spaces is quoted'); +is(shell_quote("it's"), q{'it'"'"'s'}, 'shell quote escapes apostrophes'); +ok(every_step_failed(6, 6), 'every attempted step failing is reported'); +ok(!every_step_failed(6, 5), 'one surviving step is not a total failure'); +ok(!every_step_failed(0, 0), 'attempting no steps is not a failure'); + +ok(hashes_equal({ a => 1 }, { a => 1 }), 'equal hashes match'); +ok(!hashes_equal({ a => 1 }, { a => 2 }), 'different hashes do not match'); + +my $link = "$tmp/link"; +if (symlink('alpha', $link)) { + eval { relative_files($tmp); 1 }; + like($@, qr/Symbolic links are not allowed/, 'directory walk rejects symlinks'); +} else { + fail('test filesystem supports symbolic links'); +} + +done_testing(); diff --git a/t/genesis_openembedded_consumer.t b/t/genesis_openembedded_consumer.t new file mode 100644 index 0000000..cfd44b0 --- /dev/null +++ b/t/genesis_openembedded_consumer.t @@ -0,0 +1,588 @@ +use strict; +use warnings; + +use Cwd qw(abs_path); +use File::Basename qw(basename); +use File::Copy qw(copy); +use File::Path qw(make_path); +use File::Temp qw(tempdir); +use FindBin; +use Test::More; + +use lib "$FindBin::Bin/../lib"; +use lib "$FindBin::Bin/lib"; +use XCAT::BuildUtils qw( + capture_command + command_exists + digest_file + read_binary + write_binary +); +use XCAT::GenesisRelease qw( + architectures + deb_package_name + rpm_package_name +); +use XCAT::GenesisReleaseTest qw( + make_export + run_capture + write_forkmanager_stub + write_checksums + write_release_manifest +); + +my $repo_root = abs_path("$FindBin::Bin/.."); +my $packager = "$repo_root/genesis-openembedded/package"; +my $rpm_consumer = "$repo_root/mockbuild-all.pl"; +my $deb_consumer = "$repo_root/build-apt-repo.sh"; +my $revision = 'b' x 40; +my $version = '2.19.0'; +my $release = 'snap202608210726'; +my $epoch = 1787293573; +my $tmp = tempdir(CLEANUP => 1); + +if ($ENV{XCAT_GENESIS_CI}) { + BAIL_OUT('CI requires Linux root') unless $^O eq 'linux' && $> == 0; + for my $command (qw(apt-ftparchive bash createrepo_c dpkg-deb gpg rpm rpmbuild)) { + BAIL_OUT("CI requires $command") unless command_exists($command); + } +} + +SKIP: { + skip 'RPM repository tools require a root Linux builder', 30 + unless $^O eq 'linux' + && $> == 0 + && command_exists('rpmbuild') + && command_exists('rpm') + && command_exists('createrepo_c'); + test_rpm_consumer(); + test_legacy_rpm_consumer(); + test_partial_rpm_release(); + test_failed_build_release(); + test_skip_build_collects_results(); + test_dry_run_release(); +} + +SKIP: { + skip 'APT repository tools are not installed', 17 + unless $^O eq 'linux' + && command_exists('bash') + && command_exists('dpkg-deb') + && command_exists('apt-ftparchive') + && command_exists('gpg'); + test_deb_consumer(); + test_legacy_deb_consumer(); + test_partial_deb_release(); + test_apt_lock(); +} + +done_testing(); + +# mockbuild-all.pl builds and gates each target against /packages-manifest.conf, and a +# target with no section there is fatal. These fixtures use a synthetic target, and their dependency +# packages are copies of one rpm -- so the section names the header name those copies actually carry. +# The per-package manifest gate itself is covered by t/mockbuild-all.t; what these runs exercise is +# the Genesis release path. The runs pass --skip-genesis rather than the removed --skip-xcat: this +# script no longer builds the xCAT core, so xcat-core's buildrpms.pl is required only for the +# per-EL xCAT-genesis-base build. +sub write_target_manifest { + my ($root, $target) = @_; + make_path($root); + write_binary( + "$root/packages-manifest.conf", + "[$target]\n" . rpm_package_name(capture_command('uname', '-m')) . "=*\n", + ); +} + +sub test_rpm_consumer { + my $release_root = make_package_release("$tmp/rpm", 'rpm'); + my $package = "xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; + my $source_package = "xCAT-genesis-openembedded-x86_64-$version-$release.src.rpm"; + my $output = "$tmp/rpm output"; + my $target = 'test+epel-10-' . capture_command('uname', '-m'); + my $run = "$target-consumer"; + my $run_repo = "$output/mockbuild-all/$run/repo/" . capture_command('uname', '-m'); + my $source_repo = "$output/mockbuild-all/$run/repo-src"; + my $deploy_repo = "$output/xcat-dep/rh10/" . capture_command('uname', '-m'); + make_path($run_repo, $source_repo, $deploy_repo); + write_binary("$run_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale'); + write_binary("$source_repo/xCAT-genesis-openembedded-stale.src.rpm", 'stale'); + write_binary("$deploy_repo/xCAT-genesis-openembedded-stale.noarch.rpm", 'stale'); + + my $dependencies = "$tmp/rpm-dependencies"; + my $scratch_repo_root = "$tmp/rpm-repo-root"; + make_rpm_dependencies($dependencies, "$release_root/rpm/$package"); + write_target_manifest($scratch_repo_root, $target); + write_binary( + "$dependencies/xCAT-genesis-openembedded-x86_64-$version-old.noarch.rpm", + 'stale OpenEmbedded RPM', + ); + write_binary( + "$dependencies/xCAT-genesis-openembedded-x86_64-$version-old.src.rpm", + 'stale OpenEmbedded SRPM', + ); + + my @perl_lib; + push(@perl_lib, write_forkmanager_stub("$tmp/perl-stub")) + unless eval { require Parallel::ForkManager; 1 }; + push(@perl_lib, $ENV{PERL5LIB}) + if defined($ENV{PERL5LIB}) && $ENV{PERL5LIB} ne ''; + local $ENV{PERL5LIB} = join(':', @perl_lib); + my $log = "$tmp/rpm-consumer.log"; + my $status = run_capture( + $log, + $^X, $rpm_consumer, + '--repo-root', $scratch_repo_root, + '--output', $output, + '--target', $target, + '--run-id', 'consumer', + '--build-timestamp', $epoch, + '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--skip-createrepo', '--skip-tarball', + '--genesis-release', $release_root, + '--collect-dir', $dependencies, + ); + + is($status, 0, 'RPM repository accepts a verified Genesis release'); + is(digest_file("$deploy_repo/$package"), digest_file("$release_root/rpm/$package"), + 'deployed RPM matches the release'); + is( + sprintf('%04o', (stat("$deploy_repo/$package"))[2] & 0x0fff), + sprintf('%04o', (stat("$release_root/rpm/$package"))[2] & 0x0fff), + 'deployed RPM keeps the release file mode', + ); + opendir(my $deploy_dh, $deploy_repo) or die $!; + my @staging_files = grep { /^\.xcat-deploy\./ } readdir($deploy_dh); + closedir($deploy_dh) or die $!; + is_deeply(\@staging_files, [], 'RPM deployment leaves no staging files'); + ok(!-e "$run_repo/xCAT-genesis-openembedded-stale.noarch.rpm", + 'stale run RPM is removed'); + ok(!-e "$source_repo/xCAT-genesis-openembedded-stale.src.rpm", + 'stale source RPM is removed'); + ok(!-e "$deploy_repo/xCAT-genesis-openembedded-stale.noarch.rpm", + 'stale deployed RPM is removed'); + is(digest_file("$source_repo/$source_package"), + digest_file("$release_root/srpm/$source_package"), + 'source RPM matches the release'); + ok(-f "$deploy_repo/repodata/repomd.xml", 'RPM repository metadata is generated'); + ok(-f "$deploy_repo/xCAT-genesis-base-x86_64-1.noarch.rpm", + 'legacy Genesis package remains available'); + ok(!-e "$run_repo/xCAT-genesis-openembedded-x86_64-$version-old.noarch.rpm", + 'stale OpenEmbedded RPM is not collected'); + ok(!-e "$source_repo/xCAT-genesis-openembedded-x86_64-$version-old.src.rpm", + 'stale OpenEmbedded SRPM is not collected'); + like(read_binary("$output/mockbuild-all/$run/summary.txt"), qr/^copied_rpms=15$/m, + 'release RPM is counted alongside required dependencies'); +} + +sub test_deb_consumer { + my $release_root = make_package_release("$tmp/deb", 'deb'); + my $package = "xcat-genesis-openembedded-x86-64_${version}-${release}_all.deb"; + my $apt_root = "$tmp/apt"; + my $input = "$apt_root/ubuntu24.04"; + make_path($input, "$apt_root/pool/main/noble"); + make_legacy_deb("$tmp/dummy-deb", "$input/xcat-genesis-base-amd64_1_all.deb"); + write_binary("$input/xcat-genesis-openembedded-stale.deb", 'stale'); + write_binary("$apt_root/pool/main/noble/xcat-genesis-openembedded-old.deb", 'stale'); + + local $ENV{SOURCE_DATE_EPOCH} = $epoch; + my $log = "$tmp/deb-consumer.log"; + my $status = run_capture( + $log, + 'bash', $deb_consumer, + '--repo-root', $repo_root, + '--apt-dir', $apt_root, + '--skip-sign', + '--genesis-release', $release_root, + 'ubuntu24.04', + ); + my $pool_package = "$apt_root/pool/main/noble/$package"; + my $amd64 = "$apt_root/dists/noble/main/binary-amd64/Packages"; + my $ppc64el = "$apt_root/dists/noble/main/binary-ppc64el/Packages"; + + is($status, 0, 'APT repository accepts a verified Genesis release'); + like(read_binary($log), qr/Verified copied Genesis package:/, + 'APT repository uses the shared copied-package verifier'); + is(digest_file($pool_package), digest_file("$release_root/deb/$package"), + 'pooled DEB matches the release'); + ok(!-e "$apt_root/pool/main/noble/xcat-genesis-openembedded-old.deb", + 'stale pooled DEB is removed'); + like(read_binary($amd64), qr/^Package: xcat-genesis-openembedded-x86-64$/m, + 'all-architecture DEB is indexed for amd64'); + like(read_binary($ppc64el), qr/^Package: xcat-genesis-openembedded-x86-64$/m, + 'all-architecture DEB is indexed for ppc64el'); + ok(-f "$apt_root/dists/noble/Release", 'APT Release metadata is generated'); + like(read_binary($amd64), qr/^Package: xcat-genesis-base-amd64$/m, + 'legacy Genesis DEB remains available'); + ok(!-e "$apt_root/pool/main/noble/xcat-genesis-openembedded-stale.deb", + 'stale OpenEmbedded DEB is not collected'); + # The published package must be a file of its own: sharing an inode with the release + # would make a later write through either path change what the other one holds. + my @pooled = stat($pool_package); + my @released = stat("$release_root/deb/$package"); + isnt("$pooled[0]:$pooled[1]", "$released[0]:$released[1]", + 'pooled DEB is published independently of the release file'); + like(read_binary($log), qr/^Re-verified pooled Genesis package: \Q$pool_package\E$/m, + 'pooled DEB is verified again before the indexes are generated'); + + my $collision = "$tmp/apt-collision"; + make_path("$collision/ubuntu24.04"); + write_binary("$collision/ubuntu24.04/$package", 'different'); + my $collision_log = "$tmp/deb-collision.log"; + my $collision_status = run_capture( + $collision_log, + 'bash', $deb_consumer, + '--repo-root', $repo_root, + '--apt-dir', $collision, + '--skip-sign', + '--genesis-release', $release_root, + 'ubuntu24.04', + ); + is($collision_status, 0, 'verified release replaces a stale source package'); + is(digest_file("$collision/pool/main/noble/$package"), + digest_file("$release_root/deb/$package"), + 'pooled package still matches the verified release'); +} + +sub test_legacy_rpm_consumer { + my $release_root = make_package_release("$tmp/rpm-legacy", 'rpm', 'x86_64'); + my $package = "xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; + my $dependencies = "$tmp/rpm-legacy-dependencies"; + my $output = "$tmp/rpm-legacy-output"; + my $scratch_repo_root = "$tmp/rpm-legacy-repo-root"; + my $target = 'test+epel-10-' . capture_command('uname', '-m'); + my $deployed = "$output/xcat-dep/rh10/" . capture_command('uname', '-m'); + make_rpm_dependencies($dependencies, "$release_root/rpm/$package"); + write_target_manifest($scratch_repo_root, $target); + write_binary("$scratch_repo_root/Gitepoch", ''); + + my @perl_lib; + push(@perl_lib, write_forkmanager_stub("$tmp/perl-legacy-stub")) + unless eval { require Parallel::ForkManager; 1 }; + push(@perl_lib, $ENV{PERL5LIB}) + if defined($ENV{PERL5LIB}) && $ENV{PERL5LIB} ne ''; + local $ENV{PERL5LIB} = join(':', @perl_lib); + + my $log = "$tmp/rpm-legacy-consumer.log"; + my $status = run_capture( + $log, + $^X, $rpm_consumer, + '--repo-root', $scratch_repo_root, + '--output', $output, + '--target', $target, + '--run-id', 'legacy', + '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--skip-createrepo', '--skip-tarball', + '--collect-dir', $dependencies, + ); + + is($status, 0, 'RPM legacy repository accepts an empty Gitepoch'); + ok(-f "$deployed/xCAT-genesis-base-x86_64-1.noarch.rpm", + 'RPM legacy path still deploys the existing Genesis package'); + is_deeply( + [ glob("$deployed/xCAT-genesis-openembedded-*.rpm") ], + [], + 'RPM legacy path does not add OpenEmbedded packages', + ); +} + +sub test_legacy_deb_consumer { + my $apt_root = "$tmp/apt-legacy"; + my $input = "$apt_root/ubuntu24.04"; + make_path($input); + make_legacy_deb( + "$tmp/legacy-dummy-deb", + "$input/xcat-genesis-base-amd64_1_all.deb", + ); + + local $ENV{SOURCE_DATE_EPOCH} = $epoch; + my $log = "$tmp/deb-legacy-consumer.log"; + my $status = run_capture( + $log, + 'bash', $deb_consumer, + '--repo-root', $repo_root, + '--apt-dir', $apt_root, + '--skip-sign', + 'ubuntu24.04', + ); + my $packages = "$apt_root/dists/noble/main/binary-amd64/Packages"; + + is($status, 0, 'APT repository keeps working without a Genesis release'); + like(read_binary($packages), qr/^Package: xcat-genesis-base-amd64$/m, + 'APT legacy path still indexes the existing Genesis package'); + unlike(read_binary($packages), qr/^Package: xcat-genesis-openembedded-/m, + 'APT legacy path does not add OpenEmbedded packages'); +} + +sub test_partial_rpm_release { + my $release_root = make_package_release("$tmp/rpm-partial", 'rpm', 'x86_64'); + my $output = "$tmp/partial-output"; + my $target = 'test+epel-10-' . capture_command('uname', '-m'); + my $deployed = "$output/xcat-dep/rh10/" . capture_command('uname', '-m'); + my $existing = "$deployed/xCAT-genesis-base-existing.noarch.rpm"; + make_path($deployed); + write_binary($existing, 'existing release'); + + my $log = "$tmp/rpm-partial.log"; + my $status = run_capture( + $log, + $^X, $rpm_consumer, + '--repo-root', $repo_root, + '--output', $output, + '--target', $target, + '--run-id', 'partial', + '--build-timestamp', $epoch, + '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--skip-createrepo', '--skip-tarball', + '--genesis-release', $release_root, + ); + + isnt($status, 0, 'RPM repository rejects a partial Genesis release'); + like(read_binary($log), qr/Genesis release is missing supported architectures/, + 'RPM partial-release failure names the missing architectures'); + ok(-f $existing, 'partial release does not remove the deployed package'); +} + +sub test_partial_deb_release { + my $release_root = make_package_release("$tmp/deb-partial", 'deb', 'x86_64'); + my $apt_root = "$tmp/partial-apt"; + my $pool = "$apt_root/pool/main/noble"; + my $existing = "$pool/xcat-genesis-base-existing.deb"; + make_path($pool); + write_binary($existing, 'existing release'); + + my $log = "$tmp/deb-partial.log"; + my $status = run_capture( + $log, + 'bash', $deb_consumer, + '--repo-root', $repo_root, + '--apt-dir', $apt_root, + '--skip-sign', + '--genesis-release', $release_root, + 'ubuntu24.04', + ); + + isnt($status, 0, 'APT repository rejects a partial Genesis release'); + like(read_binary($log), qr/Genesis release is missing supported architectures/, + 'APT partial-release failure names the missing architectures'); + ok(-f $existing, 'partial DEB release does not remove the deployed package'); +} + +sub test_failed_build_release { + my $release_root = make_package_release("$tmp/rpm-empty", 'rpm'); + my $package = "xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; + my $output = "$tmp/empty-output"; + my $target = 'test+epel-10-' . capture_command('uname', '-m'); + my $deployed = "$output/xcat-dep/rh10/" . capture_command('uname', '-m'); + my $scratch_repo_root = "$tmp/rpm-empty-root"; + my $collected = "$tmp/rpm-empty-collect"; + my $run_repo = "$output/mockbuild-all/$target-empty/repo/" . capture_command('uname', '-m'); + my $stale = "$run_repo/ipmitool-xcat-0-stale.noarch.rpm"; + make_path($deployed, $scratch_repo_root, $collected, $run_repo); + write_target_manifest($scratch_repo_root, $target); + write_binary($stale, 'package left by an earlier run'); + + my $log = "$tmp/rpm-empty.log"; + my $status = run_capture( + $log, + $^X, $rpm_consumer, + '--repo-root', $scratch_repo_root, + '--output', $output, + '--target', $target, + '--run-id', 'empty', + '--build-timestamp', $epoch, + '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--skip-createrepo', '--skip-tarball', + '--genesis-release', $release_root, + '--collect-dir', $collected, + ); + + isnt($status, 0, 'a run that built nothing fails even with a Genesis release'); + like(read_binary($log), qr/No binary RPMs were collected/, + 'the failure names the empty collection, not the missing dependencies'); + ok(!-e "$deployed/$package", + 'a run that built nothing publishes no release package'); + ok(!-e $stale, + 'a package left by an earlier run is cleared from the staging repository'); +} + +# --skip-build collects THIS target's previously built artifacts out of its own build-results tree, +# so a run that skips building must not destroy the tree it collects from. (Where this test came +# from the --skip-build roots were the legacy build-output/list* directories and build-results was +# only ever kept, never collected -- which is why the empty-collection case above can no longer +# leave an rpm there.) +sub test_skip_build_collects_results { + my $release_root = make_package_release("$tmp/rpm-kept", 'rpm'); + my $package = "xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; + my $output = "$tmp/kept-output"; + my $target = 'test+epel-10-' . capture_command('uname', '-m'); + my $scratch_repo_root = "$tmp/rpm-kept-root"; + my $results = "$output/mockbuild-all/$target-kept/build-results/ipmitool-xcat"; + my $kept = "$results/ipmitool-xcat-1.noarch.rpm"; + my $deployed = "$output/xcat-dep/rh10/" . capture_command('uname', '-m'); + make_path($results); + copy("$release_root/rpm/$package", $kept) or die $!; + write_target_manifest($scratch_repo_root, $target); + + my $log = "$tmp/rpm-kept.log"; + my $status = run_capture( + $log, + $^X, $rpm_consumer, + '--repo-root', $scratch_repo_root, + '--output', $output, + '--target', $target, + '--run-id', 'kept', + '--build-timestamp', $epoch, + '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--skip-createrepo', '--skip-tarball', + ); + + is($status, 0, 'a run that skips building collects its own build results'); + ok(-e $kept, 'the build results it collects from are kept'); + ok(-e "$deployed/" . basename($kept), 'the collected package reaches the deployed repo'); +} + +sub test_dry_run_release { + my $release_root = make_package_release("$tmp/rpm-dry", 'rpm'); + my $package = "xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; + my $output = "$tmp/dry-output"; + my $target = 'test+epel-10-' . capture_command('uname', '-m'); + my $run_repo = "$output/mockbuild-all/$target-dry/repo/" . capture_command('uname', '-m'); + my $dependencies = "$tmp/rpm-dry-dependencies"; + my $scratch_repo_root = "$tmp/rpm-dry-root"; + make_rpm_dependencies($dependencies, "$release_root/rpm/$package"); + write_target_manifest($scratch_repo_root, $target); + + my $log = "$tmp/rpm-dry.log"; + my $status = run_capture( + $log, + $^X, $rpm_consumer, + '--repo-root', $scratch_repo_root, + '--output', $output, + '--target', $target, + '--run-id', 'dry', + '--build-timestamp', $epoch, + '--skip-build', '--skip-genesis', '--skip-xcat-dep', '--skip-perl', + '--skip-createrepo', '--skip-tarball', + '--genesis-release', $release_root, + '--collect-dir', $dependencies, + '--dry-run', + ); + my $printed = read_binary($log); + + is($status, 0, 'a dry run accepts a verified Genesis release'); + like($printed, qr/^DRY-RUN install Genesis release package: .*\Q$package\E$/m, + 'the dry run reports the release packages it would install'); + like($printed, qr/^Collected binary RPMs: 15$/m, + 'the dry run counts the release packages a real run installs'); + ok(!-e "$run_repo/$package", 'the dry run installs nothing'); +} + +sub test_apt_lock { + my $apt_root = "$tmp/apt-lock"; + my $input = "$apt_root/ubuntu24.04"; + make_path($input, "$apt_root/.lock"); + make_legacy_deb("$tmp/lock-deb", "$input/xcat-genesis-base-amd64_1_all.deb"); + + my $locked_log = "$tmp/deb-locked.log"; + my $locked_status = run_capture( + $locked_log, + 'bash', $deb_consumer, + '--repo-root', $repo_root, + '--apt-dir', $apt_root, + '--skip-sign', + 'ubuntu24.04', + ); + isnt($locked_status, 0, 'a locked APT directory is not published into'); + like(read_binary($locked_log), qr/\Q$apt_root\E is locked/, + 'the refusal names the directory another run owns'); + + my $forced_log = "$tmp/deb-forced.log"; + my $forced_status = run_capture( + $forced_log, + 'bash', $deb_consumer, + '--repo-root', $repo_root, + '--apt-dir', $apt_root, + '--skip-sign', + '--force-unlock', + 'ubuntu24.04', + ); + is($forced_status, 0, '--force-unlock takes over a stale lock'); + ok(!-d "$apt_root/.lock", 'the lock is released when the run finishes'); +} + +sub make_package_release { + my ($root, $format, @requested_architectures) = @_; + @requested_architectures = architectures() unless @requested_architectures; + my $release_root = "$root/release"; + make_path($release_root); + for my $architecture (@requested_architectures) { + my $export = make_export("$root/exports/$architecture", $architecture); + my $packages = "$root/packages/$architecture"; + die "Cannot package test release for $architecture\n" + if run_capture( + "$root/package-$architecture.log", + $packager, + '--architecture', $architecture, + '--export-dir', $export, + '--output-dir', $packages, + '--version', $version, + '--release', $release, + '--revision', $revision, + '--source-date-epoch', $epoch, + '--format', $format, + ); + if ($format eq 'rpm') { + my $name = rpm_package_name($architecture); + make_path("$release_root/rpm", "$release_root/srpm"); + copy( + "$packages/rpm/$name-$version-$release.noarch.rpm", + "$release_root/rpm/$name-$version-$release.noarch.rpm", + ) or die $!; + copy( + "$packages/srpm/$name-$version-$release.src.rpm", + "$release_root/srpm/$name-$version-$release.src.rpm", + ) or die $!; + } else { + my $name = deb_package_name($architecture); + make_path("$release_root/deb"); + copy( + "$packages/deb/${name}_${version}-${release}_all.deb", + "$release_root/deb/${name}_${version}-${release}_all.deb", + ) or die $!; + } + } + write_release_manifest( + $release_root, $version, $release, $revision, $epoch, + join(',', @requested_architectures), $format, + ); + write_checksums($release_root); + return $release_root; +} + +sub make_rpm_dependencies { + my ($directory, $package) = @_; + make_path($directory); + for my $name (qw( + ipmitool-xcat syslinux-xcat grub2-xcat xnba-undi + perl-IO-Stty perl-HTTP-Async perl-Net-HTTPS-NB + xCAT-genesis-base-x86_64 + )) { + copy($package, "$directory/$name-1.noarch.rpm") or die $!; + } +} + +sub make_legacy_deb { + my ($root, $output) = @_; + make_path("$root/DEBIAN"); + write_binary( + "$root/DEBIAN/control", + "Package: xcat-genesis-base-amd64\nVersion: 1\nArchitecture: all\n" + . "Maintainer: xCAT \n" + . "Description: repository test package\n", + ); + die "Cannot build legacy test DEB\n" + if run_capture( + "$root.log", 'dpkg-deb', '--root-owner-group', '--build', $root, $output, + ); +} diff --git a/t/genesis_openembedded_release.t b/t/genesis_openembedded_release.t new file mode 100644 index 0000000..276061b --- /dev/null +++ b/t/genesis_openembedded_release.t @@ -0,0 +1,513 @@ +use strict; +use warnings; + +use Cwd qw(abs_path); +use File::Copy qw(copy); +use File::Path qw(make_path remove_tree); +use File::Temp qw(tempdir); +use FindBin; +use Test::More; + +use lib "$FindBin::Bin/../lib"; +use lib "$FindBin::Bin/lib"; +use XCAT::BuildUtils qw( + command_exists + digest_file + read_binary + write_binary +); +use XCAT::GenesisRelease qw( + architectures + deb_package_name + rpm_package_name + validated_release_checksums + validate_architecture + validate_complete_release + validate_export + validate_release + verify_release_file +); +use XCAT::GenesisReleaseTest qw( + copy_tree + dies_like + make_export + run_capture + write_checksums + write_release_manifest +); + +my $repo_root = abs_path("$FindBin::Bin/.."); +my $packager = "$repo_root/genesis-openembedded/package"; +my $builder = "$repo_root/genesis-openembedded/build"; +my $verifier = "$repo_root/genesis-openembedded/verify-release"; +my $revision = 'a' x 40; +my $version = '2.19.0'; +my $release = 'snap202608210726'; +my $epoch = 1787293573; + +if ($ENV{XCAT_GENESIS_CI}) { + for my $command (qw(git dpkg-deb rpm rpmbuild tar)) { + BAIL_OUT("CI requires $command") unless command_exists($command); + } +} + +is_deeply( + [ architectures() ], + [ qw(x86 x86_64 ppc64 ppc64le armv7hf aarch64 riscv64) ], + 'supported architectures keep their exact xCAT names', +); +is(rpm_package_name('ppc64le'), 'xCAT-genesis-openembedded-ppc64le', + 'RPM package keeps ppc64le distinct'); +is(deb_package_name('x86_64'), 'xcat-genesis-openembedded-x86-64', + 'DEB package uses a legal spelling of x86_64'); +dies_like(sub { validate_architecture('ppc') }, qr/Unsupported Genesis architecture/, + 'legacy ppc alias is rejected'); + +my $tmp = tempdir(CLEANUP => 1); +my $export = make_export("$tmp/export", 'x86_64'); +ok(validate_export($export, 'x86_64'), 'valid export passes'); +dies_like(sub { validate_export($export, 'ppc64le') }, qr/architecture mismatch/, + 'wrong architecture fails'); + +my $missing = make_export("$tmp/missing", 'x86_64'); +unlink("$missing/image.vex.json") or die $!; +write_checksums($missing); +dies_like(sub { validate_export($missing, 'x86_64') }, qr/missing image\.vex\.json/, + 'missing release evidence fails'); + +my $corrupt = make_export("$tmp/corrupt", 'x86_64'); +write_binary("$corrupt/kernel", 'changed'); +dies_like(sub { validate_export($corrupt, 'x86_64') }, qr/Checksum mismatch for kernel/, + 'corrupt payload fails'); + +my $unexpected = make_export("$tmp/unexpected", 'x86_64'); +write_binary("$unexpected/extra", 'not part of the export'); +write_checksums($unexpected); +dies_like(sub { validate_export($unexpected, 'x86_64') }, qr/Unexpected Genesis export file/, + 'unlisted export files fail'); + +my $linked = make_export("$tmp/linked", 'x86_64'); +unlink("$linked/kernel") or die $!; +symlink('initramfs.cpio.gz', "$linked/kernel") or die $!; +dies_like(sub { validate_export($linked, 'x86_64') }, qr/Symbolic links are not allowed/, + 'export symlinks fail'); + +my $riscv = make_export("$tmp/riscv", 'riscv64'); +ok(-f "$riscv/fw_jump.elf", 'RISC-V export carries firmware'); +ok(validate_export($riscv, 'riscv64'), 'RISC-V export passes'); + +my $release_dir = "$tmp/release"; +make_path("$release_dir/rpm", "$release_dir/srpm", "$release_dir/deb"); +for my $architecture (qw(x86_64 ppc64le)) { + my $rpm = rpm_package_name($architecture); + my $deb = deb_package_name($architecture); + write_binary("$release_dir/rpm/$rpm-$version-$release.noarch.rpm", "rpm $architecture"); + write_binary("$release_dir/srpm/$rpm-$version-$release.src.rpm", "srpm $architecture"); + write_binary("$release_dir/deb/${deb}_${version}-${release}_all.deb", "deb $architecture"); +} +write_release_manifest( + $release_dir, $version, $release, $revision, $epoch, + 'x86_64,ppc64le', 'deb,rpm', +); +write_checksums($release_dir); +my $manifest = validate_release($release_dir); +is($manifest->{xcat_revision}, $revision, 'release records xcat-core revision'); +my $qualified_release = "$tmp/qualified-release"; +my $qualified_release_name = '1+deb~1'; +my $qualified_deb = deb_package_name('x86_64'); +make_path("$qualified_release/deb"); +write_binary( + "$qualified_release/deb/${qualified_deb}_${version}-${qualified_release_name}_all.deb", + 'deb x86_64', +); +write_release_manifest( + $qualified_release, $version, $qualified_release_name, $revision, $epoch, + 'x86_64', 'deb', +); +write_checksums($qualified_release); +ok(validate_release($qualified_release), 'package filenames accept valid release qualifiers'); +my $checksum_reads = 0; +my $verified_checksums; +{ + no warnings 'redefine'; + my $reader = \&XCAT::GenesisRelease::read_checksum_manifest; + local *XCAT::GenesisRelease::read_checksum_manifest = sub { + $checksum_reads++; + return $reader->(@_); + }; + $verified_checksums = validated_release_checksums($release_dir); +} +is($checksum_reads, 1, 'validated checksums use the verified manifest read'); +my $verified_relative = "rpm/xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; +my $verified_copy = "$tmp/verified-copy.rpm"; +copy("$release_dir/$verified_relative", $verified_copy) or die $!; +ok(verify_release_file($verified_checksums, $verified_relative, $verified_copy), + 'collected package matches the verified release'); +write_binary($verified_copy, 'changed after verification'); +dies_like( + sub { verify_release_file($verified_checksums, $verified_relative, $verified_copy) }, + qr/Collected release file checksum mismatch/, + 'release changes after verification are rejected', +); +copy("$release_dir/$verified_relative", $verified_copy) or die $!; +my $copied_file_log = "$tmp/copied-file.log"; +is( + run_capture( + $copied_file_log, + $verifier, + '--checksum-file', "$release_dir/SHA256SUMS", + '--relative-file', $verified_relative, + '--copied-file', $verified_copy, + ), + 0, + 'verifier accepts a copied release file', +); +write_binary($verified_copy, 'changed after verification'); +isnt( + run_capture( + $copied_file_log, + $verifier, + '--checksum-file', "$release_dir/SHA256SUMS", + '--relative-file', $verified_relative, + '--copied-file', $verified_copy, + ), + 0, + 'verifier rejects a changed copied file', +); +like( + read_binary($copied_file_log), + qr/Collected release file checksum mismatch/, + 'copied-file failure names the checksum mismatch', +); +dies_like( + sub { validate_complete_release($release_dir) }, + qr/Genesis release is missing supported architectures/, + 'partial release cannot be published', +); + +my $complete_release = "$tmp/complete-release"; +make_path("$complete_release/rpm", "$complete_release/srpm", "$complete_release/deb"); +for my $architecture (architectures()) { + my $rpm = rpm_package_name($architecture); + my $deb = deb_package_name($architecture); + write_binary("$complete_release/rpm/$rpm-$version-$release.noarch.rpm", "rpm $architecture"); + write_binary("$complete_release/srpm/$rpm-$version-$release.src.rpm", "srpm $architecture"); + write_binary("$complete_release/deb/${deb}_${version}-${release}_all.deb", "deb $architecture"); +} +write_release_manifest( + $complete_release, $version, $release, $revision, $epoch, + join(',', architectures()), 'deb,rpm', +); +write_checksums($complete_release); +ok(validate_complete_release($complete_release), 'complete release can be published'); + +my $deb_only_release = "$tmp/deb-only-release"; +make_path("$deb_only_release/deb"); +for my $architecture (architectures()) { + my $deb = deb_package_name($architecture); + write_binary("$deb_only_release/deb/${deb}_${version}-${release}_all.deb", "deb $architecture"); +} +write_release_manifest( + $deb_only_release, $version, $release, $revision, $epoch, + join(',', architectures()), 'deb', +); +write_checksums($deb_only_release); +my $verify_all_log = "$tmp/verify-all.log"; +isnt(run_capture($verify_all_log, $verifier, $deb_only_release), 0, + 'all-format verification rejects a single-format release'); +like(read_binary($verify_all_log), qr/Release does not contain rpm packages/, + 'all-format failure names the missing format'); + +my $bad_release = "$tmp/bad-release"; +copy_tree($release_dir, $bad_release); +write_binary("$bad_release/rpm/stale.rpm", 'stale'); +write_checksums($bad_release); +dies_like(sub { validate_release($bad_release) }, qr/Unexpected Genesis release artifact/, + 'stale package fails'); + +my $missing_release = "$tmp/missing-release"; +copy_tree($release_dir, $missing_release); +unlink("$missing_release/rpm/xCAT-genesis-openembedded-ppc64le-$version-$release.noarch.rpm") or die $!; +write_checksums($missing_release); +dies_like(sub { validate_release($missing_release) }, qr/Genesis release is missing/, + 'incomplete architecture set fails'); + +SKIP: { + skip 'git is not installed', 2 unless command_exists('git'); + my $source = "$tmp/dirty-xcat-core"; + make_path("$source/xCAT-genesis-builder/oe"); + write_binary("$source/Version", "$version\n"); + write_binary("$source/xCAT-genesis-builder/oe/build", "#!/bin/sh\nexit 99\n"); + write_binary("$source/xCAT-genesis-builder/oe/export", "#!/bin/sh\nexit 99\n"); + for my $command ( + [ 'git', '-C', $source, 'init', '-q' ], + [ 'git', '-C', $source, 'add', '.' ], + [ 'git', '-C', $source, '-c', 'user.name=xCAT test', + '-c', 'user.email=xcat-test@example.invalid', 'commit', '-qm', 'fixture' ], + ) { + die "Cannot prepare test repository\n" + if run_capture("$tmp/git-fixture.log", @{$command}); + } + write_binary("$source/untracked", "not part of the commit\n"); + my $log = "$tmp/dirty-source.log"; + isnt( + run_capture( + $log, $builder, '--xcat-source', $source, + '--output-dir', "$tmp/dirty-output", + ), + 0, + 'release builder rejects untracked source files', + ); + like(read_binary($log), qr/xcat-core checkout is not clean/, + 'dirty checkout failure is explicit'); +} + +SKIP: { + skip 'rpmbuild and rpm are not installed', 18 + unless command_exists('rpmbuild') && command_exists('rpm'); + exercise_packager('rpm'); +} + +SKIP: { + skip 'rpmbuild and rpm are not installed', 2 + unless command_exists('rpmbuild') && command_exists('rpm'); + skip 'root can traverse an unsearchable working directory', 2 if $> == 0; + exercise_packager_from_unsearchable_cwd(); +} + +SKIP: { + skip 'dpkg-deb is not installed', 12 unless command_exists('dpkg-deb'); + exercise_packager('deb'); +} + +SKIP: { + skip 'git and dpkg-deb are not installed', 6 + unless command_exists('git') && command_exists('dpkg-deb'); + exercise_builder_tmpdir(); +} + +done_testing(); + +sub exercise_packager { + my ($format) = @_; + my $first = "$tmp/$format-first"; + my $second = "$tmp/$format-second"; + my $package_revision = sprintf('%032x%08x', time, $$); + my @command = ( + $packager, + '--architecture', 'x86_64', + '--export-dir', $export, + '--version', $version, + '--release', $release, + '--revision', $package_revision, + '--source-date-epoch', $epoch, + '--format', $format, + ); + my ($legacy_rpm_top, $legacy_rpm_work); + if ($format eq 'rpm') { + $legacy_rpm_top = + "/var/tmp/xcat-genesis-rpmbuild-$package_revision-x86_64-$version-$release"; + $legacy_rpm_work = "$legacy_rpm_top/BUILD/stale"; + make_path("$legacy_rpm_top/BUILD"); + write_binary($legacy_rpm_work, 'stale work'); + } + + my $original_umask = umask(0022); + is(system(@command, '--output-dir', $first), 0, + "$format package builds with umask 0022"); + umask(0002); + is(system(@command, '--output-dir', $second), 0, + "$format package rebuilds with umask 0002"); + umask($original_umask); + is(sprintf('%04o', (stat($first))[2] & 0x0fff), '0755', + "$format output is readable and searchable"); + is(sprintf('%04o', (stat($second))[2] & 0x0fff), '0755', + "$format rebuilt output is readable and searchable"); + if (defined($legacy_rpm_work)) { + ok(-e $legacy_rpm_work, 'RPM package ignores the legacy fixed work path'); + remove_tree($legacy_rpm_top); + } + + my ($relative, $source_relative); + if ($format eq 'rpm') { + $relative = "rpm/xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm"; + $source_relative = "srpm/xCAT-genesis-openembedded-x86_64-$version-$release.src.rpm"; + } else { + $relative = "deb/xcat-genesis-openembedded-x86-64_${version}-${release}_all.deb"; + } + ok(-f "$first/$relative", "$format binary exists"); + is(digest_file("$first/$relative"), digest_file("$second/$relative"), + "$format binary is reproducible across umasks"); + if ($format eq 'rpm') { + ok(-f "$first/$source_relative", 'source RPM exists'); + is(digest_file("$first/$source_relative"), digest_file("$second/$source_relative"), + 'source RPM is reproducible across umasks'); + } + + my $release_root = "$tmp/$format-release"; + make_path($release_root); + copy_tree($first, $release_root); + write_release_manifest( + $release_root, $version, $release, $package_revision, $epoch, 'x86_64', $format, + ); + write_checksums($release_root); + ok(validate_release($release_root), "$format release layout passes"); + is(system($verifier, '--format', $format, $release_root), 0, + "$format package metadata passes"); + my $contents_log = "$tmp/$format-contents.log"; + if ($format eq 'rpm') { + is(run_capture($contents_log, 'rpm', '-qpl', "$first/$relative"), 0, + 'RPM payload can be listed'); + like(read_binary($contents_log), + qr{/opt/xcat/share/xcat/netboot/genesis-openembedded/x86_64/kernel}, + 'RPM uses the OpenEmbedded staging namespace'); + like(read_binary($contents_log), + qr{^/usr/share/doc/xCAT-genesis-openembedded-x86_64/?$}m, + 'RPM owns its documentation directory'); + my $ownership_log = "$tmp/rpm-ownership.log"; + is( + run_capture( + $ownership_log, 'rpm', '-qp', + '--qf', '[%{FILEUSERNAME}:%{FILEGROUPNAME}\n]', + "$first/$relative", + ), + 0, + 'RPM file ownership can be read', + ); + is_deeply( + [ grep { $_ ne 'root:root' } split(/\n/, read_binary($ownership_log)) ], + [], + 'RPM owns every payload path as root', + ); + is(run_capture($contents_log, 'rpm', '-qp', '--scripts', "$first/$relative"), 0, + 'RPM script metadata can be read'); + is(read_binary($contents_log), '', 'RPM has no package scripts'); + } else { + is(run_capture($contents_log, 'dpkg-deb', '-c', "$first/$relative"), 0, + 'DEB payload can be listed'); + like(read_binary($contents_log), + qr{/opt/xcat/share/xcat/netboot/genesis-openembedded/x86_64/kernel}, + 'DEB uses the OpenEmbedded staging namespace'); + my $control = "$tmp/deb-control"; + make_path($control); + is(run_capture($contents_log, 'dpkg-deb', '-e', "$first/$relative", $control), 0, + 'DEB control files can be extracted'); + opendir(my $control_dh, $control) or die $!; + my @control_files = sort grep { $_ ne '.' && $_ ne '..' } readdir($control_dh); + closedir($control_dh) or die $!; + is_deeply(\@control_files, [ qw(control md5sums) ], + 'DEB has no maintainer scripts or triggers'); + } +} + +sub exercise_packager_from_unsearchable_cwd { + my $cwd = "$tmp/unsearchable-cwd"; + my $output = "$tmp/cwd-independent-rpm"; + my $log = "$tmp/cwd-independent-rpm.log"; + make_path($cwd); + + my $pid = fork(); + die "Cannot fork: $!\n" unless defined($pid); + if ($pid == 0) { + chdir($cwd) or die "Cannot enter test directory: $!\n"; + chmod(0000, $cwd) or die "Cannot restrict test directory: $!\n"; + open(STDOUT, '>:raw', $log) or die $!; + open(STDERR, '>&', STDOUT) or die $!; + exec( + $packager, + '--architecture', 'x86_64', + '--export-dir', $export, + '--output-dir', $output, + '--version', $version, + '--release', $release, + '--revision', $revision, + '--source-date-epoch', $epoch, + '--format', 'rpm', + ) or die "Cannot run packager: $!\n"; + } + waitpid($pid, 0); + my $status = $? >> 8; + chmod(0700, $cwd) or die "Cannot restore test directory: $!\n"; + + is($status, 0, 'RPM package ignores an inaccessible inherited working directory'); + ok( + -f "$output/rpm/xCAT-genesis-openembedded-x86_64-$version-$release.noarch.rpm", + 'RPM package is published from an inaccessible inherited working directory', + ); +} + +sub exercise_builder_tmpdir { + my $source = "$tmp/tmpdir-xcat-core"; + my $oe = "$source/xCAT-genesis-builder/oe"; + make_path($oe); + write_binary("$source/Version", "$version\n"); + write_binary( + "$oe/build", + <<'BUILD', +#!/bin/sh +set -eu +expected=$XCAT_GENESIS_WORK_DIR/build/tmp +[ "${TMPDIR:-}" = "$expected" ] || exit 41 +mkdir -p "$TMPDIR/deploy" +BUILD + ); + write_binary( + "$oe/export", + <<'EXPORT', +#!/bin/sh +set -eu +architecture=$1 +deploy=$2 +output=$3 +[ "$deploy" = "$XCAT_GENESIS_WORK_DIR/build/tmp/deploy" ] || exit 42 +mkdir -p "$output" +printf '%s\n' kernel >"$output/kernel" +printf '%s\n' initramfs >"$output/initramfs.cpio.gz" +printf '%s\n' packages >"$output/image.manifest" +printf '%s\n' '{}' >"$output/image.spdx.json" +printf '%s\n' '{}' >"$output/image.vex.json" +printf '%s\n' licenses >"$output/license.manifest" +printf 'format=xcat-genesis\nversion=1\narchitecture=%s\n' "$architecture" \ + >"$output/xcat-genesis.manifest" +( + cd "$output" + sha256sum -- * >SHA256SUMS +) +EXPORT + ); + chmod(0755, "$oe/build", "$oe/export") or die $!; + for my $command ( + [ 'git', '-C', $source, 'init', '-q' ], + [ 'git', '-C', $source, 'add', '.' ], + [ 'git', '-C', $source, '-c', 'user.name=xCAT test', + '-c', 'user.email=xcat-test@example.invalid', 'commit', '-qm', 'fixture' ], + ) { + die "Cannot prepare test repository\n" + if run_capture("$tmp/tmpdir-git.log", @{$command}); + } + + my $ambient_tmp = "$tmp/ambient-tmp"; + my $persistent_work = "$tmp/persistent-work"; + my $output = "$tmp/tmpdir-release"; + my $log = "$tmp/tmpdir-builder.log"; + make_path($ambient_tmp); + my $status; + { + local $ENV{TMPDIR} = $ambient_tmp; + $status = run_capture( + $log, $builder, '--xcat-source', $source, + '--output-dir', $output, '--work-dir', $persistent_work, + '--format', 'deb', + ); + } + is($status, 0, 'release builder isolates the OpenEmbedded tmpdir'); + unlike(read_binary($log), qr/Invalid OpenEmbedded deploy directory/, + 'release builder finds the configured deploy directory'); + ok(-d "$persistent_work/openembedded/build/tmp", + 'release builder preserves the requested work directory'); + is((stat($output))[2] & oct('07777'), oct('0755'), + 'release directory is readable by other users'); + my $built = validate_release($output); + is($built->{architectures}, 'x86_64', 'isolated build keeps the target architecture'); + is($built->{formats}, 'deb', 'isolated build keeps the requested format'); +} diff --git a/t/lib/XCAT/GenesisReleaseTest.pm b/t/lib/XCAT/GenesisReleaseTest.pm new file mode 100644 index 0000000..7660784 --- /dev/null +++ b/t/lib/XCAT/GenesisReleaseTest.pm @@ -0,0 +1,121 @@ +package XCAT::GenesisReleaseTest; + +use strict; +use warnings; + +use Exporter qw(import); +use File::Basename qw(dirname); +use File::Copy qw(copy); +use File::Path qw(make_path); +use Test::More (); +use XCAT::BuildUtils qw( + digest_manifest + relative_files + write_binary +); + +our @EXPORT_OK = qw( + copy_tree + dies_like + make_export + run_capture + write_forkmanager_stub + write_checksums + write_release_manifest +); + +sub make_export { + my ($directory, $architecture) = @_; + make_path($directory); + my %content = ( + 'kernel' => 'kernel', + 'initramfs.cpio.gz' => 'initramfs', + 'image.manifest' => 'packages', + 'image.spdx.json' => '{}', + 'image.vex.json' => '{}', + 'license.manifest' => 'licenses', + 'xcat-genesis.manifest' => + "format=xcat-genesis\nversion=1\narchitecture=$architecture\n", + ); + $content{'fw_jump.elf'} = 'firmware' if $architecture eq 'riscv64'; + write_binary("$directory/$_", $content{$_}) for sort keys %content; + write_checksums($directory); + return $directory; +} + +sub write_release_manifest { + my ($directory, $xcat_version, $xcat_release, $revision, $epoch, + $architectures, $formats) = @_; + write_binary( + "$directory/release.manifest", + "format=xcat-genesis-packages\n" + . "version=1\n" + . "xcat_version=$xcat_version\n" + . "xcat_release=$xcat_release\n" + . "xcat_revision=$revision\n" + . "source_date_epoch=$epoch\n" + . "architectures=$architectures\n" + . "formats=$formats\n", + ); +} + +sub write_checksums { + my ($directory) = @_; + unlink("$directory/SHA256SUMS") if -e "$directory/SHA256SUMS"; + my @files = relative_files($directory); + write_binary( + "$directory/SHA256SUMS", + digest_manifest($directory, 'sha256', @files), + ); +} + +sub copy_tree { + my ($source, $destination) = @_; + make_path($destination); + for my $relative (relative_files($source)) { + my $target = "$destination/$relative"; + make_path(dirname($target)); + copy("$source/$relative", $target) or die $!; + } +} + +sub write_forkmanager_stub { + my ($root) = @_; + my $stub = "$root/Parallel/ForkManager.pm"; + make_path("$root/Parallel"); + write_binary( + $stub, + "package Parallel::ForkManager;\n" + . "sub new { bless {}, shift }\n" + . "sub run_on_finish { \$_[0]->{callback} = \$_[1] }\n" + . "sub start { 0 }\n" + . "sub finish { my (\$self, \$exit) = \@_; " + . "\$self->{callback}->(\$\$, \$exit, undef, 0, 0) if \$self->{callback}; 0 }\n" + . "sub wait_all_children { 0 }\n1;\n", + ); + return $root; +} + +sub run_capture { + my ($log, @command) = @_; + my $pid = fork(); + die "Cannot fork: $!\n" unless defined $pid; + if ($pid == 0) { + open(STDOUT, '>:raw', $log) or die $!; + open(STDERR, '>&', STDOUT) or die $!; + exec(@command) or die "Cannot run $command[0]: $!\n"; + } + waitpid($pid, 0); + return 255 if $? == -1; + return 128 + ($? & 127) if $? & 127; + return $? >> 8; +} + +sub dies_like { + my ($code, $pattern, $name) = @_; + my $error = ''; + eval { $code->(); 1 } or $error = $@; + Test::More::like($error, $pattern, $name); +} + +1;