diff --git a/.github/workflows/xcat_test.yml b/.github/workflows/xcat_test.yml index a61196daa7..2fb86641ef 100644 --- a/.github/workflows/xcat_test.yml +++ b/.github/workflows/xcat_test.yml @@ -104,7 +104,7 @@ jobs: run: | log=$(mktemp) trap 'rm -f -- "$log"' EXIT - prove -v xCAT-test/native/install_profile_riscv64.t xCAT-test/native/netboot_profile_riscv64.t xCAT-test/native/netboot_profile_postinstall.t xCAT-test/native/kdump_condition_systemd.t xCAT-test/native/openeuler_post_kdump.t xCAT-test/native/builddebs_riscv64.t xCAT-test/native/ubuntu_subiquity_template.t | tee "$log" + prove -v xCAT-test/native/install_profile_riscv64.t xCAT-test/native/netboot_profile_riscv64.t xCAT-test/native/netboot_profile_postinstall.t xCAT-test/native/kdump_condition_systemd.t xCAT-test/native/openeuler_post_kdump.t xCAT-test/native/builddebs_riscv64.t xCAT-test/native/ubuntu_subiquity_template.t xCAT-test/native/ubuntu_subiquity_install_disk.t | tee "$log" ! grep -qi skipped "$log" rpm_lifecycle_test: runs-on: ubuntu-24.04 diff --git a/docs/source/guides/admin-guides/manage_clusters/common/deployment/cfg_partition.rst b/docs/source/guides/admin-guides/manage_clusters/common/deployment/cfg_partition.rst index 82767e6d5e..fe5a2090df 100644 --- a/docs/source/guides/admin-guides/manage_clusters/common/deployment/cfg_partition.rst +++ b/docs/source/guides/admin-guides/manage_clusters/common/deployment/cfg_partition.rst @@ -9,6 +9,8 @@ By default, xCAT will attempt to determine the first physical disk and use a gen .. warning:: **partition definition script** has only been verified on RedHat and Ubuntu, use at your own risk for SLES. +.. note:: Ubuntu 20.04 and later install with Subiquity. There the partition definition file, or the file the partition definition script writes to ``/tmp/partitionfile``, holds a curtin ``storage:`` section with a ``config`` or ``layout`` entry instead of a partman recipe. The installer keeps only that section, and it replaces the default layout. With a partition disk file or script as well, a ``layout`` without a ``match`` is restricted to the disk it selects, and a ``use_gap`` layout is refused. A layout or disk entry that fails, or that writes no layout or disk, stops the install before it partitions. The additional preseed configuration applies to the debian-installer only. + .. END_Overview @@ -480,13 +482,14 @@ If not specified, the default value will be used. nodeset osimage= - the ``d:`` preceding the filename tells nodeset that this is a partition disk file. -- For Ubuntu, when nodeset runs and generates the ``/install/autoinst`` file for a node, it will generate a script to write the content of the partition disk file to ``/tmp/install_disk``, this context to run the script will replace the ``#XCA_PARTMAN_DISK_SCRIPT#`` directive in ``/install/autoinst/.pre``. +- For Ubuntu, when nodeset runs and generates the ``/install/autoinst`` file for a node, it will generate a script to write the content of the partition disk file to ``/tmp/xcat.install_disk``, this context to run the script will replace the ``#XCA_PARTMAN_DISK_SCRIPT#`` directive in ``/install/autoinst/.pre``. +- A Subiquity install partitions only the first disk the file names. Use a ``/dev/disk/by-id/`` name when the kernel can enumerate the disks in a different order at each boot. .. END_Partition_Disk_File_ubuntu_only .. BEGIN_Partition_Disk_Script_ubuntu_only -The disk script contains a script to generate a partitioning disk file named ``/tmp/install_disk``. for example: :: +The disk script contains a script to generate a partitioning disk file named ``/tmp/xcat.install_disk``. for example: :: rm /tmp/devs-with-boot 2>/dev/null || true; for d in $(list-devices partition); do @@ -499,12 +502,12 @@ The disk script contains a script to generate a partitioning disk file named ``/ fi done; if [[ -e /tmp/devs-with-boot ]]; then - head -n1 /tmp/devs-with-boot | egrep -o '\S+[^0-9]' > /tmp/install_disk; + head -n1 /tmp/devs-with-boot | egrep -o '\S+[^0-9]' > /tmp/xcat.install_disk; rm /tmp/devs-with-boot 2>/dev/null || true; else DEV=`ls /dev/disk/by-path/* -l | egrep -o '/dev.*[s|h|v]d[^0-9]$' | sort -t : -k 1 -k 2 -k 3 -k 4 -k 5 -k 6 -k 7 -k 8 -g | head -n1 | egrep -o '[s|h|v]d.*$'`; if [[ "$DEV" == "" ]]; then DEV="sda"; fi; - echo "/dev/$DEV" > /tmp/install_disk; + echo "/dev/$DEV" > /tmp/xcat.install_disk; fi; If not specified, the default value will be used. @@ -516,6 +519,7 @@ If not specified, the default value will be used. - the ``s:`` prefix tells ``nodeset`` that is a script, the ``s:d:`` preceding the filename tells ``nodeset`` that this is a script to generate the partition disk file. - For Ubuntu, when nodeset runs and generates the ``/install/autoinst`` file for a node, this context to run the script will replace the ``#XCA_PARTMAN_DISK_SCRIPT#`` directive in ``/install/autoinst/.pre``. +- A Subiquity install runs the script in the live installer system, under ``dash`` unless its first line names another interpreter. ``list-devices``, which the example uses, exists only in the debian-installer. .. END_Partition_Disk_Script_ubuntu_only diff --git a/xCAT-server/lib/perl/xCAT/Template.pm b/xCAT-server/lib/perl/xCAT/Template.pm index 747ba01e04..d332a91f5e 100644 --- a/xCAT-server/lib/perl/xCAT/Template.pm +++ b/xCAT-server/lib/perl/xCAT/Template.pm @@ -224,15 +224,16 @@ sub subvars { $filepath = $1; } + # The last entry of each kind wins, with its own s: prefix. if ($filepath =~ /^d:(.*)/) { - $diskfile = $1; - if ($scriptflag) { $diskscriptflag = 1; } + $diskfile = $1; + $diskscriptflag = $scriptflag; } elsif ($filepath =~ /^c:(.*)/) { - $configfile = $1; - if ($scriptflag) { $configscriptflag = 1; } + $configfile = $1; + $configscriptflag = $scriptflag; } elsif ($filepath) { - $partitionfile = $filepath; - if ($scriptflag) { $partscriptflag = 1; } + $partitionfile = $filepath; + $partscriptflag = $scriptflag; } } @@ -483,7 +484,7 @@ sub subvars { # Put the base64 coded partitionfile into %pre part $partcontent = "cat > /tmp/partscript.enc << EOFEOF\n" . $partcontent . "\nEOFEOF\n"; - if ($scriptflag) { + if ($partscriptflag) { # Put the code to decode preseed script and run it to generate pressed recipe file #$partcontent .= "python -c 'import base64; print base64.b64decode(open(\"/tmp/partscript.enc\",\"rb\").read())' >/tmp/partscript\n"; diff --git a/xCAT-server/share/xcat/install/scripts/pre.ubuntu.subiquity b/xCAT-server/share/xcat/install/scripts/pre.ubuntu.subiquity index 92707fb716..90b3e363b1 100644 --- a/xCAT-server/share/xcat/install/scripts/pre.ubuntu.subiquity +++ b/xCAT-server/share/xcat/install/scripts/pre.ubuntu.subiquity @@ -139,63 +139,29 @@ chmod 755 /tmp/foo.sh /tmp/foo.sh >/tmp/foo.log 2>&1 & +# Subiquity runs this script under dash, which has no read -n, and its live system ships coreutils. base64decode() { - h0="0"; h1="1"; h2="2"; h3="3"; h4="4"; h5="5"; h6="6"; h7="7"; - h8="8"; h9="9"; h10="A"; h11="B"; h12="C"; h13="D"; h14="E"; h15="F"; - - while : - do - i=0 - while [ "$i" -lt 4 ] - do - read -n 1 c || return - - case "${c}" in - "A") d=0 ;; "B") d=1 ;; "C") d=2 ;; "D") d=3 ;; - "E") d=4 ;; "F") d=5 ;; "G") d=6 ;; "H") d=7 ;; - "I") d=8 ;; "J") d=9 ;; "K") d=10 ;; "L") d=11 ;; - "M") d=12 ;; "N") d=13 ;; "O") d=14 ;; "P") d=15 ;; - "Q") d=16 ;; "R") d=17 ;; "S") d=18 ;; "T") d=19 ;; - "U") d=20 ;; "V") d=21 ;; "W") d=22 ;; "X") d=23 ;; - "Y") d=24 ;; "Z") d=25 ;; "a") d=26 ;; "b") d=27 ;; - "c") d=28 ;; "d") d=29 ;; "e") d=30 ;; "f") d=31 ;; - "g") d=32 ;; "h") d=33 ;; "i") d=34 ;; "j") d=35 ;; - "k") d=36 ;; "l") d=37 ;; "m") d=38 ;; "n") d=39 ;; - "o") d=40 ;; "p") d=41 ;; "q") d=42 ;; "r") d=43 ;; - "s") d=44 ;; "t") d=45 ;; "u") d=46 ;; "v") d=47 ;; - "w") d=48 ;; "x") d=49 ;; "y") d=50 ;; "z") d=51 ;; - "0") d=52 ;; "1") d=53 ;; "2") d=54 ;; "3") d=55 ;; - "4") d=56 ;; "5") d=57 ;; "6") d=58 ;; "7") d=59 ;; - "8") d=60 ;; "9") d=61 ;; "+") d=62 ;; "/") d=63 ;; - "=") d=0 ;; - *) continue ;; - esac - - eval "a${i}=${c}" - eval "b${i}=${d}" - - i=$(( i + 1 )) - done - - o0=$(( ((b0 << 2) | (b1 >> 4)) & 0xff )) - o1=$(( ((b1 << 4) | (b2 >> 2)) & 0xff )) - o2=$(( ((b2 << 6) | b3) & 0xff )) - - [ "${a2}" == "=" ] && - eval "printf \"\\x\${h$(( o0 / 16 ))}\${h$(( o0 % 16 - ))}\"" && - return - [ "${a3}" == "=" ] && - eval "printf \"\\x\${h$(( o0 / 16 ))}\${h$(( o0 % 16 - ))}\\x\${h$(( o1 / 16 ))}\${h$(( o1 % 16 - ))}\"" && - return - eval "printf \"\\x\${h$(( o0 / 16 ))}\${h$(( o0 % 16 - ))}\\x\${h$(( o1 / 16 ))}\${h$(( o1 % 16 - ))}\\x\${h$(( o2 / 16 ))}\${h$(( o2 % 16 ))}\"" - done + base64 -d } + +# A d: or s:d: entry in linuximage.partitionfile replaces the disk getinstdisk chose. +rm -f /tmp/diskscript.enc +[ -e /tmp/xcat.install_disk ] && mv /tmp/xcat.install_disk /tmp/xcat.install_disk.detected +#XCA_PARTMAN_DISK_SCRIPT# +override_rc=$? +disk_entry="" +if [ -e /tmp/diskscript.enc ]; then + disk_entry=1 + # Falling back to the detected disk could wipe the disk the entry protects. + if [ "$override_rc" -ne 0 ] || [ -z "$(awk 'NF {print $1; exit}' /tmp/xcat.install_disk 2>/dev/null)" ]; then + logger -t $log_label -p "err" "The partitionfile disk entry selected no disk, stopping the install" + exit 1 + fi +elif [ -e /tmp/xcat.install_disk.detected ]; then + mv /tmp/xcat.install_disk.detected /tmp/xcat.install_disk +fi + # Note that the subiquity/curtin installation config is fairly picky about # the order in which things are specified - you have to specify all the # partitions before you can specify any filesystems, which means this is @@ -211,7 +177,10 @@ base64decode() # to this unwrapped file and must start at column 0. logger -t $log_label -p "info" "Generate partition file..." if [ -e /tmp/xcat.install_disk ]; then - INSTALL_DISK=$(cat /tmp/xcat.install_disk) + # The layout uses one disk, the first one the disk file names. + INSTALL_DISK=$(awk 'NF {print $1; exit}' /tmp/xcat.install_disk) + # Subiquity matches the disk path literally, so resolve a link such as /dev/disk/by-id/. + INSTALL_DISK=$(readlink -f "$INSTALL_DISK" || echo "$INSTALL_DISK") else logger -t $log_label -p "err" "Failed to find partition file - getinstdisk must be run first" logger -t $log_label -p "err" "Using fallback default disk /dev/sda" @@ -385,8 +354,40 @@ storage: EOF fi +# Keep only a storage section that holds a config or a layout, rewritten as one YAML fragment +# for autoinstall.yaml. Subiquity puts LVM on the largest disk when that section is missing, +# and a layout without a match goes to the largest disk too, so it gets the disk in $2, if any. +# Some Subiquity releases ignore match in use_gap mode, so such a layout is refused with $2. +normalize_storage_layout() +{ + python3 -c 'import sys, yaml +s = yaml.safe_load(open(sys.argv[1])) +t = s.get("storage") if isinstance(s, dict) else None +if not (isinstance(t, dict) and (t.get("config") or t.get("layout"))): + sys.exit(1) +l = t.get("layout") +if sys.argv[2] and isinstance(l, dict): + if l.get("mode") == "use_gap": + sys.exit(1) + l.setdefault("match", {"path": sys.argv[2]}) +open(sys.argv[1], "w").write(yaml.safe_dump({"storage": t}, default_flow_style=False, sort_keys=False))' "$1" "$2" 2>/dev/null +} + +# A partitionfile layout or s: script must come after the default layout, which it replaces. +rm -f /tmp/partscript.enc +mv /tmp/partitionfile /tmp/partitionfile.default #XCA_PARTMAN_RECIPE_SCRIPT# -#XCA_PARTMAN_DISK_SCRIPT# +override_rc=$? +if [ -e /tmp/partscript.enc ]; then + # Keeping the default layout could wipe a disk the administrator layout protects. + if [ "$override_rc" -ne 0 ] || ! normalize_storage_layout /tmp/partitionfile "${disk_entry:+$INSTALL_DISK}"; then + logger -t $log_label -p "err" "The partitionfile layout entry wrote no usable layout, stopping the install" + rm -f /tmp/partitionfile + exit 1 + fi +else + mv /tmp/partitionfile.default /tmp/partitionfile +fi #XCA_PARTMAN_ADDITIONAL_CONFIG_SCRIPT# exit 0 diff --git a/xCAT-test/native/README.md b/xCAT-test/native/README.md index e875373802..8d03a9db8b 100644 --- a/xCAT-test/native/README.md +++ b/xCAT-test/native/README.md @@ -422,13 +422,14 @@ prove xCAT-test/native/install_profile_riscv64.t \ xCAT-test/native/netboot_profile_riscv64.t \ xCAT-test/native/netboot_profile_postinstall.t \ xCAT-test/native/builddebs_riscv64.t \ - xCAT-test/native/ubuntu_subiquity_template.t + xCAT-test/native/ubuntu_subiquity_template.t \ + xCAT-test/native/ubuntu_subiquity_install_disk.t ``` The renderers need the system Perl with the xCAT dependencies, XML::Simple, DBD::SQLite, Capture::Tiny, and File::Slurper. The sandbox clears the environment -and does not mount perlbrew or local::lib directories. The Subiquity case also -needs /usr/bin/python3 with PyYAML; on EL8, install python36 for that path. +and does not mount perlbrew or local::lib directories. The Subiquity cases also +need /usr/bin/python3 with PyYAML; on EL8, install python36 for that path. The Debian case needs debhelper, devscripts, fakeroot, and reprepro, and skips on non-Debian hosts. Each case creates its own temporary files and database. The temporary directory must allow execution; set TMPDIR if /tmp is noexec. diff --git a/xCAT-test/native/ubuntu_subiquity_install_disk.t b/xCAT-test/native/ubuntu_subiquity_install_disk.t new file mode 100644 index 0000000000..1131c30629 --- /dev/null +++ b/xCAT-test/native/ubuntu_subiquity_install_disk.t @@ -0,0 +1,137 @@ +#!/usr/bin/env perl +# The storage config a Subiquity install receives for each linuximage.partitionfile form. +use strict; +use warnings; +use FindBin; +use lib "$FindBin::Bin/../lib"; +use lib "$FindBin::Bin/../../perl-xCAT", "$FindBin::Bin/../../xCAT-server/lib/perl"; +use File::Path qw(make_path); +use File::Slurper qw(read_text write_text); +use Test::More; +use XCAT::Test::File qw(repo_path); +use XCAT::Test::Sandbox qw(sandbox_root sandbox_run); +use XCAT::Test::Template qw(template_database render_template); + +plan skip_all => 'installer execution requires Linux namespaces' unless $^O eq 'linux'; +my $database = template_database(); +# Subiquity runs its early commands under dash. +my ($shell) = grep { -x $_ } qw(/usr/bin/dash /bin/dash /bin/sh); +my $sequence = 0; + +sub render_part { + my ($script, $pattern, @entries) = @_; + my $dir = "$database/case-" . ++$sequence; + make_path($dir); + my @partitionfile; + for my $n (0 .. $#entries) { + my ($prefix, $contents) = @{ $entries[$n] }; + write_text("$dir/partition-$n", $contents); + push @partitionfile, "$prefix$dir/partition-$n"; + } + render_template($database, repo_path("xCAT-server/share/xcat/install/scripts/$script"), "$dir/node.pre", + 'node', '', '', '', join(',', @partitionfile), {xcatmaster => '192.0.2.1'}); + my ($part) = read_text("$dir/node.pre") =~ $pattern; + return $part; +} + +# getinstdisk has already chosen /dev/sdz. +sub installer_root { + my $root = sandbox_root(); + make_path("$root/efi", "$root/by-id"); + write_text("$root/bin/logger", "#!/bin/sh\nexit 0\n"); + chmod 0755, "$root/bin/logger" or die $!; + write_text("$root/tmp/xcat.install_disk", "/dev/sdz\n"); + write_text("$root/nvme7n1", ''); + symlink('../../nvme7n1', "$root/by-id/nvme-link") or die $!; + return $root; +} + +sub storage_config { + my ($root) = @_; + return -e "$root/tmp/partitionfile" ? read_text("$root/tmp/partitionfile") : ''; +} + +# Written as PyYAML writes it, so it reads the same before and after the pre-script rewrites it. +my $layout = "storage:\n version: 1\n config:\n - id: custom-disk\n type: disk\n path: /dev/disk/by-id/custom\n"; +my $writer = "#!/bin/sh\ncat >/tmp/partitionfile <<'EOF'\n${layout}EOF\n"; +my $diskscript = "#!/bin/sh\necho /dev/disk/by-id/scripted >/tmp/xcat.install_disk\n"; + +# Each case: label, the partitionfile entries as [prefix, file contents], and the disk the storage +# config must name, or undef when the install must stop before a storage config exists. A fourth +# value is the whole storage config of a layout. +my @cases = ( + ['no partitionfile', [], '/dev/sdz'], + ['a disk file', [['d:', "/dev/disk/by-id/nvme-pinned /dev/sdb\n"]], '/dev/disk/by-id/nvme-pinned'], + ['a disk file naming a link', [['d:', "/dev/disk/by-id/nvme-link\n"]], '/dev/nvme7n1'], + ['a disk script', [['s:d:', $diskscript]], '/dev/disk/by-id/scripted'], + ['a layout file', [['', $layout]], '/dev/disk/by-id/custom'], + ['a layout script', [['s:', $writer]], '/dev/disk/by-id/custom'], + ['a layout file after a layout script', [['s:', "#!/bin/sh\nexit 0\n"], ['', $layout]], '/dev/disk/by-id/custom'], + ['a disk file after a disk script', [['s:d:', "#!/bin/sh\nexit 0\n"], ['d:', "/dev/disk/by-id/nvme-pinned\n"]], + '/dev/disk/by-id/nvme-pinned'], + ['a layout file before a disk script', [['', $layout], ['s:d:', $diskscript]], '/dev/disk/by-id/custom'], + ['a layout script before a disk file', [['s:', $writer], ['d:', "/dev/disk/by-id/nvme-pinned\n"]], '/dev/disk/by-id/custom'], + ['a disk script that fails', [['s:d:', "#!/bin/sh\nexit 42\n"]], undef], + ['a disk script that selects no disk', [['s:d:', "#!/bin/sh\nexit 0\n"]], undef], + ['a layout script that fails', [['s:', "#!/bin/sh\nexit 42\n"]], undef], + ['a layout script that writes no layout', [['s:', "#!/bin/sh\nexit 0\n"]], undef], + ['a layout file with only blank lines', [['', "\n \n"]], undef], + ['a layout file with only comments', [['', "# no layout yet\n"]], undef], + ['a disk file with only blank lines', [['d:', " \n\n"]], undef], + ['a layout file with an empty storage section', [['', "storage:\n"]], undef], + ['a layout file with a null storage section', [['', "storage: null\n"]], undef], + ['a layout file that starts a YAML document', [['', "---\n$layout"]], '/dev/disk/by-id/custom'], + ['a layout without a match after a disk file', [['', "storage:\n layout:\n name: direct\n"], + ['d:', "/dev/disk/by-id/nvme-link\n"]], undef, + "storage:\n layout:\n name: direct\n match:\n path: /dev/nvme7n1\n"], + ['a layout with its own match after a disk file', [['', "storage:\n layout:\n name: direct\n match:\n ssd: true\n"], + ['d:', "/dev/disk/by-id/nvme-link\n"]], undef, + "storage:\n layout:\n name: direct\n match:\n ssd: true\n"], + ['a layout without a match and no disk entry', [['', "storage:\n layout:\n name: direct\n"]], undef, + "storage:\n layout:\n name: direct\n"], + ['a use_gap layout after a disk file', [['', "storage:\n layout:\n name: direct\n mode: use_gap\n"], + ['d:', "/dev/disk/by-id/nvme-link\n"]], undef], +); +for my $case (@cases) { + my ($label, $entries, $expected, $expected_layout) = @$case; + subtest $label => sub { + my $part = render_part('pre.ubuntu.subiquity', qr/(^base64decode\(\)\n.*^exit 0$)/ms, @$entries); + ok($part, 'the rendered script defines base64decode before the partitioning') or return; + my $root = installer_root(); + write_text("$root/part.sh", $part); + my ($rc, $output) = sandbox_run($root, {read_only => {"$root/efi" => '/sys/firmware/efi', + "$root/by-id" => '/dev/disk/by-id', "$root/nvme7n1" => '/dev/nvme7n1'}}, $shell, '/fixture/part.sh'); + if (defined $expected_layout) { + is($rc, 0, "the partitioning runs under $shell") or diag($output); + is(storage_config($root), $expected_layout, 'the storage config is the expected layout'); + return; + } + if (!defined $expected) { + isnt($rc, 0, "the pre-install script fails under $shell"); + ok(!-e "$root/tmp/partitionfile", 'and leaves no storage config to inject'); + return; + } + is($rc, 0, "the partitioning runs under $shell") or diag($output); + my $config = storage_config($root); + my ($disk) = $config =~ /^\s+type: disk\n(?:\s+\S+:.*\n)*?\s+path: (\S+)$/m; + is($disk, $expected, "the storage config partitions $expected"); + is($config, $layout, 'the administrator layout replaces the default one') if $expected eq '/dev/disk/by-id/custom'; + # The early command appends the storage config to Subiquity's autoinstall.yaml. + write_text("$root/autoinstall.yaml", "version: 1\nidentity:\n hostname: node\n$config"); + is(system('/usr/bin/python3', '-I', '-c', 'import sys, yaml; sys.exit("storage" not in yaml.safe_load(open(sys.argv[1])))', + "$root/autoinstall.yaml"), 0, 'autoinstall.yaml stays one YAML document with a storage section'); + }; +} + +# The debian-installer path renders the same entries into pre.ubuntu, under a shell with read -n. +subtest 'a debian-installer recipe file after a recipe script' => sub { + my $part = render_part('pre.ubuntu', qr/(^base64decode\(\)\n.*?^exit 0$)/ms, + ['s:', "#!/bin/sh\nexit 0\n"], ['', "recipe ::\n .\n"]); + ok($part, 'the rendered script defines base64decode before the recipe') or return; + my $root = installer_root(); + write_text("$root/part.sh", $part); + my ($rc, $output) = sandbox_run($root, '/bin/bash', '/fixture/part.sh'); + is($rc, 0, 'the recipe step runs') or diag($output); + is(storage_config($root), "recipe ::\n .\n", 'the later recipe file replaces the default recipe'); +}; +done_testing();