From af66f78dfd05adc488e8e89d4d1dfc66c6267911 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:35:07 -0300 Subject: [PATCH] fix(xcat-core): Genesis on Ubuntu answers no ssh, so every case that reaches a node fails A compute node booted into legacy Genesis on Ubuntu refuses every ssh connection: [xcat100-mn]: xcat100-cn: ssh: connect to host xcat100-cn port 22: Connection refused and on the node's own console, once per boot: Missing privilege separation directory Debian and Ubuntu build sshd with --with-privsep-path=/run/sshd. The payload's own sshd carries that path and the format string "Missing privilege separation directory: %s", and the image ships /run holding only rpcbind. So sshd exits at startup and the node is unreachable. nodeset_shell, nodeset_cmdline, nodeset_runimg and genesis_boot_proof_legacy all fail on it, while seq_discovery, which needs no ssh, passes. doxcat already does this for the other family: it creates /usr/share/empty.sshd, the EL privsep path, with the comment "On some systems /usr/share/empty.sshd needs to be created before calling sshd". The Debian path was never added, although the tree knows about it twice -- remoteshell says "prepare the /var/run/sshd for ubuntu" and etc/init.d/functions creates it. This adds the same lines for /run/sshd and /var/run/sshd. Created immediately before sshd starts, not baked into the image: /run is a tmpfs on a running system, so a directory under it does not survive the mount. The mode is set whether or not the directory had to be created, because sshd refuses a group or world writable privsep directory with the SAME message as an absent one, so a wrong mode is indistinguishable from this bug. genesis_sshd_privsep.bats asserts what sshd imposes -- the path is a directory, owned by the user that will run sshd, and not group or world writable -- read at the moment sshd runs. The recorder is bound over /usr/sbin/sshd because doxcat calls that absolute path, and because recording from a later command cannot see the ORDER: with the recorder on a PATH command instead, moving the directories after the sshd line still passed. Five mutations, each red: removing the loop fails 2, never setting the mode fails 1, dropping /var/run/sshd fails 2, a group-writable mode fails 2, and creating the directories after sshd starts fails 2. The whole bats suite goes from 21 failures to 19 with this change, the difference being these two tests; the other 19 are unrelated and fail on the base. doxcat_sandbox.bash gains chmod, and a third test keeps the next one from being missed. The sandbox curates PATH without /usr/bin, so a command it does not copy resolves to nothing and fails SILENTLY: doxcat's chmod did nothing under test for as long as this sandbox has existed, and any assertion about a mode could never pass. The guard installs bash's command_not_found_handle through BASH_ENV and fails naming whatever doxcat invoked and could not resolve, so it needs no list to keep current. With chmod removed again it reports exactly "chmod". Its reach is the commands doxcat runs on the path the test drives, which is a floor rather than a completeness check on the helper's list. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- xCAT-genesis-scripts/usr/bin/doxcat | 12 ++ xCAT-test/bats/genesis_sshd_privsep.bats | 138 +++++++++++++++++++++ xCAT-test/bats/helpers/doxcat_sandbox.bash | 5 +- 3 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 xCAT-test/bats/genesis_sshd_privsep.bats diff --git a/xCAT-genesis-scripts/usr/bin/doxcat b/xCAT-genesis-scripts/usr/bin/doxcat index 2338e3337b..0e451050c2 100755 --- a/xCAT-genesis-scripts/usr/bin/doxcat +++ b/xCAT-genesis-scripts/usr/bin/doxcat @@ -100,6 +100,18 @@ fi if [ ! -e /usr/share/empty.sshd ]; then mkdir -p /usr/share/empty.sshd fi +# Debian and Ubuntu build sshd with --with-privsep-path=/run/sshd, and sshd exits with +# "Missing privilege separation directory" when it is absent. The image carries /run and not +# /run/sshd, so Genesis on Ubuntu answered no ssh and every case that reaches a booted node +# over xdsh failed. Created here rather than baked into the image: /run is a tmpfs on a +# running system, and a directory under it does not survive the mount. +# The mode is set whether or not the directory had to be created: sshd refuses a privsep +# directory that is group or world writable, with the same message, so a wrong mode looks +# exactly like an absent one. +for sshd_privsep in /run/sshd /var/run/sshd; do + mkdir -p "$sshd_privsep" + chmod 0755 "$sshd_privsep" +done echo 'Protocol 2' >> /etc/ssh/sshd_config /usr/sbin/sshd mkdir -p /etc/xcat diff --git a/xCAT-test/bats/genesis_sshd_privsep.bats b/xCAT-test/bats/genesis_sshd_privsep.bats new file mode 100644 index 0000000000..25f2f48f7f --- /dev/null +++ b/xCAT-test/bats/genesis_sshd_privsep.bats @@ -0,0 +1,138 @@ +#!/usr/bin/env bats +# +# The privilege separation directory sshd needs, at the moment doxcat starts sshd. +# +# Debian and Ubuntu build sshd with --with-privsep-path=/run/sshd. The Genesis image carries +# /run and not /run/sshd, so sshd exited with "Missing privilege separation directory" and the +# node answered no ssh at all. Every legacy Genesis case that reaches a booted node over xdsh +# failed on "Connection refused" while sequential discovery, which needs no ssh, passed. +# +# doxcat already did this for the other family -- it creates /usr/share/empty.sshd, the EL +# privsep path -- so this is the same line for Debian. The assertions are what SSHD IMPOSES, +# read at the moment sshd runs: the path is a directory, its owner is the user that will run +# sshd, and it is not group or world writable. sshd refuses a loose mode with the same message +# as an absent directory, so a wrong mode is indistinguishable from the bug. +# +# Read from inside the sandbox on purpose. /run there is a tmpfs, exactly as it is on a running +# system, so a directory made before that mount -- or baked into the image -- is not there when +# sshd starts and cannot be seen from outside afterwards. + +load 'helpers/doxcat_sandbox' + +PRIVSEP_PATHS='/run/sshd /var/run/sshd' + +# Record what each privsep path looks like AT THE MOMENT sshd runs. +# +# doxcat runs sshd as /usr/sbin/sshd, an absolute path, so a stub on PATH is never reached and +# the recorder is bound over that path instead. Recording from a later PATH command instead -- +# openssl is the next one -- cannot see the ORDER: moving the directories after the sshd line +# then passes, which is the one mutation that matters most here. +record_privsep() +{ + local stub="${BATS_TEST_TMPDIR}/sshd-recorder" + + cat >"$stub" <>/run/fixture/sshd-privsep + else + printf '%s absent\n' "\$d" >>/run/fixture/sshd-privsep + fi +done +exit 0 +SH + chmod 0755 "$stub" + sandbox+=(--ro-bind "$stub" /usr/sbin/sshd) +} + +seen() +{ + local path="$1" + awk -v p="$path" '$1 == p { print; exit }' "$fixture/sshd-privsep" +} + +@test "doxcat makes every privilege separation directory before it starts sshd" { + setup_doxcat + record_privsep + add_doxcat_link eth0 BROADCAST,MULTICAST,UP ether 02:bb:cc:dd:ee:ff + run_genesis 0 --setenv BOOTIF 01-02-bb-cc-dd-ee-ff + + # The recording ran, so it happened one statement after sshd was started. + [ -s "$fixture/sshd-privsep" ] + + for path in $PRIVSEP_PATHS; do + local line mode owner + line=$(seen "$path") + [ -n "$line" ] + [ "$(printf '%s\n' "$line" | cut -d' ' -f2)" = dir ] + mode=$(printf '%s\n' "$line" | cut -d' ' -f3) + owner=$(printf '%s\n' "$line" | cut -d' ' -f4) + # Owned by whoever runs sshd: root in Genesis, the invoking user here. ls in + # the sandbox has no passwd lookup, so the owner comes back numeric. + [ "$owner" = "$(id -u)" ] + # sshd refuses a group or world writable privsep directory. Positions 6 and 9 of the + # mode string are the group and other write bits. + [ "$(printf '%s\n' "$mode" | cut -c6)" = - ] + [ "$(printf '%s\n' "$mode" | cut -c9)" = - ] + done +} + +@test "a privilege separation directory left group writable is corrected, not kept" { + # The directory can already exist from an earlier boot with a mode sshd will not accept, + # and that failure prints the same message as an absent one. So the mode is set whether or + # not doxcat had to create the directory. + setup_doxcat + record_privsep + add_doxcat_link eth0 BROADCAST,MULTICAST,UP ether 02:bb:cc:dd:ee:ff + # /var is bound from the fixture, so this one survives into the sandbox. + mkdir -p "$fixture/var/run/sshd" + chmod 0777 "$fixture/var/run/sshd" + run_genesis 0 --setenv BOOTIF 01-02-bb-cc-dd-ee-ff + + local mode + mode=$(printf '%s\n' "$(seen /var/run/sshd)" | cut -d' ' -f3) + [ -n "$mode" ] + [ "$(printf '%s\n' "$mode" | cut -c6)" = - ] + [ "$(printf '%s\n' "$mode" | cut -c9)" = - ] +} + +@test "every command doxcat runs resolves inside the sandbox" { + # The sandbox curates PATH and does NOT include /usr/bin, so a command it does not copy + # resolves to nothing. That fails silently: doxcat's chmod did nothing under test for as + # long as this sandbox has existed, and an assertion about a mode could never pass. The + # mode half of the privsep fix was unmeasurable until chmod was added to the helper. + # + # Derived, not a list: bash calls command_not_found_handle for anything it cannot resolve, + # so this records whatever doxcat actually invokes and goes missing. A list here would go + # stale the next time doxcat gains a command. + # + # Its reach is the commands doxcat runs ON THE PATH THIS TEST DRIVES, which is not every + # command in the script -- a branch this boot does not take is not covered. It is a floor, + # not a completeness check on the helper's list. + setup_doxcat + record_privsep + local env_file="${BATS_TEST_TMPDIR}/bash_env" + cat >"$env_file" <<'SH' +command_not_found_handle() +{ + printf '%s\n' "$1" >>/run/fixture/not-found + return 127 +} +SH + sandbox+=(--ro-bind "$env_file" /run/bash_env --setenv BASH_ENV /run/bash_env) + add_doxcat_link eth0 BROADCAST,MULTICAST,UP ether 02:bb:cc:dd:ee:ff + run_genesis 0 --setenv BOOTIF 01-02-bb-cc-dd-ee-ff + + # The handler was installed, so an absence would have been recorded. + [ -s "$fixture/sshd-privsep" ] + if [ -e "$fixture/not-found" ]; then + printf 'doxcat runs commands the sandbox does not provide:\n%s\n' \ + "$(sort -u "$fixture/not-found")" >&2 + return 1 + fi +} diff --git a/xCAT-test/bats/helpers/doxcat_sandbox.bash b/xCAT-test/bats/helpers/doxcat_sandbox.bash index cbab15c095..559de3aa81 100644 --- a/xCAT-test/bats/helpers/doxcat_sandbox.bash +++ b/xCAT-test/bats/helpers/doxcat_sandbox.bash @@ -10,8 +10,11 @@ setup_doxcat() fips=$(repo_path xCAT-genesis-scripts/usr/lib/xcat/fips.sh) [ -r "$fips" ] mkdir -p "$fixture"/{etc/ssh,etc/pki/tls,real,links,addrs} + # chmod is in this list because doxcat runs it: the sandbox has no /usr/bin on PATH, so a + # command that is missing here is a command that silently does nothing under test. doxcat's + # chmod of the sshd privilege separation directory was not exercised until it was added. for utility in bash sh cat cp awk grep sed sort tr tail rm wc cut head uname \ - timeout env ls mkdir touch readlink basename sleep; do + timeout env ls mkdir touch readlink basename sleep chmod; do cp -L "$(PATH=/usr/bin:/bin type -P "$utility")" "$fixture/bin/$utility" done if [ -x /usr/bin/coreutils ]; then