mirror of
https://github.com/xcat2/confluent.git
synced 2026-11-21 09:52:24 +00:00
0839299bd4
An untethered or uncompressed diskless boot downloaded the root image only from the deployment server and retried it forever when that server failed. Try the other managers that answered discovery as well, as the tethered urlmount already does. urlmount limits each small range request to 10 seconds, but the single whole-image download cannot take a total limit. Give up on a connect after 10 seconds and on a transfer that stalls for 30 seconds instead, so a hung server moves on to the next one.
258 lines
12 KiB
Bash
258 lines
12 KiB
Bash
. /lib/dracut-lib.sh
|
|
confluent_whost=$confluent_mgr
|
|
if [[ "$confluent_whost" == *:* ]] && [[ "$confluent_whost" != "["* ]]; then
|
|
confluent_whost="[$confluent_mgr]"
|
|
fi
|
|
mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay /sysroot
|
|
imagemethod=$(getarg confluent_imagemethod)
|
|
if grep installtodisk /proc/cmdline > /dev/null; then
|
|
# installimage copies from the mounted image, which an untethered or
|
|
# uncompressed boot would extract to RAM and unmount first
|
|
imagemethod=tethered
|
|
fi
|
|
if [ "untethered" = "$imagemethod" -o "uncompressed" = "$imagemethod" ]; then
|
|
mount -t tmpfs untethered /mnt/remoteimg
|
|
# Fall back to the other managers when the deployment server fails
|
|
while ! for url in https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs $confluent_urls; do curl -f --connect-timeout 10 --speed-limit 1 --speed-time 30 $url -o /mnt/remoteimg/rootimg.sfs && break; done; do
|
|
echo "Failed downloading image, retrying..."
|
|
sleep $((RANDOM%3)).$((RANDOM))
|
|
done
|
|
else
|
|
confluent_urls="$confluent_urls https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs"
|
|
/opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg
|
|
fi
|
|
/opt/confluent/bin/confluent_imginfo /mnt/remoteimg/rootimg.sfs > /tmp/rootimg.info
|
|
loopdev=$(losetup -f)
|
|
export mountsrc=$loopdev
|
|
losetup -r $loopdev /mnt/remoteimg/rootimg.sfs
|
|
if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then
|
|
while ! curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_whost/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key; do
|
|
echo "Unable to retrieve private key from $confluent_mgr (verify that confluent can access /var/lib/confluent/private/os/$confluent_profile/pending/rootimg.key)"
|
|
sleep 1
|
|
done
|
|
cipher=$(head -n 1 /tmp/rootimg.key)
|
|
key=$(tail -n 1 /tmp/rootimg.key)
|
|
len=$(wc -c /mnt/remoteimg/rootimg.sfs | awk '{print $1}')
|
|
len=$(((len-4096)/512))
|
|
dmsetup create cryptimg --table "0 $len crypt $cipher $key 0 $loopdev 8"
|
|
/opt/confluent/bin/confluent_imginfo /dev/mapper/cryptimg > /tmp/rootimg.info
|
|
mountsrc=/dev/mapper/cryptimg
|
|
fi
|
|
|
|
rootimgformat=$(awk -F': ' '/^Format:/ {print $2; exit}' /tmp/rootimg.info)
|
|
if [ "$rootimgformat" = squashfs ]; then
|
|
mount -o ro $mountsrc /mnt/remote
|
|
elif [ "$rootimgformat" = confluent_multisquash ]; then
|
|
tail -n +3 /tmp/rootimg.info | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh
|
|
. /tmp/setupmount.sh
|
|
cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh
|
|
. /tmp/mountparts.sh
|
|
fi
|
|
|
|
|
|
#mount -t tmpfs overlay /mnt/overlay
|
|
if [ ! "uncompressed" = "$imagemethod" ]; then
|
|
modprobe zram
|
|
memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
|
|
memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
|
|
echo $memtot > /sys/block/zram0/disksize
|
|
mkfs.xfs /dev/zram0 > /dev/null
|
|
fi
|
|
TETHERED=0
|
|
if [ "untethered" = "$imagemethod" -o "uncompressed" = "$imagemethod" ]; then
|
|
if [ "untethered" = "$imagemethod" ]; then
|
|
mount -o discard /dev/zram0 /sysroot
|
|
else
|
|
mount -t tmpfs disklessroot /sysroot
|
|
fi
|
|
extract_untethered_rootimg
|
|
else
|
|
TETHERED=1
|
|
mount -o discard /dev/zram0 /mnt/overlay
|
|
if [ ! -f /tmp/mountparts.sh ]; then
|
|
mkdir -p /mnt/overlay/upper /mnt/overlay/work
|
|
mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
|
|
else
|
|
for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
|
|
mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
|
|
mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
|
|
done
|
|
fi
|
|
fi
|
|
mkdir -p /sysroot/etc/ssh
|
|
mkdir -p /sysroot/etc/confluent
|
|
mkdir -p /sysroot/root/.ssh
|
|
cp /root/.ssh/* /sysroot/root/.ssh
|
|
chmod 700 /sysroot/root/.ssh
|
|
cp /etc/confluent/* /sysroot/etc/confluent/
|
|
cp /etc/ssh/*key* /sysroot/etc/ssh/
|
|
for pubkey in /etc/ssh/ssh_host*key.pub; do
|
|
certfile=${pubkey/.pub/-cert.pub}
|
|
privfile=${pubkey%.pub}
|
|
if [ -s $certfile ]; then
|
|
echo HostCertificate $certfile >> /sysroot/etc/ssh/sshd_config
|
|
fi
|
|
echo HostKey $privfile >> /sysroot/etc/ssh/sshd_config
|
|
done
|
|
|
|
mkdir -p /sysroot/dev /sysroot/sys /sysroot/proc /sysroot/run
|
|
if [ ! -z "$autocons" ]; then
|
|
autocons=${autocons%,*}
|
|
mkdir -p /run/systemd/generator/getty.target.wants
|
|
ln -s /usr/lib/systemd/system/serial-getty@.service /run/systemd/generator/getty.target.wants/serial-getty@${autocons}.service
|
|
fi
|
|
while [ ! -e /sysroot/sbin/init ]; do
|
|
echo "Failed to access root filesystem or it is missing /sbin/init"
|
|
echo "System should be accessible through ssh at port 2222 with the appropriate key"
|
|
while [ ! -e /sysroot/sbin/init ]; do
|
|
sleep 1
|
|
done
|
|
done
|
|
rootpassword=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg)
|
|
rootpassword=${rootpassword#rootpassword: }
|
|
if [ "$rootpassword" = "null" ]; then
|
|
rootpassword=""
|
|
fi
|
|
|
|
if [ ! -z "$rootpassword" ]; then
|
|
sed -i "s@root:[^:]*:@root:$rootpassword:@" /sysroot/etc/shadow
|
|
fi
|
|
for i in /ssh/*.ca; do
|
|
echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
|
|
done
|
|
echo HostbasedAuthentication yes >> /sysroot/etc/ssh/sshd_config
|
|
echo HostbasedUsesNameFromPacketOnly yes >> /sysroot/etc/ssh/sshd_config
|
|
echo IgnoreRhosts no >> /sysroot/etc/ssh/sshd_config
|
|
sshconf=/sysroot/etc/ssh/ssh_config
|
|
if [ -d /sysroot/etc/ssh/ssh_config.d/ ]; then
|
|
sshconf=/sysroot/etc/ssh/ssh_config.d/01-confluent.conf
|
|
fi
|
|
echo 'Host *' >> $sshconf
|
|
echo ' HostbasedAuthentication yes' >> $sshconf
|
|
echo ' EnableSSHKeysign yes' >> $sshconf
|
|
echo ' HostbasedKeyTypes *ed25519*' >> $sshconf
|
|
curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_whost/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
|
|
cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
|
|
if grep ^ssh_keys: /etc/group > /dev/null; then
|
|
chmod 640 /sysroot/etc/ssh/*_key
|
|
chroot /sysroot chgrp ssh_keys /etc/ssh/*_key
|
|
fi
|
|
cp /tls/*.pem /sysroot/etc/pki/ca-trust/source/anchors/
|
|
chroot /sysroot/ update-ca-trust
|
|
curl -sf https://$confluent_whost/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
|
|
mkdir -p /sysroot/opt/confluent/bin
|
|
curl -sf https://$confluent_whost/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh
|
|
chmod +x /sysroot/opt/confluent/bin/onboot.sh
|
|
cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin
|
|
set_selinux_context system_u:object_r:systemd_unit_file_t:s0 /sysroot/etc/systemd/system/onboot.service
|
|
set_selinux_context system_u:object_r:bin_t:s0 /sysroot/opt/confluent/bin/onboot.sh
|
|
set_selinux_context system_u:object_r:bin_t:s0 /sysroot/opt/confluent/bin/apiclient
|
|
ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service
|
|
cp /etc/confluent/functions /sysroot/etc/confluent/functions
|
|
rm -f /sysroot/etc/dracut.conf.d/diskless.conf # remove diskless dracut from runtime, to make kdump happier
|
|
if [ -x /sysroot/usr/sbin/matchpathcon ] && grep -Eq '^SELINUX=(enforcing|permissive)' /sysroot/etc/selinux/config 2> /dev/null; then
|
|
# Files written to /sysroot before policy load are unlabeled.
|
|
selinuxpolicy=$(sed -n 's/^SELINUXTYPE=//p' /sysroot/etc/selinux/config)
|
|
fcontexts=/etc/selinux/$selinuxpolicy/contexts/files/file_contexts
|
|
if [ $TETHERED -eq 1 ]; then
|
|
# Label only the upper layer, using globstar since the initramfs has no find.
|
|
shopt -s globstar dotglob nullglob
|
|
relabel=()
|
|
if [ -f /tmp/mountparts.sh ]; then
|
|
overlayparts=$(awk '{print $3}' /tmp/mountparts.sh)
|
|
else
|
|
overlayparts=/mnt/remote
|
|
fi
|
|
for srcmount in $overlayparts; do
|
|
upperdir=/mnt/overlay/upper
|
|
if [ -f /tmp/mountparts.sh ]; then
|
|
upperdir=/mnt/overlay${srcmount}/upper
|
|
fi
|
|
prefix=${srcmount#/mnt/remote}
|
|
prefix=${prefix%/}
|
|
relabel+=("${prefix:-/}")
|
|
for relpath in "$upperdir"/**; do
|
|
relabel+=("$prefix${relpath#$upperdir}")
|
|
done
|
|
done
|
|
shopt -u globstar dotglob nullglob
|
|
# Not setfiles (recurses into the image) or restorecon (no-op before policy load).
|
|
chroot /sysroot /usr/sbin/matchpathcon -f $fcontexts "${relabel[@]}" | while IFS=$'\t' read -r relpath context; do
|
|
# deleted files leave whiteouts in the upper layer
|
|
if [ "$context" != "<<none>>" ] && { [ -e "/sysroot$relpath" ] || [ -L "/sysroot$relpath" ]; }; then
|
|
chcon -h "$context" "/sysroot$relpath"
|
|
fi
|
|
done
|
|
elif [ -x /sysroot/usr/bin/python3 ] && [ -x /sysroot/usr/sbin/setfiles ]; then
|
|
# The root is a local copy: label every inode without a label, which
|
|
# also covers sockets that unsquashfs creates without attributes.
|
|
# setfiles only recurses into directories that are new here.
|
|
chroot /sysroot /usr/bin/python3 -c '
|
|
import errno, os, sys
|
|
out = sys.stdout.buffer
|
|
rootdev = os.lstat(b"/").st_dev
|
|
def check(path):
|
|
try:
|
|
os.getxattr(path, b"security.selinux", follow_symlinks=False)
|
|
except OSError as e:
|
|
if e.errno == errno.ENODATA:
|
|
out.write(path + b"\0")
|
|
check(b"/")
|
|
for root, dirs, files in os.walk(b"/"):
|
|
for name in dirs + files:
|
|
check(os.path.join(root, name))
|
|
dirs[:] = [d for d in dirs if os.lstat(os.path.join(root, d)).st_dev == rootdev]
|
|
' | chroot /sysroot /usr/sbin/setfiles -0 -F -f - $fcontexts
|
|
relabelstatus=("${PIPESTATUS[@]}")
|
|
if [ "${relabelstatus[0]}" != 0 ] || [ "${relabelstatus[1]}" != 0 ]; then
|
|
echo "Warning: SELinux labeling of the root filesystem failed"
|
|
fi
|
|
else
|
|
echo "Warning: python3 or setfiles missing in the image, files written at boot stay unlabeled"
|
|
fi
|
|
fi
|
|
if grep installtodisk /proc/cmdline > /dev/null; then
|
|
. /etc/confluent/functions
|
|
run_remote installimage
|
|
exec reboot -f
|
|
fi
|
|
mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
|
|
ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
|
|
if [ -d /lib/firmware ]; then
|
|
mv /lib/firmware /lib/firmware-ramfs
|
|
fi
|
|
ln -s /sysroot/lib/firmware /lib/firmware
|
|
kill $(grep -l ^/usr/lib/systemd/systemd-udevd /proc/*/cmdline|cut -d/ -f 3)
|
|
if grep debugssh /proc/cmdline >& /dev/null; then
|
|
debugssh=1
|
|
else
|
|
debugssh=0
|
|
fi
|
|
if [ $TETHERED -eq 1 ]; then
|
|
# In tethered mode, the double-caching is useful to get through tricky part of
|
|
# onboot with confignet. After that, it's excessive cache usage.
|
|
# Give the onboot script a hook to have us come in and enable directio to the
|
|
# squashfs and drop the cache of the rootimg so far
|
|
(
|
|
sleep 86400 &
|
|
ONBOOTPID=$!
|
|
mkdir -p /run/confluent
|
|
echo $ONBOOTPID > /run/confluent/onboot_sleep.pid
|
|
wait $ONBOOTPID
|
|
dd if=/mnt/remoteimg/rootimg.sfs iflag=nocache count=0 >& /dev/null
|
|
if [ $debugssh -eq 0 ]; then
|
|
rm -rf /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs /lib/firmware-ramfs /usr/lib64/libcrypto.so* /usr/lib64/systemd/ /kernel/ /usr/bin/ /usr/sbin/ /usr/libexec/
|
|
fi
|
|
) &
|
|
while [ ! -f /run/confluent/onboot_sleep.pid ]; do
|
|
sleep 0.1
|
|
done
|
|
elif [ $debugssh -eq 0 ]; then
|
|
rm -rf /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs /lib/firmware-ramfs /usr/lib64/libcrypto.so* /usr/lib64/systemd/ /kernel/ /usr/bin/ /usr/sbin/ /usr/libexec/
|
|
fi
|
|
if grep debugssh /proc/cmdline >& /dev/null; then
|
|
exec /opt/confluent/bin/start_root
|
|
else
|
|
exec /opt/confluent/bin/start_root -s # share mount namespace, keep kernel callbacks intact
|
|
fi
|