diff --git a/confluent_client/bin/nodeconfig b/confluent_client/bin/nodeconfig index 80732644..00764970 100755 --- a/confluent_client/bin/nodeconfig +++ b/confluent_client/bin/nodeconfig @@ -159,7 +159,7 @@ def parse_config_line(arguments, single=False): if setmode is None: setmode = True if setmode != True: - bailout('Cannot do set and query in same command') + bailout('Cannot do set and query in same command: Query detected but "{0}" appears to be set'.format(param)) if '=' in param: key, _, value = param.partition('=') _assign_value() @@ -171,7 +171,7 @@ def parse_config_line(arguments, single=False): if setmode is None: setmode = False if setmode != False: - bailout('Cannot do set and query in same command') + bailout('Cannot do set and query in same command: Set mode detected but "{0}" appears to be a query'.format(param)) if '.' not in param: if param == 'bmc': printallbmc = True diff --git a/confluent_osdeploy/confluent_osdeploy.spec.tmpl b/confluent_osdeploy/confluent_osdeploy.spec.tmpl index 7ff4f88b..a39e540a 100644 --- a/confluent_osdeploy/confluent_osdeploy.spec.tmpl +++ b/confluent_osdeploy/confluent_osdeploy.spec.tmpl @@ -25,7 +25,7 @@ cd utils make all cp copernicus clortho autocons ../opt/confluent/bin cd .. -for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04; do +for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 coreos; do mkdir ${os}out cd ${os}out cp -a ../opt . @@ -48,7 +48,7 @@ cp -a esxi7out esxi6out cp -a esxi7 esxi6 %install -for os in rhvh4 el7 el8 genesis suse15 ubuntu20.04 esxi6 esxi7; do +for os in rhvh4 el7 el8 genesis suse15 ubuntu20.04 esxi6 esxi7 coreos; do mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs diff --git a/confluent_osdeploy/coreos/initramfs/etc/systemd/system/initrd-root-fs.target.requires/confluent-rootfs.service b/confluent_osdeploy/coreos/initramfs/etc/systemd/system/initrd-root-fs.target.requires/confluent-rootfs.service new file mode 120000 index 00000000..4391533f --- /dev/null +++ b/confluent_osdeploy/coreos/initramfs/etc/systemd/system/initrd-root-fs.target.requires/confluent-rootfs.service @@ -0,0 +1 @@ +/usr/lib/systemd/system/confluent-rootfs.service \ No newline at end of file diff --git a/confluent_osdeploy/coreos/initramfs/opt/confluent/bin/initconfluent.sh b/confluent_osdeploy/coreos/initramfs/opt/confluent/bin/initconfluent.sh new file mode 100755 index 00000000..8ffeb722 --- /dev/null +++ b/confluent_osdeploy/coreos/initramfs/opt/confluent/bin/initconfluent.sh @@ -0,0 +1,106 @@ +#!/bin/bash +TRIES=0 +oum=$(umask) +umask 0077 +mkdir -p /etc/confluent +echo -n > /etc/confluent/confluent.info +umask $oum +cd /sys/class/net +while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do + TRIES=$((TRIES + 1)) + for currif in *; do + ip link set $currif up + done + /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info +done +cd / +grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0 # Do absolutely nothing if no data at all yet +echo -n "" > /tmp/confluent.initq +# restart cmdline +echo -n "" > /etc/cmdline.d/01-confluent.conf +nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') +cat /tls/*.pem > /etc/confluent/ca.pem +confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info|head -n 1 | awk '{print $2}') +if [[ $confluent_mgr == *%* ]]; then + echo $confluent_mgr | awk -F% '{print $2}' > /tmp/confluent.ifidx + ifidx=$(cat /tmp/confluent.ifidx) + ifname=$(ip link |grep ^$ifidx:|awk '{print $2}') + ifname=${ifname%:} +fi +needseal=1 +oldumask=$(umask) +umask 0077 +while [ -z "$confluent_apikey" ]; do + /opt/confluent/bin/clortho $nodename $confluent_mgr > /etc/confluent/confluent.apikey + if grep ^SEALED: /etc/confluent/confluent.apikey > /dev/null; then + needseal=0 + sed -e s/^SEALED:// /etc/confluent/confluent.apikey | clevis-decrypt-tpm2 > /etc/confluent/confluent.apikey.decrypt + mv /etc/confluent/confluent.apikey.decrypt /etc/confluent/confluent.apikey + fi + confluent_apikey=$(cat /etc/confluent/confluent.apikey) + if [ -z "$confluent_apikey" ]; then + echo "Unable to acquire node api key, no TPM2 sealed nor fresh token available, retrying..." + sleep 10 + fi +done +if [[ $confluent_mgr == *:* ]]; then + confluent_mgr="[$confluent_mgr]" +fi +if [ $needseal == 1 ]; then + sealed=$(echo $confluent_apikey | clevis-encrypt-tpm2 {}) + if [ ! -z "$sealed" ]; then + curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d $sealed https://$confluent_mgr/confluent-api/self/saveapikey + fi +fi +curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg +umask $oldumask +autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg) +autoconfigmethod=${autoconfigmethod#ipv4_method: } +confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg) +confluent_profile=${confluent_profile#profile: } + +if [ "$autoconfigmethod" = "dhcp" ]; then + echo ip=$ifname:dhcp >> /etc/cmdline.d/01-confluent.conf +else + v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg) + v4addr=${v4addr#ipv4_address: } + v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg) + v4gw=${v4gw#ipv4_gateway: } + if [ "$v4gw" = "null" ]; then + v4gw="" + fi + v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg) + v4nm=${v4nm#ipv4_netmask: } + echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf +fi +nameserversec=0 +while read -r entry; do + if [ $nameserversec = 1 ]; then + if [[ $entry == "-"* ]] && [[ $entry != "- ''" ]]; then + echo nameserver=${entry#- } >> /etc/cmdline.d/01-confluent.conf + continue + fi + fi + nameserversec=0 + if [ "${entry%:*}" = "nameservers" ]; then + nameserversec=1 + continue + fi +done < /etc/confluent/confluent.deploycfg + +if [ -e /lib/nm-lib.sh ]; then + . /lib/nm-lib.sh + nm_generate_connections + if [[ "$ifname" == ib* ]]; then + sed -i s/type=ethernet/type=infiniband/ /run/NetworkManager/system-connections/$ifname.nmconnection + if ! grep '\[infiniband\]' /run/NetworkManager/system-connections/$ifname.nmconnection > /dev/null; then + echo >> /run/NetworkManager/system-connections/$ifname.nmconnection + echo '[infiniband]' >> /run/NetworkManager/system-connections/$ifname.nmconnection + echo transport-mode=datagram >> /run/NetworkManager/system-connections/$ifname.nmconnection + fi + fi +fi +cat /proc/cmdline /etc/cmdline.d/01-confluent.conf | tr '\n' ' ' > /run/fakecmdline +mount -o bind /run/fakecmdline /proc/cmdline + +curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/rootfs.img | rdcore stream-hash /etc/coreos-live-want-rootfs | bsdtar -xf - -C / diff --git a/confluent_osdeploy/coreos/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh b/confluent_osdeploy/coreos/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh new file mode 100644 index 00000000..445c68ea --- /dev/null +++ b/confluent_osdeploy/coreos/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh @@ -0,0 +1,10 @@ +cat /tls/*.0 >> /etc/pki/tls/certs/ca-bundle.crt +if ! grep console= /proc/cmdline >& /dev/null; then + autocons=$(/opt/confluent/bin/autocons) + if [ -n "$autocons" ]; then + echo console=$autocons |sed -e 's!/dev/!!' >> /tmp/01-autocons.conf + autocons=${autocons%,*} + echo $autocons > /tmp/01-autocons.devnode + echo "Detected firmware specified console at $(cat /tmp/01-autocons.conf)" > $autocons + fi +fi diff --git a/confluent_osdeploy/coreos/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh b/confluent_osdeploy/coreos/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh new file mode 100644 index 00000000..e92f77e2 --- /dev/null +++ b/confluent_osdeploy/coreos/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# if ignition configuration is driving, defer to it +if [ ! -f /config.ign ]; then + + nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}') + confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info| head -n 1| awk '{print $2}' | sed -e s/%/%25/) + if [[ $confluent_mgr = *:* ]]; then + confluent_mgr=[$confluent_mgr] + fi + rootpassword=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg) + rootpassword=${rootpassword#rootpassword: } + if [ "$rootpassword" = "null" ]; then + rootpassword="" + fi + + if [ ! -z "$rootpassword" ]; then + sed -i "s@root:[^:]*:@root:$rootpassword:@" /sysroot/etc/shadow + fi + + mount -o bind /dev /sysroot/dev + chroot /sysroot ssh-keygen -A + umount /sysroot/dev + for i in /sysroot/etc/ssh/ssh_host*key.pub; do + certname=${i/.pub/-cert.pub} + curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -d @$i https://$confluent_mgr/confluent-api/self/sshcert > $certname + echo HostKey ${i%.pub} | sed -e 's!/sysroot!!' >> /sysroot/etc/ssh/sshd_config + echo HostCertificate $certname | sed -e 's!/sysroot!!' >> /sysroot/etc/ssh/sshd_config +done diff --git a/confluent_osdeploy/coreos/initramfs/usr/lib/systemd/system/confluent-rootfs.service b/confluent_osdeploy/coreos/initramfs/usr/lib/systemd/system/confluent-rootfs.service new file mode 100644 index 00000000..6e98786c --- /dev/null +++ b/confluent_osdeploy/coreos/initramfs/usr/lib/systemd/system/confluent-rootfs.service @@ -0,0 +1,19 @@ +[Unit] +Description=Confluent initialization +DefaultDependencies=false + +After=basic.target +# Network is enabled here +After=dracut-initqueue.service +Before=coreos-livepxe-rootfs.service + + +# If we fail, the boot will fail. Be explicit about it. +OnFailure=emergency.target +OnFailureJobMode=isolate + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/opt/confluent/bin/initconfluent.sh + diff --git a/confluent_osdeploy/coreos/profiles/default/initprofile.sh b/confluent_osdeploy/coreos/profiles/default/initprofile.sh new file mode 100644 index 00000000..3064dd73 --- /dev/null +++ b/confluent_osdeploy/coreos/profiles/default/initprofile.sh @@ -0,0 +1,8 @@ +#!/bin/sh +ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \ +ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution && \ +ln -s $1/images/ignition.img $2/boot/initramfs/ignition.img && \ +mkdir -p $2/boot/efi/boot/ && \ +ln -s $1/images/pxeboot/rootfs.img $2/ && \ +mcopy -i $1/images/efiboot.img ::efi/redhat/grubx64.efi $2/boot/efi/boot/ && \ +mcopy -i $1/images/efiboot.img ::efi/boot/bootx64.efi $2/boot/efi/boot/ \ No newline at end of file diff --git a/confluent_osdeploy/coreos/profiles/default/profile.yaml b/confluent_osdeploy/coreos/profiles/default/profile.yaml new file mode 100644 index 00000000..839f33a9 --- /dev/null +++ b/confluent_osdeploy/coreos/profiles/default/profile.yaml @@ -0,0 +1,5 @@ +label: RedHat CoreOS %%VERSION%% %%ARCH%% (Default Profile) +kernelargs: quiet random.trust_cpu=on ignition.firstboot ignition.platform.id=metal +#the above boots and will have ignition agent phone home and await instructions +#kernelargs: quiet random.trust_cpu=on ignition.firstboot ignition.platform.id=metal coreos.inst=yes coreos.inst.install_dev=sda +#the above runs the installer to write to disk \ No newline at end of file diff --git a/confluent_osdeploy/diskless/build/dracut/install b/confluent_osdeploy/diskless/build/dracut/install index 9799ff13..29a0671b 100644 --- a/confluent_osdeploy/diskless/build/dracut/install +++ b/confluent_osdeploy/diskless/build/dracut/install @@ -5,7 +5,7 @@ dracut_install curl openssl tar cpio gzip lsmod ethtool xz lsmod ethtool dracut_install modprobe touch echo cut wc bash netstat uniq grep ip hostname dracut_install awk egrep dirname expr sort dracut_install ssh sshd reboot parted mkfs mkfs.ext4 mkfs.xfs xfs_db mkswap -dracut_install efibootmgr +dracut_install efibootmgr uuidgen dracut_install du df ssh-keygen scp clear dhclient dracut_install /lib64/libnss_dns-2.28.so /lib64/libnss_dns.so.2 dracut_install /usr/lib64/libnl-3.so.200 @@ -15,13 +15,7 @@ inst /bin/bash /bin/sh dracut_install /lib64/libfuse.so.2 /lib64/libfuse.so.2.9.7 dracut_install chown chroot dd expr kill parted rsync sort blockdev findfs insmod lvm dracut_install /usr/lib/udev/rules.d/10-dm.rules /usr/sbin/dmsetup /usr/lib/udev/rules.d/95-dm-notify.rules - - - - - - - +dracut_install /usr/lib/udev/rules.d/60-net.rules /lib/udev/rename_device /usr/lib/systemd/network/99-default.link #this would be nfs with lock, but not needed, go nolock #dracut_install mount.nfs rpcbind rpc.statd /etc/netconfig sm-notify diff --git a/confluent_osdeploy/diskless/build/dracut/installkernel b/confluent_osdeploy/diskless/build/dracut/installkernel index c2e9cfdb..5e953464 100644 --- a/confluent_osdeploy/diskless/build/dracut/installkernel +++ b/confluent_osdeploy/diskless/build/dracut/installkernel @@ -5,7 +5,7 @@ instmods nvme instmods cdc_ether instmods mptctl instmods mlx4_ib mlx5_ub ib_umad ib_ipoib -instmods i40e hfi1 bnxt_en qed qede +instmods ice i40e hfi1 bnxt_en qed qede instmods dm-mod dm-log raid0 raid1 raid10 raid456 dm-raid dm-thin-pool dm-crypt dm-snapshot linear dm-era # nfs root and optionally gocryptfs instmods fuse overlay squashfs loop diff --git a/confluent_osdeploy/diskless/build/genimage b/confluent_osdeploy/diskless/build/genimage index 0240aede..50b49e57 100644 --- a/confluent_osdeploy/diskless/build/genimage +++ b/confluent_osdeploy/diskless/build/genimage @@ -1,4 +1,5 @@ installroot=$1 +destdir=$2 touch $installroot/.testcap if setcap cap_net_raw+p $installroot/.testcap >& /dev/null; then @@ -7,9 +8,29 @@ else capsargs="--setopt=tsflags=nocapps" fi rm $installroot/.testcap -yum $capsargs --releasever=8 --installroot=/var/tmp/testgenimage/ --releasever=8.3 install '@minimal-environment' chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils shim-x64 grub2-efi-x64 -dracut -v --xz -N -m "stateless base terminfo" -f boot/initramfs-diskless 4.18.0-240.22.1.el8_3.x86_64 +pkglist=$(cat $(dirname $0)/pkglist | tr "\r\n" " ") +mydir=$(dirname $0) +mkdir -p $installroot/proc $installroot/sys $installroot/dev +unshare -f -p -m bash -c " +mount -o bind /proc $installroot/proc +mount -o bind /sys $installroot/sys +mount -o bind /dev $installroot/dev +yum -y $capsargs --releasever=8 --installroot=$installroot install $pkglist +cp -a $dirname/dracut $installroot/usr/lib/dracut/modules.d/97diskless +chmod a+x $installroot/usr/lib/dracut/modules.d/97diskless/* +for kernel in $(ls $installroot/boot/vmlinuz-*|grep -v rescue|sed -e s/.*vmlinuz-//); do + echo -n "Creating diskless initramfs for $kernel" + chroot $installroot dracut -v --xz -N -m "diskless base terminfo" -f boot/initramfs-diskless-$kernel.img $kernel +done +" +latestkernel=$(ls $installroot/boot/vmlinuz-*|grep -v rescue|sed -e s/.*vmlinuz-//|tail -n 1) +mkdir -p $destdir/boot/efi/boot $destdir/boot/initramfs +cp $installroot/boot/vmlinuz-$latestkernel $destdir/boot/kernel +cp $installroot/boot/initramfs-diskless-$latestkernel.img $destdir/boot/initramfs/distribution +cp $installroot/boot/efi/EFI/BOOT/BOOTX64.EFI $destdir/boot/efi/boot +cp $installroot/boot/efi/EFI/centos/grubx64.efi $destdir/boot/efi/boot + # link kernel, initrd, grub, and shim as appropriate # use xz, minimize https burden and transfer penalty -mksquashfs -comp xz +mksquashfs $installroot $destdir/rootimg.sfs -comp xz diff --git a/confluent_osdeploy/diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh b/confluent_osdeploy/diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh index 03dbcb75..aa3dea0d 100644 --- a/confluent_osdeploy/diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh +++ b/confluent_osdeploy/diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh @@ -62,6 +62,8 @@ if [[ $confluent_mgr == *%* ]]; then ifname=${ifname%:} fi needseal=1 +oldumask=$(umask) +umask 0077 while [ -z "$confluent_apikey" ]; do /opt/confluent/bin/clortho $nodename $confluent_mgr > /etc/confluent/confluent.apikey if grep ^SEALED: /etc/confluent/confluent.apikey > /dev/null; then @@ -85,6 +87,7 @@ if [ $needseal == 1 ]; then fi fi curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg +umask $oldumask autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg |awk '{print $2}') if [ "$autoconfigmethod" = "dhcp" ]; then echo -n "Attempting to use dhcp to bring up $ifname..." @@ -106,7 +109,62 @@ else if [ ! -z "$v4gw" ]; then ip route add default via $v4gw fi + mkdir -p /run/NetworkManager/system-connections + cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC +[connection] +id=eno1 +EOC + echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection + cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC +type=ethernet +autoconnect-retries=1 +EOC + echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection + cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC +multi-connect=1 +permissions= +wait-device-timeout=60000 + +[ethernet] +mac-address-blacklist= + +[ipv4] +EOC + echo address1=$v4addr/$v4nm >> /run/NetworkManager/system-connections/$ifname.nmconnection + if [ ! -z "$v4gw" ]; then + echo gateway=$v4gw >> /run/NetworkManager/system-connections/$ifname.nmconnection + fi + nameserversec=0 + nameservers="" + while read -r entry; do + if [ $nameserversec = 1 ]; then + if [[ $entry == "-"* ]]; then + nameservers="$nameservers"${entry#- }";" + continue + fi + fi + nameserversec=0 + if [ "${entry%:*}" = "nameservers" ]; then + nameserversec=1 + continue + fi + done < /etc/confluent/confluent.deploycfg + echo dns=$nameservers >> /run/NetworkManager/system-connections/$ifname.nmconnection + dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg) + dnsdomain=${dnsdomain#dnsdomain: } + echo dns-search=$dnsdomain >> /run/NetworkManager/system-connections/$ifname.nmconnection + cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC +may-fail=false +method=manual + +[ipv6] +addr-gen-mode=eui64 +method=auto + +[proxy] +EOC fi +chmod 600 /run/NetworkManager/system-connections/*.nmconnection echo -n "Initializing ssh..." ssh-keygen -A for pubkey in /etc/ssh/ssh_host*key.pub; do @@ -129,8 +187,10 @@ for addr in $(grep ^MANAGER: /etc/confluent/confluent.info|awk '{print $2}'|sed confluent_urls="$confluent_urls $confluent_proto://$addr/confluent-public/os/$confluent_profile/rootimg.sfs" fi done +confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}') +confluent_urls="$confluent_urls https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs" mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay -curlmount $confluent_urls /mnt/remoteimg +/opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg mount -o loop,ro /mnt/remoteimg/*.sfs /mnt/remote mount -t tmpfs overlay /mnt/overlay mkdir -p /mnt/overlay/upper /mnt/overlay/work @@ -139,6 +199,7 @@ mkdir -p /sysroot/etc/ssh mkdir -p /sysroot/etc/confluent mkdir -p /sysroot/root/.ssh cp /root/.ssh/* /sysroot/root/.ssh +chmod 700 /sysroot/root/.ssh cp /etc/confluent/* /sysroot/etc/confluent/ cp /etc/ssh/*key* /sysroot/etc/ssh/ for pubkey in /etc/ssh/ssh_host*key.pub; do @@ -149,6 +210,7 @@ for pubkey in /etc/ssh/ssh_host*key.pub; do fi echo HostKey $privfile >> /sysroot/etc/ssh/sshd_config done + mkdir -p /sysroot/dev /sysroot/sys /sysroot/proc /sysroot/run if [ ! -z "$autocons" ]; then autocons=${autocons%,*} @@ -162,6 +224,39 @@ while [ ! -e /sysroot/sbin/init ]; do sleep 1 done done +rootpassword=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg) +rootpassword=${rootpassword#rootpassword: } +if [ "$rootpassword" = "null" ]; then + rootpassword="" +fi + +if [ ! -z "$rootpassword" ]; then + sed -i "s@root:[^:]*:@root:$rootpassword:@" /sysroot/etc/shadow +fi +for i in /ssh/*.ca; do + echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts +done +echo HostbasedAuthentication yes >> /sysroot/etc/ssh/sshd_config +echo HostbasedUsesNameFromPacketOnly yes >> /sysroot/etc/ssh/sshd_config +echo IgnoreRhosts no >> /sysroot/etc/ssh/sshd_config +sshconf=/sysroot/etc/ssh/ssh_config +if [ -d /sysroot/etc/ssh/ssh_config.d/ ]; then + sshconf=/sysroot/etc/ssh/ssh_config.d/01-confluent.conf +fi +echo 'Host *' >> $sshconf +echo ' HostbasedAuthentication yes' >> $sshconf +echo ' EnableSSHKeysign yes' >> $sshconf +echo ' HostbasedKeyTypes *ed25519*' >> $sshconf +curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv +cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts +chmod 640 /sysroot/etc/ssh/*_key +chroot /sysroot chgrp ssh_keys /etc/ssh/*_key +chroot /sysroot cat /etc/confluent/ca.pem >> /etc/pki/tls/certs/ca-bundle.crt +curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service +mkdir -p /sysroot/opt/confluent/bin +curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh +chmod +x /sysroot/opt/confluent/bin/onboot.sh +ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service exec /opt/confluent/bin/start_root diff --git a/confluent_osdeploy/diskless/profile/common/scripts/onboot.service b/confluent_osdeploy/diskless/profile/common/scripts/onboot.service new file mode 100644 index 00000000..f9235033 --- /dev/null +++ b/confluent_osdeploy/diskless/profile/common/scripts/onboot.service @@ -0,0 +1,11 @@ +[Unit] +Description=Confluent onboot hook +Requires=network-online.target +After=network-online.target + +[Service] +ExecStart=/opt/confluent/bin/onboot.sh + +[Install] +WantedBy=multi-user.target + diff --git a/confluent_osdeploy/diskless/profile/common/scripts/onboot.sh b/confluent_osdeploy/diskless/profile/common/scripts/onboot.sh new file mode 100644 index 00000000..9f26fb6e --- /dev/null +++ b/confluent_osdeploy/diskless/profile/common/scripts/onboot.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# This script is executed on each boot as it is +# completed. It is best to edit the middle of the file as +# noted below so custom commands are executed before +# the script notifies confluent that install is fully complete. + +nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') +confluent_apikey=$(cat /etc/confluent/confluent.apikey) +confluent_mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}') +confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}') +export nodename confluent_mgr confluent_profile +. /etc/confluent/functions +exec >> /var/log/confluent/confluent-onboot.log +exec 2>> /var/log/confluent/confluent-onboot.log +tail -f /var/log/confluent/confluent-onboot.log > /dev/console & +logshowpid=$! + +run_remote onboot.custom +# onboot scripts may be placed into onboot.d, e.g. onboot.d/01-firstaction.sh, onboot.d/02-secondaction.sh +run_remote_parts onboot.d + +# Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/ +run_remote_config onboot.d + +#curl -X POST -d 'status: booted' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus +kill $logshowpid diff --git a/confluent_server/confluent/discovery/protocols/pxe.py b/confluent_server/confluent/discovery/protocols/pxe.py index 625ab99b..e0d1719f 100644 --- a/confluent_server/confluent/discovery/protocols/pxe.py +++ b/confluent_server/confluent/discovery/protocols/pxe.py @@ -276,6 +276,13 @@ def proxydhcp(): bootfile = b'confluent/x86_64/ipxe.efi' elif disco['arch'] == 'bios-x86': bootfile = b'confluent/x86_64/ipxe.kkpxe' + if len(bootfile) > 127: + log.log( + {'info': 'Boot offer cannot be made to {0} as the ' + 'profile name "{1}" is {2} characters longer than is supported ' + 'for this boot method.'.format( + node, profile, len(bootfile) - 127)}) + continue rpv[:240] = rqv[:240].tobytes() rpv[0:1] = b'\x02' rpv[108:108 + len(bootfile)] = bootfile @@ -485,6 +492,13 @@ def check_reply(node, info, packet, sock, cfg, reqview): ) if not isinstance(bootfile, bytes): bootfile = bootfile.encode('utf8') + if len(bootfile) > 127: + log.log( + {'info': 'Boot offer cannot be made to {0} as the ' + 'profile name "{1}" is {2} characters longer than is supported ' + 'for this boot method.'.format( + node, profile, len(bootfile) - 127)}) + return repview[108:108 + len(bootfile)] = bootfile repview[20:24] = myipn gateway = None diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index 52efacbb..6db8820e 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -23,6 +23,7 @@ READFILES = set([ 'media.2/products', '.DISCINFO', '.discinfo', + 'zipl.prm', ]) HEADERSUMS = set([b'\x85\xeddW\x86\xc5\xbdhx\xbe\x81\x18X\x1e\xb4O\x14\x9d\x11\xb7C8\x9b\x97R\x0c-\xb8Ht\xcb\xb3']) @@ -424,6 +425,37 @@ def _priv_check_oraclelinux(isoinfo): return {'name': 'oraclelinux-{0}-{1}'.format(ver, arch), 'method': EXTRACT, 'category': 'el{0}'.format(major)} + +def fixup_coreos(targpath): + # the efi boot image holds content that the init script would want + # to mcopy, but the boot sector is malformed usually, so change it to 1 + # sector per track + if os.path.exists(targpath + '/images/efiboot.img'): + with open(targpath + '/images/efiboot.img', 'rb+') as bootimg: + bootimg.seek(0x18) + if bootimg.read != b'\x00\x00': + bootimg.seek(0x18) + bootimg.write(b'\x01') + + +def check_coreos(isoinfo): + arch = 'x86_64' # TODO: would check magic of vmlinuz to see which arch + if 'zipl.prm' in isoinfo[1]: + prodinfo = isoinfo[1]['zipl.prm'] + if not isinstance(prodinfo, str): + prodinfo = prodinfo.decode('utf8') + for inf in prodinfo.split(): + if inf.startswith('coreos.liveiso=rhcos-'): + ver = inf.split('-')[1] + return {'name': 'rhcos-{0}-{1}'.format(ver, arch), + 'method': EXTRACT, 'category': 'coreos'} + elif inf.startswith('coreos.liveiso=fedore-coreos-'): + ver = inf.split('-')[2] + return {'name': 'fedoracoreos-{0}-{1}'.format(ver, arch), + 'method': EXTRACT, 'category': 'coreos'} + + + def check_rhel(isoinfo): ver = None arch = None @@ -502,7 +534,7 @@ def fingerprint(archive): if fun.startswith('check_'): name = globals()[fun](isoinfo) if name: - return name, isoinfo[0] + return name, isoinfo[0], fun.replace('check_', '') return None else: sum = hashlib.sha256(header) @@ -514,7 +546,7 @@ def fingerprint(archive): chunk = archive.read(32768) imginfo = HASHPRINTS.get(sum.hexdigest(), None) if imginfo: - return imginfo, None + return imginfo, None, None def import_image(filename, callback, backend=False, mfd=None): @@ -525,7 +557,7 @@ def import_image(filename, callback, backend=False, mfd=None): identity = fingerprint(archive) if not identity: return -1 - identity, imginfo = identity + identity, imginfo, funname = identity targpath = identity['name'] distpath = '/var/lib/confluent/distributions/' + targpath if identity.get('subname', None): @@ -566,6 +598,8 @@ def import_image(filename, callback, backend=False, mfd=None): del identity['subname'] with open(distpath + '/distinfo.yaml', 'w') as distinfo: distinfo.write(yaml.dump(identity, default_flow_style=False)) + if 'fixup_{0}'.format(funname) in globals(): + globals()['fixup_{0}'.format(funname)](targpath) callback({'progress': 1.0}) sys.stdout.write('\n') @@ -648,7 +682,7 @@ class MediaImporter(object): if not identity: raise exc.InvalidArgumentException('Unsupported Media') self.percent = 0.0 - identity, _ = identity + identity, _, _ = identity self.phase = 'copying' if not identity: raise Exception('Unrecognized OS Media') diff --git a/confluent_server/confluent/sshutil.py b/confluent_server/confluent/sshutil.py index 141f2c2a..29ff4442 100644 --- a/confluent_server/confluent/sshutil.py +++ b/confluent_server/confluent/sshutil.py @@ -11,6 +11,16 @@ import tempfile agent_pid = None ready_keys = {} +_sshver = None + +def sshver(): + global _sshver + if _sshver is None: + p = subprocess.Popen(['ssh', '-V'], stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + _, output = p.communicate() + _sshver = float(output.split()[0].split(b'_')[1].split(b'p')[0]) + return _sshver def normalize_uid(): curruid = os.geteuid() @@ -23,6 +33,8 @@ def normalize_uid(): def assure_agent(): + if sshver() <= 7.6: + return global agent_pid if agent_pid is None: sai = subprocess.check_output(['ssh-agent']) @@ -41,6 +53,8 @@ def assure_agent(): os.environ[k] = v def get_passphrase(): + if sshver() <= 7.6: + return '' # convert the master key to base64 # for use in ssh passphrase context if cfm._masterkey is None: @@ -106,8 +120,9 @@ def sign_host_key(pubkey, nodename, principals=()): principals = set(principals) principals.add(nodename) principals = ','.join(sorted(principals)) + flags = '-Us' if sshver() > 7.6 else '-s' subprocess.check_call( - ['ssh-keygen', '-Us', '/etc/confluent/ssh/ca.pub', '-I', nodename, + ['ssh-keygen', flags, '/etc/confluent/ssh/ca.pub', '-I', nodename, '-n', principals, '-h', pkeyname]) certname = pkeyname.replace('.pub', '-cert.pub') with open(certname) as cert: diff --git a/genesis/97genesis/installkernel b/genesis/97genesis/installkernel index 8e58ed24..7bc20a35 100644 --- a/genesis/97genesis/installkernel +++ b/genesis/97genesis/installkernel @@ -4,5 +4,5 @@ instmods nvme instmods cdc_ether instmods mptctl instmods mlx4_ib mlx5_ub ib_umad ib_ipoib -instmods i40e hfi1 bnxt_en qed qede +instmods ice i40e hfi1 bnxt_en qed qede instmods dm-mod dm-log raid0 raid1 raid10 raid456 dm-raid dm-thin-pool dm-crypt dm-snapshot linear dm-era diff --git a/misc/urlmount.c b/misc/urlmount.c new file mode 100644 index 00000000..174d1d85 --- /dev/null +++ b/misc/urlmount.c @@ -0,0 +1,242 @@ +/* +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2021 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/ + +#define FUSE_USE_VERSION 26 +#include +#include +#include +#include +#include +#include +#include + +CURL *curl; + +char curlerror[CURL_ERROR_SIZE]; +curl_off_t filesize; + +typedef struct downloadbuffer { + char *response; + size_t completed; + size_t total; +} downloadbuffer; + +#define MAX_FILE_LEN 1024 +#define MAX_URL_PATHS 512 +static char filename[MAX_FILE_LEN]; +static int urlidx; +static char* urls[MAX_URL_PATHS]; + + +static int http_readdir(const char *path, void *buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info *fi) +{ + if (strcmp(path, "/") != 0) // We don't support subdirs + return -ENOENT; + filler(buf, ".", NULL, 0); + filler(buf, "..", NULL, 0); + filler(buf, filename + 1, NULL, 0); + return 0; +} + +size_t fill_buffer(char *data, size_t size, size_t nmemb, downloadbuffer *userdata) { + size_t amount; + amount = size * nmemb; + if (userdata->total < amount + userdata->completed) return 0; + memcpy(&(userdata->response[userdata->completed]), data, amount); + userdata->completed += amount; + return amount; +} + +static int http_read(const char *path, char *buf, size_t size, off_t offset, + struct fuse_file_info *fi) { + char headbuffer[512]; + double dldbl = 0.0; + int startidx; + int reconnecting = 0; + FILE* fd; + startidx = urlidx; + memset(buf, 0, size); + curl_off_t downloaded; + //Would be needed for multithread, however preferring to conserve + //filehandles rather than go multithread + // Some comparisons showed that the threaded performance boost doesn't + // do even offset the overhead of the new curl handles, so better + // to use single threaded curl overall for now + //CURL *tmpcurl = curl_easy_duphandle(curl); + downloadbuffer dlbuf; + dlbuf.response = buf; + dlbuf.completed = 0; + dlbuf.total = size; + fd = NULL; + + if (strcmp(path, filename) != 0) return -ENOENT; + memset(headbuffer, 0, 512); + snprintf(headbuffer, 512, "%ld-%ld", offset, offset + size - 1); + curl_easy_setopt(curl, CURLOPT_RANGE, headbuffer); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &dlbuf); + while (curl_easy_perform(curl) != CURLE_OK) { + reconnecting = 1; + fd = fopen("/dev/kmsg", "w+"); + dlbuf.completed = 0; + fprintf(fd, "<1>urlmount: error while communicating with %s: %s\n", urls[urlidx], curlerror); + urlidx++; + if (urls[urlidx] == NULL) + urlidx = 0; + if (urlidx == startidx) + sleep(10); + fprintf(fd, "urlmount: Connecting to %s\n", urls[urlidx]); + curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]); + fclose(fd); + } + if (reconnecting) { + fd = fopen("/dev/kmsg", "w+"); + fprintf(fd, "<1>urlmount: Successfully connected to %s\n", urls[urlidx]); + fclose(fd); + } + curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &dldbl); + downloaded = round(dldbl); + //Would be needed for multithread + //curl_easy_cleanup(tmpcurl); + return downloaded; +} + +static int http_open(const char *path, struct fuse_file_info *fi) { + if (strcmp(path, filename) != 0) + return -ENOENT; + + if ((fi->flags & 3) != O_RDONLY) + return -EACCES; + + return 0; +} + +static void* http_init(struct fuse_conn_info *conn) { + // Because we fork, we need to redo curl + // or else suffer the wrath of NSS TLS + curl_global_init(CURL_GLOBAL_DEFAULT); + curl = curl_easy_init(); + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror); + //We want to consider error conditions fatal, rather than + //passing error text as data + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L); + curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fill_buffer); + return NULL; +} + +static int http_getattr(const char *path, struct stat *st) { + memset(st, 0, sizeof(struct stat)); + + if (strcmp(path, "/") == 0) { + st->st_mode = S_IFDIR | 0555; + st->st_nlink = 2; + } else if (strcmp(path, filename) == 0) { + st->st_mode = S_IFREG | 0444; + st->st_nlink = 1; + st->st_size = filesize; // TODO: fix with curl HEAD + } else + return -ENOENT; + return 0; +} + + +static const struct fuse_operations http_ops = { + .getattr = http_getattr, + .readdir = http_readdir, + .read = http_read, + .open = http_open, + .init = http_init, +}; + +int main(int argc, char* argv[]) { + char *tmp; + double fsize; + unsigned int i; + int j; + j = open("/dev/urandom", O_RDONLY); + if (j <= 0 || read(j, (char*)&i, 4) < 0) { + i = time(NULL); + } + if (j > 0) { + close(j); + } + srand(i); + j = 0; + memset(urls, 0, 32*sizeof(char*)); + urlidx = 0; + curl_global_init(CURL_GLOBAL_DEFAULT); + curl = curl_easy_init(); + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror); + //We want to consider error conditions fatal, rather than + //passing error text as data + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 15L); + memset(filename, 0, MAX_FILE_LEN); + for (i=0; i < argc; i++) { + if (strstr(argv[i], ":") > 0) { + if (j < MAX_URL_PATHS) { + urls[j] = argv[i]; + tmp = strrchr(urls[j++], '/'); + strncpy(filename, tmp, MAX_FILE_LEN); + } + //Request single threaded mode, as curl would need more + // filehandles for multithread + argv[i] = "-s"; + } + } + if (filename[0] == 0) { + fprintf(stderr, "No URL given in arguments\n"); + exit(1); + } + for (i=0; urls[i] != NULL; i++) { + printf("Registering mount path: %s\n", urls[i]); + } + urlidx = rand() % j; + j = urlidx; + printf("Connecting to %s\n", urls[urlidx]); + curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]); + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + while (curl_easy_perform(curl) != CURLE_OK) { + fprintf(stderr, "urlmount: error while communicating with %s: %s\n", urls[urlidx++], curlerror); + if (urls[urlidx] == NULL) + urlidx = 0; + if (urlidx == j) { + fprintf(stderr, "urlmount: Unable to reach any target url, aborting\n"); + exit(1); + } + printf("Connecting to %s\n", urls[urlidx]); + curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]); + } + printf("Successfully connected to %s\n", urls[urlidx]); + curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &fsize); + filesize = round(fsize); + curl_easy_setopt(curl, CURLOPT_NOBODY, 0); + if (filesize < 1) { + fprintf(stderr, "Unable to reach designated URL\n"); + exit(1); + } + if (!curl) { + fprintf(stderr, "Unable to initialize CURL!\n"); + exit(1); + } + curl_easy_cleanup(curl); + curl_global_cleanup(); + fuse_main(argc, argv, &http_ops, NULL); +}