From b2feb62d8a94452bc904ba7c0a67def08281d1c6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 May 2022 09:25:49 -0400 Subject: [PATCH] Add ubuntu22.04 profile Ubuntu 22.04 makes some changes, notably removing the custom-installation hooks. Change to injecting our modifications more directly to where the custom-installation hooks used to be. --- .../initramfs/conf/conf.d/confluent | 8 + .../initramfs/custom-installation/post.sh | 7 + .../initramfs/custom-installation/pre.sh | 6 + .../scripts/casper-bottom/99confluent | 75 +++++ .../initramfs/scripts/init-premount/confluent | 31 ++ .../profiles/default/autoinstall/meta-data | 0 .../profiles/default/autoinstall/user-data | 17 ++ .../profiles/default/initprofile.sh | 7 + .../ubuntu22.04/profiles/default/profile.yaml | 3 + .../default/scripts/firstboot.d/.gitignore | 0 .../profiles/default/scripts/firstboot.sh | 21 ++ .../profiles/default/scripts/functions | 196 +++++++++++++ .../profiles/default/scripts/getinstalldisk | 88 ++++++ .../default/scripts/post.d/.gitignore | 0 .../profiles/default/scripts/post.sh | 85 ++++++ .../profiles/default/scripts/pre.sh | 29 ++ .../profiles/default/scripts/syncfileclient | 272 ++++++++++++++++++ 17 files changed, 845 insertions(+) create mode 100644 confluent_osdeploy/ubuntu22.04/initramfs/conf/conf.d/confluent create mode 100755 confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/post.sh create mode 100755 confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/pre.sh create mode 100644 confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent create mode 100755 confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/meta-data create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/profile.yaml create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.d/.gitignore create mode 100755 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/functions create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.d/.gitignore create mode 100755 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh create mode 100755 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/conf/conf.d/confluent b/confluent_osdeploy/ubuntu22.04/initramfs/conf/conf.d/confluent new file mode 100644 index 00000000..8d11e9f5 --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/initramfs/conf/conf.d/confluent @@ -0,0 +1,8 @@ +if ! grep console= /proc/cmdline > /dev/null; then + /opt/confluent/bin/autocons > /custom-installation/autocons.info + cons=$(cat /custom-installation/autocons.info) + if [ ! -z "$cons" ]; then + echo "Auto-detected serial console: $cons" > ${cons%,*} + fi +fi +echo /scripts/init-premount/confluent >> /scripts/init-premount/ORDER diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/post.sh b/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/post.sh new file mode 100755 index 00000000..5bd43bc6 --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/post.sh @@ -0,0 +1,7 @@ +#!/bin/bash +deploycfg=/custom-installation/confluent/confluent.deploycfg +confluent_mgr=$(grep ^deploy_server $deploycfg|awk '{print $2}') +confluent_profile=$(grep ^profile: $deploycfg|awk '{print $2}') +export deploycfg confluent_mgr confluent_profile +curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/post.sh > /tmp/post.sh +. /tmp/post.sh diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/pre.sh b/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/pre.sh new file mode 100755 index 00000000..70def99c --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/pre.sh @@ -0,0 +1,6 @@ +#!/bin/bash +deploycfg=/custom-installation/confluent/confluent.deploycfg +confluent_mgr=$(grep ^deploy_server $deploycfg|awk '{print $2}') +confluent_profile=$(grep ^profile: $deploycfg|awk '{print $2}') +curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/pre.sh > /tmp/pre.sh +. /tmp/pre.sh diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent b/confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent new file mode 100644 index 00000000..fc68c7fb --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent @@ -0,0 +1,75 @@ +cp -a /custom-installation/ /root/custom-installation +cd /root +cat /tls/*.0 >> /root/etc/ssl/certs/ca-certificates.crt +mkdir -p /root/custom-installation/ssh +mkdir -p /root/custom-installation/tls +cp /ssh/* /root/custom-installation/ssh +cp /tls/* /root/custom-installation/tls +NODENAME=$(grep ^NODENAME: /custom-installation/confluent/confluent.info|awk '{print $2}') +MGR=$(grep ^EXTMGRINFO: /custom-installation/confluent/confluent.info |awk -F'|' '{print $1 " " $3}'|grep " 1$" | head -n 1 | awk '{print $2}') +MGR=$(grep ^MANAGER: /custom-installation/confluent/confluent.info|head -n 1| awk '{print $2}') +MGTIFACE=$(grep $MGR /custom-installation/confluent/confluent.info | grep ^EXTMGRINFO: | head -n 1 | awk -F'|' '{print $2}') +oum=$(umask) +umask 077 +chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey +MGR=[$MGR] +deploycfg=/root/custom-installation/confluent/confluent.deploycfg +if [ -z "$MGTIFACE" ]; then + chroot . usr/bin/curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg +else + chroot . usr/bin/curl -f -H "CONFLUENT_MGTIFACE: $MGTIFACE" -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg +fi +umask $oum +nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1) +nic=$(ip link |grep ^$nic:|awk '{print $2}') +DEVICE=${nic%:} +ipv4m=$(grep ^ipv4_method $deploycfg|awk '{print$2}') +. /scripts/functions +if [ "$ipv4m" = "dhcp" ]; then + IP=dhcp + configure_networking +elif [ "$ipv4m" = "static" ]; then + v4addr=$(grep ^ipv4_address: $deploycfg) + v4addr=${v4addr#ipv4_address: } + v4gw=$(grep ^ipv4_gateway: $deploycfg) + v4gw=${v4gw#ipv4_gateway: } + if [ "$v4gw" = "null" ]; then + v4gw="" + fi + v4nm=$(grep ipv4_netmask: $deploycfg) + v4nm=${v4nm#ipv4_netmask: } + dnsdomain=$(grep ^dnsdomain: $deploycfg) + dnsdomain=${dnsdomain#dnsdomain: } + if [ "$dnsdomain" = "null" ]; then dnsdomain=""; fi + dns=$(grep -A1 ^nameservers: $deploycfg|head -n 2|tail -n 1|sed -e 's/^- //'|sed -e "s/''//") + { + echo "DEVICE='$DEVICE'" + echo "PROTO='none'" + echo "IPV4PROTO='none'" + echo "IPV4ADDR='$v4addr'" + echo "IPV4NETMASK='$v4nm'" + echo "IPV4BROADCAST='$v4nm'" + echo "IPV4GATEWAY='$v4gw'" + echo "IPV4DNS1='$dns'" + echo "HOSTNAME='$NODENAME'" + echo "DNSDOMAIN='$dnsdomain'" + echo "DOMAINSEARCH='$dnsdomain'" + } > "/run/net-$DEVICE.conf" + configure_networking +else + IP=off +fi +ipv4s=$(grep ^deploy_server $deploycfg|awk '{print $2}') +osprofile=$(cat /custom-installation/confluent/osprofile) +fcmdline="$(cat /custom-installation/confluent/cmdline.orig) autoinstall ds=nocloud-net;s=https://${ipv4s}/confluent-public/os/${osprofile}/autoinstall/" +if [ -f /custom-installation/autocons.info ]; then + cons=$(cat /custom-installation/autocons.info) +fi +if [ ! -z "$cons" ]; then + echo "Installation will proceed on graphics console, autoconsole not supported during autoinstall phase" > ${cons%,*} + echo "Progress can be checked by using ssh to access and running the screendump command" > ${cons%,*} + echo ${cons%,*} > /root/tmp/autoconsdev + #fcmdline="$fcmdline console=${cons#/dev/}" +fi +echo $fcmdline > /custom-installation/confluent/fakecmdline +/scripts/casper-bottom/58server_network diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent b/confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent new file mode 100755 index 00000000..ffc13c1b --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent @@ -0,0 +1,31 @@ +cd /sys/class/net +for nic in *; do + ip link set $nic up +done +mkdir -p /custom-installation +cp -a /opt/confluent /custom-installation +touch /custom-installation/confluent/confluent.info +while ! grep NODENAME /custom-installation/confluent/confluent.info; do + /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info +done +MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]" +osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline) +cat /proc/cmdline > /custom-installation/confluent/cmdline.orig +if [ -f /custom-installation/autocons.info ]; then + cons=$(cat /custom-installation/autocons.info) +fi +if [ ! -z "$cons" ]; then + echo "Preparing to deploy $osprofile from $MGR" > ${cons%,*} +fi +echo "Preparing to deploy $osprofile from $MGR" +echo $osprofile > /custom-installation/confluent/osprofile +echo URL=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf +fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso" +if [ ! -z "$cons" ]; then + fcmdline="$fcmdline console=${cons#/dev/}" +fi +echo $fcmdline > /custom-installation/confluent/fakecmdline +mount -o bind /custom-installation/confluent/fakecmdline /proc/cmdline +echo '/scripts/casper-bottom/99confluent "$@"' >> /scripts/casper-bottom/ORDER + + diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/meta-data b/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/meta-data new file mode 100644 index 00000000..e69de29b diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data b/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data new file mode 100644 index 00000000..5b6c9894 --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data @@ -0,0 +1,17 @@ +#cloud-config +autoinstall: + version: 1 + early-commands: + - /custom-installation/pre.sh + late-commands: + - /custom-installation/post.sh + ssh: + install-server: true + storage: + layout: + name: lvm + match: + path: "%%INSTALLDISK%%" + user-data: + runcmd: + - /etc/confluent/firstboot.sh diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh new file mode 100644 index 00000000..20e12471 --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh @@ -0,0 +1,7 @@ +#!/bin/sh +sed -i 's/label: ubuntu/label: Ubuntu/' $2/profile.yaml && \ +ln -s $1/casper/vmlinuz $2/boot/kernel && \ +ln -s $1/casper/initrd $2/boot/initramfs/distribution && \ +mkdir -p $2/boot/efi/boot && \ +ln -s $1/EFI/boot/* $2/boot/efi/boot + diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/profile.yaml b/confluent_osdeploy/ubuntu22.04/profiles/default/profile.yaml new file mode 100644 index 00000000..b76cdfbf --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/profile.yaml @@ -0,0 +1,3 @@ +label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile) +kernelargs: quiet osprofile=%%PROFILE%% +#installedargs: example # These arguments would be added to the installed system diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.d/.gitignore b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.d/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh new file mode 100755 index 00000000..d90d5f7f --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh @@ -0,0 +1,21 @@ +#!/bin/bash +echo "Confluent first boot is running" +cp -a /etc/confluent/ssh/* /etc/ssh/ +systemctl restart sshd +rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}') +if [ ! -z "$rootpw" -a "$rootpw" != "null" ]; then + echo root:$rootpw | chpasswd -e +fi +nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}') +confluent_apikey=$(cat /etc/confluent/confluent.apikey) +confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg |awk '{print $2}') +while ! ping -c 1 $confluent_mgr >& /dev/null; do + sleep 1 +done +hostnamectl set-hostname $(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}') +touch /etc/cloud/cloud-init.disabled +source /etc/confluent/functions + +run_remote_parts firstboot.d +run_remote_config firstboot.d +curl --capath /etc/confluent/tls -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/functions b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/functions new file mode 100644 index 00000000..d70c63db --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/functions @@ -0,0 +1,196 @@ +#!/bin/bash +function test_mgr() { + if curl -s https://${1}/confluent-api/ > /dev/null; then + return 0 + fi + return 1 +} + +function confluentpython() { + if [ -x /usr/libexec/platform-python ]; then + /usr/libexec/platform-python $* + elif [ -x /usr/bin/python3 ]; then + /usr/bin/python3 $* + elif [ -x /usr/bin/python ]; then + /usr/bin/python $* + elif [ -x /usr/bin/python2 ]; then + /usr/bin/python2 $* + fi +} + +function set_confluent_vars() { + if [ -z "$nodename" ]; then + nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}') + fi + if [[ "$confluent_mgr" == *"%"* ]]; then + confluent_mgr="" + fi + if [ -z "$confluent_mgr" ]; then + confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') + if ! test_mgr $confluent_mgr; then + confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') + if [[ "$confluent_mgr" = *":"* ]]; then + confluent_mgr="[$confluent_mgr]" + fi + fi + if ! test_mgr $confluent_mgr; then + BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//') + OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//') + for confluent_mgr in $BESTMGRS $OKMGRS; do + if [[ $confluent_mgr == *":"* ]]; then + confluent_mgr="[$confluent_mgr]" + fi + if test_mgr $confluent_mgr; then + break + fi + done + fi + fi + if [ -z "$confluent_profile" ]; then + confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') + fi +} + +fetch_remote() { + curlargs="" + if [ -f /etc/confluent/ca.pem ]; then + curlargs=" --cacert /etc/confluent/ca.pem" + fi + set_confluent_vars + mkdir -p $(dirname $1) + curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1 + if [ $? != 0 ]; then echo $1 failed to download; return 1; fi +} + +source_remote_parts() { + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + apiclient=/opt/confluent/bin/apiclient + if [ -f /etc/confluent/apiclient ]; then + apiclient=/etc/confluent/apiclient + fi + scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //') + for script in $scriptlist; do + source_remote $1/$script + done + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir +} + +run_remote_parts() { + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + apiclient=/opt/confluent/bin/apiclient + if [ -f /etc/confluent/apiclient ]; then + apiclient=/etc/confluent/apiclient + fi + scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //') + for script in $scriptlist; do + run_remote $1/$script + done + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir +} + +source_remote() { + set_confluent_vars + unsettmpdir=0 + echo + echo '---------------------------------------------------------------------------' + echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ + if [ -z "$confluentscripttmpdir" ]; then + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + unsettmpdir=1 + fi + echo Sourcing from $confluentscripttmpdir + cd $confluentscripttmpdir + fetch_remote $1 + if [ $? != 0 ]; then echo $1 failed to download; return 1; fi + chmod +x $1 + cmd=$1 + shift + source ./$cmd + cd - > /dev/null + if [ "$unsettmpdir" = 1 ]; then + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir + unsettmpdir=0 + fi + rm -rf $confluentscripttmpdir + return $retcode +} + +run_remote() { + requestedcmd="'$*'" + unsettmpdir=0 + set_confluent_vars + echo + echo '---------------------------------------------------------------------------' + echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ + if [ -z "$confluentscripttmpdir" ]; then + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + unsettmpdir=1 + fi + echo Executing in $confluentscripttmpdir + cd $confluentscripttmpdir + fetch_remote $1 + if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi + chmod +x $1 + cmd=$1 + if [ -x /usr/bin/chcon ]; then + chcon system_u:object_r:bin_t:s0 $cmd + fi + shift + ./$cmd $* + retcode=$? + if [ $retcode -ne 0 ]; then + echo "$requestedcmd exited with code $retcode" + fi + cd - > /dev/null + if [ "$unsettmpdir" = 1 ]; then + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir + unsettmpdir=0 + fi + return $retcode +} + +run_remote_python() { + echo + set_confluent_vars + if [ -f /etc/confluent/ca.pem ]; then + curlargs=" --cacert /etc/confluent/ca.pem" + fi + echo '---------------------------------------------------------------------------' + echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + echo Executing in $confluentscripttmpdir + cd $confluentscripttmpdir + mkdir -p $(dirname $1) + curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1 + if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi + confluentpython $* + retcode=$? + echo "'$*' exited with code $retcode" + cd - > /dev/null + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir + return $retcode +} + +run_remote_config() { + echo + set_confluent_vars + apiclient=/opt/confluent/bin/apiclient + if [ -f /etc/confluent/apiclient ]; then + apiclient=/etc/confluent/apiclient + fi + echo '---------------------------------------------------------------------------' + echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile + confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {} + confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204 + echo + echo 'Completed remote configuration' + echo '---------------------------------------------------------------------------' + return +} +#If invoked as a command, use the arguments to actually run a function +(return 0 2>/dev/null) || $1 "${@:2}" diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk new file mode 100644 index 00000000..4af31c0b --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk @@ -0,0 +1,88 @@ +import subprocess +import os + +class DiskInfo(object): + def __init__(self, devname): + self.name = devname + self.wwn = None + self.path = None + self.model = '' + self.size = 0 + self.driver = None + self.mdcontainer = '' + devnode = '/dev/{0}'.format(devname) + qprop = subprocess.check_output( + ['udevadm', 'info', '--query=property', devnode]) + if not isinstance(qprop, str): + qprop = qprop.decode('utf8') + for prop in qprop.split('\n'): + if '=' not in prop: + continue + k, v = prop.split('=', 1) + if k == 'DEVTYPE' and v != 'disk': + raise Exception('Not a disk') + elif k == 'DM_NAME': + raise Exception('Device Mapper') + elif k == 'ID_MODEL': + self.model = v + elif k == 'DEVPATH': + self.path = v + elif k == 'ID_WWN': + self.wwn = v + elif k == 'MD_CONTAINER': + self.mdcontainer = v + attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode]) + if not isinstance(attrs, str): + attrs = attrs.decode('utf8') + for attr in attrs.split('\n'): + if '==' not in attr: + continue + k, v = attr.split('==', 1) + k = k.strip() + if k == 'ATTRS{size}': + self.size = v.replace('"', '') + elif (k == 'DRIVERS' and not self.driver + and v not in ('"sd"', '""')): + self.driver = v.replace('"', '') + if not self.driver and 'imsm' not in self.mdcontainer: + raise Exception("No driver detected") + + @property + def priority(self): + if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'): + return 0 + if 'imsm' in self.mdcontainer: + return 1 + if self.driver == 'ahci': + return 2 + if self.driver.startswith('megaraid'): + return 3 + if self.driver.startswith('mpt'): + return 4 + return 99 + + def __repr__(self): + return repr({ + 'name': self.name, + 'path': self.path, + 'wwn': self.wwn, + 'driver': self.driver, + 'size': self.size, + 'model': self.model, + }) + + +def main(): + disks = [] + for disk in sorted(os.listdir('/sys/class/block')): + try: + disk = DiskInfo(disk) + disks.append(disk) + except Exception as e: + print("Skipping {0}: {1}".format(disk, str(e))) + nd = [x.name for x in sorted(disks, key=lambda x: x.priority)] + if nd: + open('/tmp/installdisk', 'w').write(nd[0]) + +if __name__ == '__main__': + main() diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.d/.gitignore b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.d/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh new file mode 100755 index 00000000..6c99735c --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh @@ -0,0 +1,85 @@ +#!/bin/bash +cp -a /root/.ssh /target/root/ +mkdir -p /target/etc/confluent/ssh/sshd_config.d/ +chmod 700 /target/etc/confluent +cp /custom-installation/confluent/* /target/etc/confluent/ +cp -a /custom-installation/tls /target/etc/confluent/ +chmod go-rwx /etc/confluent/* +for i in /custom-installation/ssh/*.ca; do + echo '@cert-authority *' $(cat $i) >> /target/etc/ssh/ssh_known_hosts +done + +cp -a /etc/ssh/ssh_host* /target/etc/confluent/ssh/ +cp -a /etc/ssh/sshd_config.d/confluent.conf /target/etc/confluent/ssh/sshd_config.d/ +sshconf=/target/etc/ssh/ssh_config +if [ -d /target/etc/ssh/ssh_config.d/ ]; then + sshconf=/target/etc/ssh/ssh_config.d/01-confluent.conf +fi +echo 'Host *' >> $sshconf +echo ' HostbasedAuthentication yes' >> $sshconf +echo ' EnableSSHKeysign yes' >> $sshconf +echo ' HostbasedKeyTypes *ed25519*' >> $sshconf + +curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /target/etc/confluent/firstboot.sh +curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /target/etc/confluent/functions +source /target/etc/confluent/functions +chmod +x /target/etc/confluent/firstboot.sh +cp /tmp/allnodes /target/root/.shosts +cp /tmp/allnodes /target/etc/ssh/shosts.equiv +if grep ^ntpservers: /target/etc/confluent/confluent.deploycfg > /dev/null; then + ntps=$(sed -n '/^ntpservers:/,/^[^-]/p' /target/etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //' | paste -sd ' ') + sed -i "s/#NTP=/NTP=$ntps/" /target/etc/systemd/timesyncd.conf +fi +textcons=$(grep ^textconsole: /target/etc/confluent/confluent.deploycfg |awk '{print $2}') +updategrub=0 +if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then + cons="" + if [ -f /custom-installation/autocons.info ]; then + cons=$(cat /custom-installation/autocons.info) + fi + if [ ! -z "$cons" ]; then + sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 console='${cons#/dev/}'"/' /target/etc/default/grub + updategrub=1 + fi +fi +kargs=$(curl https://$confluent_mgr/confluent-public/os/$confluent_profile/profile.yaml | grep ^installedargs: | sed -e 's/#.*//') +if [ ! -z "$kargs" ]; then + sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /target/etc/default/grub +fi +mkdir -p /opt/confluent/bin +mkdir -p /etc/confluent +cp -a /target/etc/confluent/* /etc/confluent +mkdir -p /target/opt/confluent/bin +cp /custom-installation/confluent/bin/apiclient /opt/confluent/bin/ +cp /custom-installation/confluent/bin/apiclient /target/opt/confluent/bin + +mount -o bind /dev /target/dev +mount -o bind /proc /target/proc +mount -o bind /sys /target/sys +if [ 1 = $updategrub ]; then + chroot /target update-grub +fi +echo "Port 22" >> /etc/ssh/sshd_config +echo "Port 2222" >> /etc/ssh/sshd_config +echo "Match LocalPort 22" >> /etc/ssh/sshd_config +echo " ChrootDirectory /target" >> /etc/ssh/sshd_config +kill -HUP $(cat /run/sshd.pid) +if [ -e /sys/firmware/efi ]; then + bootnum=$(chroot /target efibootmgr | grep ubuntu | sed -e 's/ .*//' -e 's/\*//' -e s/Boot//) + if [ ! -z "$bootnum" ]; then + currboot=$(chroot /target efibootmgr | grep ^BootOrder: | awk '{print $2}') + nextboot=$(echo $currboot| awk -F, '{print $1}') + [ "$nextboot" = "$bootnum" ] || chroot /target efibootmgr -o $bootnum,$currboot + chroot /target efibootmgr -D + fi +fi +cat /target/etc/confluent/tls/*.pem > /target/etc/confluent/ca.pem +cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem +chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient" +chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d" +source /target/etc/confluent/functions + +run_remote_config post + +umount /target/sys /target/dev /target/proc + diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh new file mode 100755 index 00000000..ddfe598b --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -0,0 +1,29 @@ +#!/bin/bash +deploycfg=/custom-installation/confluent/confluent.deploycfg + +cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //') +if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then + echo "****Encrypted boot requested, but not implemented for this OS, halting install" > /dev/console + [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but not implemented for this OS,halting install" >> $(cat /tmp/autoconsdev)) + while :; do sleep 86400; done +fi + + +cat /custom-installation/ssh/*pubkey > /root/.ssh/authorized_keys +nodename=$(grep ^NODENAME: /custom-installation/confluent/confluent.info|awk '{print $2}') +apikey=$(cat /custom-installation/confluent/confluent.apikey) +for pubkey in /etc/ssh/ssh_host*key.pub; do + certfile=${pubkey/.pub/-cert.pub} + keyfile=${pubkey%.pub} + curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" -d @$pubkey https://$confluent_mgr/confluent-api/self/sshcert > $certfile + echo HostKey $keyfile >> /etc/ssh/sshd_config.d/confluent.conf + echo HostCertificate $certfile >> /etc/ssh/sshd_config.d/confluent.conf +done +echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/confluent.conf +echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.conf +echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf +systemctl restart sshd +curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$confluent_mgr/confluent-api/self/nodelist > /tmp/allnodes +curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /custom-installation/getinstalldisk +python3 /custom-installation/getinstalldisk +sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient new file mode 100644 index 00000000..3fdd1f08 --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient @@ -0,0 +1,272 @@ +#!/usr/bin/python +import importlib +import tempfile +import json +import os +import shutil +import pwd +import grp +from importlib.machinery import SourceFileLoader +try: + apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() +except FileNotFoundError: + apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module() + + +def partitionhostsline(line): + comment = '' + try: + cmdidx = line.index('#') + comment = line[cmdidx:] + line = line[:cmdidx].strip() + except ValueError: + pass + if not line: + return '', [], comment + ipaddr, names = line.split(maxsplit=1) + names = names.split() + return ipaddr, names, comment + +class HostMerger(object): + def __init__(self): + self.byip = {} + self.byname = {} + self.sourcelines = [] + self.targlines = [] + + def read_source(self, sourcefile): + with open(sourcefile, 'r') as hfile: + self.sourcelines = hfile.read().split('\n') + while not self.sourcelines[-1]: + self.sourcelines = self.sourcelines[:-1] + for x in range(len(self.sourcelines)): + line = self.sourcelines[x] + currip, names, comment = partitionhostsline(line) + if currip: + self.byip[currip] = x + for name in names: + self.byname[name] = x + + def read_target(self, targetfile): + with open(targetfile, 'r') as hfile: + lines = hfile.read().split('\n') + if not lines[-1]: + lines = lines[:-1] + for y in range(len(lines)): + line = lines[y] + currip, names, comment = partitionhostsline(line) + if currip in self.byip: + x = self.byip[currip] + if self.sourcelines[x] is None: + # have already consumed this enntry + continue + self.targlines.append(self.sourcelines[x]) + self.sourcelines[x] = None + continue + for name in names: + if name in self.byname: + x = self.byname[name] + if self.sourcelines[x] is None: + break + self.targlines.append(self.sourcelines[x]) + self.sourcelines[x] = None + break + else: + self.targlines.append(line) + + def write_out(self, targetfile): + while not self.targlines[-1]: + self.targlines = self.targlines[:-1] + if not self.targlines: + break + while not self.sourcelines[-1]: + self.sourcelines = self.sourcelines[:-1] + if not self.sourcelines: + break + with open(targetfile, 'w') as hosts: + for line in self.targlines: + hosts.write(line + '\n') + for line in self.sourcelines: + if line is not None: + hosts.write(line + '\n') + + +class CredMerger: + def __init__(self): + try: + with open('/etc/login.defs', 'r') as ldefs: + defs = ldefs.read().split('\n') + except FileNotFoundError: + defs = [] + lkup = {} + self.discardnames = {} + self.shadowednames = {} + for line in defs: + try: + line = line[:line.index('#')] + except ValueError: + pass + keyval = line.split() + if len(keyval) < 2: + continue + lkup[keyval[0]] = keyval[1] + self.uidmin = int(lkup.get('UID_MIN', 1000)) + self.uidmax = int(lkup.get('UID_MAX', 60000)) + self.gidmin = int(lkup.get('GID_MIN', 1000)) + self.gidmax = int(lkup.get('GID_MAX', 60000)) + self.shadowlines = None + + def read_passwd(self, source, targfile=False): + self.read_generic(source, self.uidmin, self.uidmax, targfile) + + def read_group(self, source, targfile=False): + self.read_generic(source, self.gidmin, self.gidmax, targfile) + + def read_generic(self, source, minid, maxid, targfile): + if targfile: + self.targdata = [] + else: + self.sourcedata = [] + with open(source, 'r') as inputfile: + for line in inputfile.read().split('\n'): + try: + name, _, uid, _ = line.split(':', 3) + uid = int(uid) + except ValueError: + continue + if targfile: + if uid < minid or uid > maxid: + self.targdata.append(line) + else: + self.discardnames[name] = 1 + else: + if name[0] in ('+', '#', '@'): + self.sourcedata.append(line) + elif uid >= minid and uid <= maxid: + self.sourcedata.append(line) + + def read_shadow(self, source): + self.shadowlines = [] + try: + with open(source, 'r') as inshadow: + for line in inshadow.read().split('\n'): + try: + name, _ = line.split(':' , 1) + except ValueError: + continue + if name in self.discardnames: + continue + self.shadowednames[name] = 1 + self.shadowlines.append(line) + except FileNotFoundError: + return + + def write_out(self, outfile): + with open(outfile, 'w') as targ: + for line in self.targdata: + targ.write(line + '\n') + for line in self.sourcedata: + targ.write(line + '\n') + if outfile == '/etc/passwd': + if self.shadowlines is None: + self.read_shadow('/etc/shadow') + with open('/etc/shadow', 'w') as shadout: + for line in self.shadowlines: + shadout.write(line + '\n') + for line in self.sourcedata: + name, _ = line.split(':', 1) + if name[0] in ('+', '#', '@'): + continue + if name in self.shadowednames: + continue + shadout.write(name + ':!:::::::\n') + if outfile == '/etc/group': + if self.shadowlines is None: + self.read_shadow('/etc/gshadow') + with open('/etc/gshadow', 'w') as shadout: + for line in self.shadowlines: + shadout.write(line + '\n') + for line in self.sourcedata: + name, _ = line.split(':' , 1) + if name in self.shadowednames: + continue + shadout.write(name + ':!::\n') + +def appendonce(basepath, filename): + with open(filename, 'rb') as filehdl: + thedata = filehdl.read() + targname = filename.replace(basepath, '') + try: + with open(targname, 'rb') as filehdl: + targdata = filehdl.read() + except IOError: + targdata = b'' + if thedata in targdata: + return + with open(targname, 'ab') as targhdl: + targhdl.write(thedata) + +def synchronize(): + tmpdir = tempfile.mkdtemp() + appendoncedir = tempfile.mkdtemp() + try: + ac = apiclient.HTTPSClient() + data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir}) + status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status == 202: + lastrsp = '' + while status != 204: + status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') + if not isinstance(rsp, str): + rsp = rsp.decode('utf8') + if status == 200: + lastrsp = rsp + pendpasswd = os.path.join(tmpdir, 'etc/passwd') + if os.path.exists(pendpasswd): + cm = CredMerger() + cm.read_passwd(pendpasswd, targfile=False) + cm.read_passwd('/etc/passwd', targfile=True) + cm.write_out('/etc/passwd') + pendgroup = os.path.join(tmpdir, 'etc/group') + if os.path.exists(pendgroup): + cm = CredMerger() + cm.read_group(pendgroup, targfile=False) + cm.read_group('/etc/group', targfile=True) + cm.write_out('/etc/group') + pendhosts = os.path.join(tmpdir, 'etc/hosts') + if os.path.exists(pendhosts): + cm = HostMerger() + cm.read_source(pendhosts) + cm.read_target('/etc/hosts') + cm.write_out('/etc/hosts') + for dirn in os.walk(appendoncedir): + for filen in dirn[2]: + appendonce(appendoncedir, os.path.join(dirn[0], filen)) + if lastrsp: + lastrsp = json.loads(lastrsp) + opts = lastrsp.get('options', {}) + for fname in opts: + uid = -1 + gid = -1 + for opt in opts[fname]: + if opt == 'owner': + try: + uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid + except KeyError: + uid = opts[fname][opt]['id'] + elif opt == 'group': + try: + gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid + except KeyError: + gid = opts[fname][opt]['id'] + elif opt == 'permissions': + os.chmod(fname, int(opts[fname][opt], 8)) + if uid != -1 or gid != -1: + os.chown(fname, uid, gid) + finally: + shutil.rmtree(tmpdir) + shutil.rmtree(appendoncedir) + + +if __name__ == '__main__': + synchronize()