diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/add_local_repositories b/confluent_osdeploy/el10-diskless/profiles/default/scripts/add_local_repositories deleted file mode 100644 index fb26d5ef..00000000 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/add_local_repositories +++ /dev/null @@ -1,58 +0,0 @@ -try: - import configparser -except ImportError: - import ConfigParser as configparser - import cStringIO -import importlib.util -import importlib.machinery -import sys -modloader = importlib.machinery.SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient') -modspec = importlib.util.spec_from_file_location('apiclient', '/opt/confluent/bin/apiclient', loader=modloader) -apiclient = importlib.util.module_from_spec(modspec) -modspec.loader.exec_module(apiclient) -repo = None -server = None -v4cfg = None -server4 = None -server6 = None -profile = None -with open('/etc/confluent/confluent.deploycfg') as dplcfgfile: - lines = dplcfgfile.read().split('\n') - for line in lines: - if line.startswith('deploy_server:'): - _, server4 = line.split(' ', 1) - if line.startswith('deploy_server_v6:'): - _, server6 = line.split(' ', 1) - if line.startswith('profile: '): - _, profile = line.split(' ', 1) - if line.startswith('ipv4_method: '): - _, v4cfg = line.split(' ', 1) -if v4cfg == 'static' or v4cfg =='dhcp': - server = server4 -if not server: - server = '[{}]'.format(server6) - -path = '/confluent-public/os/{0}/distribution/'.format(profile) -clnt = apiclient.HTTPSClient() -cfgdata = clnt.grab_url(path + '.treeinfo').decode() -c = configparser.ConfigParser() -try: - c.read_string(cfgdata) -except AttributeError: - f = cStringIO.StringIO(cfgdata) - c.readfp(f) -for sec in c.sections(): - if sec.startswith('variant-'): - try: - repopath = c.get(sec, 'repository') - except Exception: - continue - _, varname = sec.split('-', 1) - reponame = '/etc/yum.repos.d/local-{0}.repo'.format(varname.lower()) - with open(reponame, 'w') as repout: - repout.write('[local-{0}]\n'.format(varname.lower())) - repout.write('name=Local install repository for {0}\n'.format(varname)) - if repopath[0] == '.': - repopath = repopath[1:] - repout.write('baseurl=https://{}/confluent-public/os/{}/distribution/{}\n'.format(server, profile, repopath)) - repout.write('enabled=1\n') diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/add_local_repositories b/confluent_osdeploy/el10-diskless/profiles/default/scripts/add_local_repositories new file mode 120000 index 00000000..9363a791 --- /dev/null +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/add_local_repositories @@ -0,0 +1 @@ +../../../../el9-diskless/profiles/default/scripts/add_local_repositories \ No newline at end of file diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.custom b/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.custom deleted file mode 100644 index eea34051..00000000 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.custom +++ /dev/null @@ -1,4 +0,0 @@ -. /etc/confluent/functions -# This is a convenient place to keep customizations separate from modifying the stock scripts -# While modification of the stock scripts is fine, it may be easier to rebase to a newer -# stock profile if the '.custom' files are used. diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.custom b/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.custom new file mode 120000 index 00000000..4c873444 --- /dev/null +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.custom @@ -0,0 +1 @@ +../../../../el9-diskless/profiles/default/scripts/firstboot.custom \ No newline at end of file diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.service b/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.service deleted file mode 100644 index 209a95e6..00000000 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=First Boot Process -Requires=network-online.target -After=network-online.target - -[Service] -ExecStart=/opt/confluent/bin/firstboot.sh - -[Install] -WantedBy=multi-user.target - diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.service b/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.service new file mode 120000 index 00000000..7c756923 --- /dev/null +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.service @@ -0,0 +1 @@ +../../../../el9-diskless/profiles/default/scripts/firstboot.service \ No newline at end of file diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.sh b/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.sh deleted file mode 100644 index fabb9385..00000000 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh - -# This script is executed on the first boot after install has -# completed. It is best to edit the middle of the file as -# noted below so custom commands are executed before -# the script notifies confluent that install is fully complete. - -HOME=$(getent passwd $(whoami)|cut -d: -f 6) -export HOME -nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') -confluent_apikey=$(cat /etc/confluent/confluent.apikey) -confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg|awk '{print $2}') -if [ -z "$confluent_mgr" ] || [ "$confluent_mgr" == "null" ] || ! ping -c 1 $confluent_mgr >& /dev/null; then - confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}') -fi -confluent_websrv=$confluent_mgr -if [[ "$confluent_mgr" == *:* ]]; then - confluent_websrv="[$confluent_mgr]" -fi -confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}') -export nodename confluent_mgr confluent_profile confluent_websrv -. /etc/confluent/functions -( -exec >> /var/log/confluent/confluent-firstboot.log -exec 2>> /var/log/confluent/confluent-firstboot.log -chmod 600 /var/log/confluent/confluent-firstboot.log -while ! ping -c 1 $confluent_mgr >& /dev/null; do - sleep 1 -done - -if [ ! -f /etc/confluent/firstboot.ran ]; then - touch /etc/confluent/firstboot.ran - - cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt - - run_remote firstboot.custom - # Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh - run_remote_parts firstboot.d - - # Induce execution of remote configuration, e.g. ansible plays in ansible/firstboot.d/ - run_remote_config firstboot.d -fi - -curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_websrv/confluent-api/self/updatestatus -systemctl disable firstboot -rm /etc/systemd/system/firstboot.service -rm /etc/confluent/firstboot.ran -) & -tail --pid $! -F /var/log/confluent/confluent-firstboot.log > /dev/console diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.sh b/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.sh new file mode 120000 index 00000000..5d76a4b3 --- /dev/null +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/firstboot.sh @@ -0,0 +1 @@ +../../../../el9-diskless/profiles/default/scripts/firstboot.sh \ No newline at end of file diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/functions b/confluent_osdeploy/el10-diskless/profiles/default/scripts/functions deleted file mode 100644 index f68f3a5e..00000000 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/functions +++ /dev/null @@ -1,209 +0,0 @@ -#!/bin/bash -function test_mgr() { - whost=$1 - if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then - whost="[$whost]" - fi - if curl -gs https://${whost}/confluent-api/ > /dev/null; then - return 0 - fi - return 1 -} - -function confluentpython() { - if [ -x /usr/libexec/platform-python ]; then - /usr/libexec/platform-python $* - elif [ -x /usr/bin/python3 ]; then - /usr/bin/python3 $* - elif [ -x /usr/bin/python ]; then - /usr/bin/python $* - elif [ -x /usr/bin/python2 ]; then - /usr/bin/python2 $* - fi -} - -function set_confluent_vars() { - if [ -z "$nodename" ]; then - nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}') - fi - if [[ "$confluent_mgr" == *"%"* ]]; then - confluent_mgr="" - fi - if [ -z "$confluent_mgr" ]; then - confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') - if ! test_mgr $confluent_mgr; then - confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') - if [[ "$confluent_mgr" = *":"* ]]; then - confluent_mgr="[$confluent_mgr]" - fi - fi - if ! test_mgr $confluent_mgr; then - BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//') - OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//') - for confluent_mgr in $BESTMGRS $OKMGRS; do - if [[ $confluent_mgr == *":"* ]]; then - confluent_mgr="[$confluent_mgr]" - fi - if test_mgr $confluent_mgr; then - break - fi - done - fi - fi - if [ -z "$confluent_profile" ]; then - confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') - fi - export confluent_profile confluent_mgr nodename -} - -fetch_remote() { - curlargs="" - if [ -f /etc/confluent/ca.pem ]; then - curlargs=" --cacert /etc/confluent/ca.pem" - fi - set_confluent_vars - mkdir -p $(dirname $1) - whost=$confluent_mgr - if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then - whost="[$whost]" - fi - curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1 - if [ $? != 0 ]; then echo $1 failed to download; return 1; fi -} - -source_remote_parts() { - confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) - apiclient=/opt/confluent/bin/apiclient - if [ -f /etc/confluent/apiclient ]; then - apiclient=/etc/confluent/apiclient - fi - scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //') - for script in $scriptlist; do - source_remote $1/$script - done - rm -rf $confluentscripttmpdir - unset confluentscripttmpdir -} - -run_remote_parts() { - confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) - apiclient=/opt/confluent/bin/apiclient - if [ -f /etc/confluent/apiclient ]; then - apiclient=/etc/confluent/apiclient - fi - scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //') - for script in $scriptlist; do - run_remote $1/$script - done - rm -rf $confluentscripttmpdir - unset confluentscripttmpdir -} - -source_remote() { - set_confluent_vars - unsettmpdir=0 - echo - echo '---------------------------------------------------------------------------' - echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ - if [ -z "$confluentscripttmpdir" ]; then - confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) - unsettmpdir=1 - fi - echo Sourcing from $confluentscripttmpdir - cd $confluentscripttmpdir - fetch_remote $1 - if [ $? != 0 ]; then echo $1 failed to download; return 1; fi - chmod +x $1 - cmd=$1 - shift - source ./$cmd - cd - > /dev/null - if [ "$unsettmpdir" = 1 ]; then - rm -rf $confluentscripttmpdir - unset confluentscripttmpdir - unsettmpdir=0 - fi - rm -rf $confluentscripttmpdir - return $retcode -} - -run_remote() { - requestedcmd="'$*'" - unsettmpdir=0 - set_confluent_vars - echo - echo '---------------------------------------------------------------------------' - echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ - if [ -z "$confluentscripttmpdir" ]; then - confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) - unsettmpdir=1 - fi - echo Executing in $confluentscripttmpdir - cd $confluentscripttmpdir - fetch_remote $1 - if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi - chmod +x $1 - cmd=$1 - if [ -x /usr/bin/chcon ]; then - chcon system_u:object_r:bin_t:s0 $cmd - fi - shift - ./$cmd $* - retcode=$? - if [ $retcode -ne 0 ]; then - echo "$requestedcmd exited with code $retcode" - fi - cd - > /dev/null - if [ "$unsettmpdir" = 1 ]; then - rm -rf $confluentscripttmpdir - unset confluentscripttmpdir - unsettmpdir=0 - fi - return $retcode -} - -run_remote_python() { - echo - set_confluent_vars - if [ -f /etc/confluent/ca.pem ]; then - curlargs=" --cacert /etc/confluent/ca.pem" - fi - echo '---------------------------------------------------------------------------' - echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ - confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) - echo Executing in $confluentscripttmpdir - cd $confluentscripttmpdir - mkdir -p $(dirname $1) - whost=$confluent_mgr - if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then - whost="[$whost]" - fi - curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1 - if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi - confluentpython $* - retcode=$? - echo "'$*' exited with code $retcode" - cd - > /dev/null - rm -rf $confluentscripttmpdir - unset confluentscripttmpdir - return $retcode -} - -run_remote_config() { - echo - set_confluent_vars - apiclient=/opt/confluent/bin/apiclient - if [ -f /etc/confluent/apiclient ]; then - apiclient=/etc/confluent/apiclient - fi - echo '---------------------------------------------------------------------------' - echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile - confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {} - confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204 - echo - echo 'Completed remote configuration' - echo '---------------------------------------------------------------------------' - return -} -#If invoked as a command, use the arguments to actually run a function -(return 0 2>/dev/null) || $1 "${@:2}" diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/functions b/confluent_osdeploy/el10-diskless/profiles/default/scripts/functions new file mode 120000 index 00000000..b20bb48f --- /dev/null +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/functions @@ -0,0 +1 @@ +../../../../el9-diskless/profiles/default/scripts/functions \ No newline at end of file diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el10-diskless/profiles/default/scripts/getinstalldisk deleted file mode 100644 index c954a254..00000000 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/getinstalldisk +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/python3 -import subprocess -import os - -class DiskInfo(object): - def __init__(self, devname): - if devname.startswith('nvme') and 'c' in devname: - raise Exception("Skipping multipath devname") - self.name = devname - self.wwn = None - self.path = None - self.model = '' - self.size = 0 - self.driver = '' - self.mdcontainer = '' - self.subsystype = '' - devnode = '/dev/{0}'.format(devname) - qprop = subprocess.check_output( - ['udevadm', 'info', '--query=property', devnode]) - if not isinstance(qprop, str): - qprop = qprop.decode('utf8') - for prop in qprop.split('\n'): - if '=' not in prop: - continue - k, v = prop.split('=', 1) - if k == 'DEVTYPE' and v != 'disk': - raise Exception('Not a disk') - elif k == 'DM_NAME': - raise Exception('Device Mapper') - elif k == 'ID_MODEL': - self.model = v - elif k == 'DEVPATH': - self.path = v - elif k == 'ID_WWN': - self.wwn = v - elif k == 'MD_CONTAINER': - self.mdcontainer = v - attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode]) - if not isinstance(attrs, str): - attrs = attrs.decode('utf8') - for attr in attrs.split('\n'): - if '==' not in attr: - continue - k, v = attr.split('==', 1) - k = k.strip() - if k == 'ATTRS{size}': - self.size = v.replace('"', '') - elif (k == 'DRIVERS' and not self.driver - and v not in ('"sd"', '""')): - self.driver = v.replace('"', '') - elif k == 'ATTRS{subsystype}': - self.subsystype = v.replace('"', '') - if not self.driver and 'imsm' not in self.mdcontainer and self.subsystype != 'nvm': - raise Exception("No driver detected") - if os.path.exists('/sys/block/{0}/size'.format(self.name)): - with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc: - self.size = int(sizesrc.read()) * 512 - if int(self.size) < 536870912: - raise Exception("Device too small for install") - - @property - def priority(self): - if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'): - return 0 - if 'imsm' in self.mdcontainer: - return 1 - if self.driver == 'ahci': - return 2 - if self.driver.startswith('megaraid'): - return 3 - if self.driver.startswith('mpt'): - return 4 - return 99 - - def __repr__(self): - return repr({ - 'name': self.name, - 'path': self.path, - 'wwn': self.wwn, - 'driver': self.driver, - 'size': self.size, - 'model': self.model, - }) - - -def main(): - disks = [] - for disk in sorted(os.listdir('/sys/class/block')): - try: - disk = DiskInfo(disk) - disks.append(disk) - except Exception as e: - print("Skipping {0}: {1}".format(disk, str(e))) - nd = [x.name for x in sorted(disks, key=lambda x: x.priority)] - if nd: - open('/tmp/installdisk', 'w').write(nd[0]) - -if __name__ == '__main__': - main() diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el10-diskless/profiles/default/scripts/getinstalldisk new file mode 120000 index 00000000..0b399a6f --- /dev/null +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/getinstalldisk @@ -0,0 +1 @@ +../../../../el9-diskless/profiles/default/scripts/getinstalldisk \ No newline at end of file diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el10-diskless/profiles/default/scripts/image2disk.py deleted file mode 100644 index 4a08716a..00000000 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/image2disk.py +++ /dev/null @@ -1,561 +0,0 @@ -#!/usr/bin/python3 -import glob -import json -import os -import re -import time -import shutil -import socket -import stat -import struct -import sys -import subprocess -import traceback - -bootuuid = None -vgname = 'localstorage' -oldvgname = None - -def convert_lv(oldlvname): - if oldvgname is None: - return None - return oldlvname.replace(oldvgname, vgname) - -def get_partname(devname, idx): - if devname[-1] in '0123456789': - return '{}p{}'.format(devname, idx) - else: - return '{}{}'.format(devname, idx) - -def get_next_part_meta(img, imgsize): - if img.tell() == imgsize: - return None - pathlen = struct.unpack('!H', img.read(2))[0] - mountpoint = img.read(pathlen).decode('utf8') - jsonlen = struct.unpack('!I', img.read(4))[0] - metadata = json.loads(img.read(jsonlen).decode('utf8')) - img.seek(16, 1) # skip the two 64-bit values we don't use, they are in json - nextlen = struct.unpack('!H', img.read(2))[0] - img.seek(nextlen, 1) # skip filesystem type - nextlen = struct.unpack('!H', img.read(2))[0] - img.seek(nextlen, 1) # skip orig devname (redundant with json) - nextlen = struct.unpack('!H', img.read(2))[0] - img.seek(nextlen, 1) # skip padding - nextlen = struct.unpack('!Q', img.read(8))[0] - img.seek(nextlen, 1) # go to next section - return metadata - -def get_multipart_image_meta(img): - img.seek(0, 2) - imgsize = img.tell() - img.seek(16) - seekamt = img.read(1) - img.seek(struct.unpack('B', seekamt)[0], 1) - partinfo = get_next_part_meta(img, imgsize) - while partinfo: - yield partinfo - partinfo = get_next_part_meta(img, imgsize) - -def get_image_metadata(imgpath): - with open(imgpath, 'rb') as img: - header = img.read(16) - if header == b'\x63\x7b\x9d\x26\xb7\xfd\x48\x30\x89\xf9\x11\xcf\x18\xfd\xff\xa1': - for md in get_multipart_image_meta(img): - if md.get('device', '').startswith('/dev/zram'): - continue - yield md - else: - # plausible filesystem structure to apply to a nominally "diskless" image - yield {'mount': '/', 'filesystem': 'xfs', 'minsize': 39513563136, 'initsize': 954128662528, 'flags': 'rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota', 'device': '/dev/mapper/root', 'compressed_size': 27022069760} - yield {'mount': '/boot', 'filesystem': 'xfs', 'minsize': 232316928, 'initsize': 1006632960, 'flags': 'rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota', 'device': '/dev/nvme1n1p2', 'compressed_size': 171462656} - yield {'mount': '/boot/efi', 'filesystem': 'vfat', 'minsize': 7835648, 'initsize': 627900416, 'flags': 'rw,relatime,fmask=0077,dmask=0077,codepage=437,iocharset=ascii,shortname=winnt,errors=remount-ro', 'device': '/dev/nvme1n1p1', 'compressed_size': 1576960} - #raise Exception('Installation from single part image not supported') - -class PartedRunner(): - def __init__(self, disk): - self.disk = disk - - def run(self, command, check=True): - command = command.split() - command = ['parted', '-a', 'optimal', '-s', self.disk] + command - if check: - return subprocess.check_output(command).decode('utf8') - else: - return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8') - -def fixup(rootdir, vols): - devbymount = {} - for vol in vols: - devbymount[vol['mount']] = vol['targetdisk'] - fstabfile = os.path.join(rootdir, 'etc/fstab') - if os.path.exists(fstabfile): - with open(fstabfile) as tfile: - fstab = tfile.read().split('\n') - else: - #diskless image, need to invent fstab - fstab = [ - "#ORIGFSTAB#/dev/mapper/root# / xfs defaults 0 0", - "#ORIGFSTAB#UUID=aaf9e0f9-aa4d-4d74-9e75-3537620cfe23# /boot xfs defaults 0 0", - "#ORIGFSTAB#UUID=C21D-B881# /boot/efi vfat umask=0077,shortname=winnt 0 2", - "#ORIGFSTAB#/dev/mapper/swap# none swap defaults 0 0", - ] - while not fstab[0]: - fstab = fstab[1:] - if os.path.exists(os.path.join(rootdir, '.autorelabel')): - os.unlink(os.path.join(rootdir, '.autorelabel')) - with open(fstabfile, 'w') as tfile: - for tab in fstab: - entry = tab.split() - if tab.startswith('#ORIGFSTAB#'): - if entry[1] in devbymount: - targetdev = devbymount[entry[1]] - if targetdev.startswith('/dev/{}/'.format(vgname)): - entry[0] = targetdev - else: - uuid = subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', targetdev]).decode('utf8') - uuid = uuid.strip() - entry[0] = 'UUID={}'.format(uuid) - elif entry[2] == 'swap': - entry[0] = '/dev/mapper/{}-swap'.format(vgname.replace('-', '--')) - entry[0] = entry[0].ljust(42) - entry[1] = entry[1].ljust(16) - entry[3] = entry[3].ljust(28) - tab = '\t'.join(entry) - tfile.write(tab + '\n') - with open(os.path.join(rootdir, 'etc/hostname'), 'w') as nameout: - nameout.write(socket.gethostname() + '\n') - selinuxconfig = os.path.join(rootdir, 'etc/selinux/config') - policy = None - if os.path.exists(selinuxconfig): - with open(selinuxconfig) as cfgin: - sec = cfgin.read().split('\n') - for l in sec: - l = l.split('#', 1)[0] - if l.startswith('SELINUXTYPE='): - _, policy = l.split('=') - for sshkey in glob.glob(os.path.join(rootdir, 'etc/ssh/*_key*')): - os.unlink(sshkey) - for sshkey in glob.glob('/etc/ssh/*_key*'): - newkey = os.path.join(rootdir, sshkey[1:]) - shutil.copy2(sshkey, newkey) - finfo = os.stat(sshkey) - os.chown(newkey, finfo[stat.ST_UID], finfo[stat.ST_GID]) - for ifcfg in glob.glob(os.path.join(rootdir, 'etc/sysconfig/network-scripts/*')): - os.unlink(ifcfg) - for ifcfg in glob.glob(os.path.join(rootdir, 'etc/NetworkManager/system-connections/*')): - os.unlink(ifcfg) - for ifcfg in glob.glob('/run/NetworkManager/system-connections/*'): - newcfg = ifcfg.split('/')[-1] - newcfg = os.path.join(rootdir, 'etc/NetworkManager/system-connections/{0}'.format(newcfg)) - shutil.copy2(ifcfg, newcfg) - rootconfluentdir = os.path.join(rootdir, 'etc/confluent/') - if os.path.exists(rootconfluentdir): - shutil.rmtree(rootconfluentdir) - shutil.copytree('/etc/confluent', rootconfluentdir) - if policy: - sys.stdout.write('Applying SELinux labeling...') - sys.stdout.flush() - subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'etc')]) - subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'opt')]) - sys.stdout.write('Done\n') - sys.stdout.flush() - for metafs in ('proc', 'sys', 'dev'): - subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)]) - if os.path.exists(os.path.join(rootdir, 'etc/lvm/devices/system.devices')): - os.remove(os.path.join(rootdir, 'etc/lvm/devices/system.devices')) - grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub') - if not os.path.exists(grubsyscfg): - grubsyscfg = os.path.join(rootdir, 'etc/default/grub') - kcmdline = os.path.join(rootdir, 'etc/kernel/cmdline') - if os.path.exists(kcmdline): - with open(kcmdline) as kcmdlinein: - kcmdlinecontent = kcmdlinein.read() - newkcmdlineent = [] - for ent in kcmdlinecontent.split(): - if ent.startswith('resume='): - newkcmdlineent.append('resume={}'.format(newswapdev)) - elif ent.startswith('root='): - newkcmdlineent.append('root={}'.format(newrootdev)) - elif ent.startswith('rd.lvm.lv='): - ent = convert_lv(ent) - if ent: - newkcmdlineent.append(ent) - else: - newkcmdlineent.append(ent) - with open(kcmdline, 'w') as kcmdlineout: - kcmdlineout.write(' '.join(newkcmdlineent) + '\n') - for loadent in glob.glob(os.path.join(rootdir, 'boot/loader/entries/*.conf')): - with open(loadent) as loadentin: - currentry = loadentin.read().split('\n') - with open(loadent, 'w') as loadentout: - for cfgline in currentry: - cfgparts = cfgline.split() - if not cfgparts or cfgparts[0] != 'options': - loadentout.write(cfgline + '\n') - continue - newcfgparts = [cfgparts[0]] - for cfgpart in cfgparts[1:]: - if cfgpart.startswith('root='): - newcfgparts.append('root={}'.format(newrootdev)) - elif cfgpart.startswith('resume='): - newcfgparts.append('resume={}'.format(newswapdev)) - elif cfgpart.startswith('rd.lvm.lv='): - cfgpart = convert_lv(cfgpart) - if cfgpart: - newcfgparts.append(cfgpart) - else: - newcfgparts.append(cfgpart) - loadentout.write(' '.join(newcfgparts) + '\n') - if os.path.exists(grubsyscfg): - with open(grubsyscfg) as defgrubin: - defgrub = defgrubin.read().split('\n') - else: - defgrub = [ - 'GRUB_TIMEOUT=5', - 'GRUB_DISTRIBUTOR="$(sed ' + "'s, release .*$,,g'" + ' /etc/system-release)"', - 'GRUB_DEFAULT=saved', - 'GRUB_DISABLE_SUBMENU=true', - 'GRUB_TERMINAL=""', - 'GRUB_SERIAL_COMMAND=""', - 'GRUB_CMDLINE_LINUX="crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M rd.lvm.lv=vg/root rd.lvm.lv=vg/swap"', - 'GRUB_DISABLE_RECOVERY="true"', - 'GRUB_ENABLE_BLSCFG=true', - ] - if not os.path.exists(os.path.join(rootdir, "etc/kernel/cmdline")): - with open(os.path.join(rootdir, "etc/kernel/cmdline"), "w") as cmdlineout: - cmdlineout.write("root=/dev/mapper/localstorage-root rd.lvm.lv=localstorage/root") - with open(grubsyscfg, 'w') as defgrubout: - for gline in defgrub: - gline = gline.split() - newline = [] - for ent in gline: - if ent.startswith('resume='): - newline.append('resume={}'.format(newswapdev)) - elif ent.startswith('root='): - newline.append('root={}'.format(newrootdev)) - elif ent.startswith('rd.lvm.lv='): - ent = convert_lv(ent) - if ent: - newline.append(ent) - elif '""' in ent: - newline.append('""') - else: - newline.append(ent) - defgrubout.write(' '.join(newline) + '\n') - grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/') - grubcfg = grubcfg.split('\n') - if not grubcfg[-1]: - grubcfg = grubcfg[:-1] - if len(grubcfg) == 1: - grubcfg = grubcfg[0] - elif not grubcfg: - grubcfg = '/boot/grub2/grub.cfg' - paths = glob.glob(os.path.join(rootdir, 'boot/efi/EFI/*')) - for path in paths: - with open(os.path.join(path, 'grub.cfg'), 'w') as stubgrubout: - stubgrubout.write("search --no-floppy --root-dev-only --fs-uuid --set=dev " + bootuuid + "\nset prefix=($dev)/grub2\nexport $prefix\nconfigfile $prefix/grub.cfg\n") - else: - for gcfg in grubcfg: - rgcfg = os.path.join(rootdir, gcfg[1:]) # gcfg has a leading / to get rid of - if os.stat(rgcfg).st_size > 256: - grubcfg = gcfg - else: - with open(rgcfg, 'r') as gin: - tgrubcfg = gin.read() - tgrubcfg = tgrubcfg.split('\n') - if 'search --no-floppy --fs-uuid --set=dev' in tgrubcfg[0]: - tgrubcfg[0] = 'search --no-floppy --fs-uuid --set=dev ' + bootuuid - with open(rgcfg, 'w') as gout: - for gcline in tgrubcfg: - gout.write(gcline) - gout.write('\n') - try: - subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg]) - except Exception as e: - print(repr(e)) - print(rootdir) - print(grubcfg) - time.sleep(86400) - newroot = None - with open('/etc/shadow') as shadowin: - shents = shadowin.read().split('\n') - for shent in shents: - shent = shent.split(':') - if not shent: - continue - if shent[0] == 'root' and shent[1] not in ('*', '!!', ''): - newroot = shent[1] - if newroot: - shlines = None - with open(os.path.join(rootdir, 'etc/shadow')) as oshadow: - shlines = oshadow.read().split('\n') - with open(os.path.join(rootdir, 'etc/shadow'), 'w') as oshadow: - for line in shlines: - if line.startswith('root:'): - line = line.split(':') - line[1] = newroot - line = ':'.join(line) - oshadow.write(line + '\n') - partnum = None - targblock = None - for vol in vols: - if vol['mount'] == '/boot/efi': - targdev = vol['targetdisk'] - partnum = re.search('(\d+)$', targdev).group(1) - targblock = re.search('(.*)\d+$', targdev).group(1) - if targblock: - if targblock.endswith('p') and 'nvme' in targblock: - targblock = targblock[:-1] - shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip() - shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\') - subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum]) - - try: - os.makedirs(os.path.join(rootdir, 'opt/confluent/bin')) - except Exception: - pass - shutil.copy2('/opt/confluent/bin/apiclient', os.path.join(rootdir, 'opt/confluent/bin/apiclient')) - #other network interfaces - - -def had_swap(): - if not os.path.exists('/etc/fstab'): - # diskless source, assume swap - return True - with open('/etc/fstab') as tabfile: - tabs = tabfile.read().split('\n') - for tab in tabs: - tab = tab.split() - if len(tab) < 3: - continue - if tab[2] == 'swap': - return True - return False - -newrootdev = None -newswapdev = None -def install_to_disk(imgpath): - global bootuuid - global newrootdev - global newswapdev - global vgname - global oldvgname - lvmvols = {} - deftotsize = 0 - mintotsize = 0 - deflvmsize = 0 - minlvmsize = 0 - biggestsize = 0 - biggestfs = None - plainvols = {} - allvols = [] - swapsize = 0 - if had_swap(): - with open('/proc/meminfo') as meminfo: - swapsize = meminfo.read().split('\n')[0] - swapsize = int(swapsize.split()[1]) - if swapsize < 2097152: - swapsize = swapsize * 2 - elif swapsize > 8388608 and swapsize < 67108864: - swapsize = swapsize * 0.5 - elif swapsize >= 67108864: - swapsize = 33554432 - swapsize = int(swapsize * 1024) - deftotsize = swapsize - mintotsize = swapsize - for fs in get_image_metadata(imgpath): - allvols.append(fs) - deftotsize += fs['initsize'] - mintotsize += fs['minsize'] - if fs['initsize'] > biggestsize: - biggestfs = fs - biggestsize = fs['initsize'] - if fs['device'].startswith('/dev/mapper'): - oldvgname = fs['device'].rsplit('/', 1)[-1] - # if node has - then /dev/mapper will double up the hypen - if '_' in oldvgname and '-' in oldvgname.split('_', 1)[-1]: - oldvgname = oldvgname.rsplit('-', 1)[0].replace('--', '-') - osname = oldvgname.split('_')[0] - nodename = socket.gethostname().split('.')[0] - vgname = '{}_{}'.format(osname, nodename) - lvmvols[fs['device'].replace('/dev/mapper/', '')] = fs - deflvmsize += fs['initsize'] - minlvmsize += fs['minsize'] - else: - plainvols[int(re.search('(\d+)$', fs['device'])[0])] = fs - with open('/tmp/installdisk') as diskin: - instdisk = diskin.read() - instdisk = '/dev/' + instdisk - parted = PartedRunner(instdisk) - dinfo = parted.run('unit s print', check=False) - dinfo = dinfo.split('\n') - sectors = 0 - sectorsize = 0 - for inf in dinfo: - if inf.startswith('Disk {0}:'.format(instdisk)): - _, sectors = inf.split(': ') - sectors = int(sectors.replace('s', '')) - if inf.startswith('Sector size (logical/physical):'): - _, sectorsize = inf.split(':') - sectorsize = sectorsize.split('/')[0] - sectorsize = sectorsize.replace('B', '') - sectorsize = int(sectorsize) - # for now, only support resizing/growing the largest partition - minexcsize = deftotsize - biggestfs['initsize'] - mintotsize = deftotsize - biggestfs['initsize'] + biggestfs['minsize'] - minsectors = mintotsize // sectorsize - if sectors < (minsectors + 65536): - raise Exception('Disk too small to fit image') - biggestsectors = sectors - (minexcsize // sectorsize) - biggestsize = sectorsize * biggestsectors - parted.run('mklabel gpt') - curroffset = 2048 - for volidx in sorted(plainvols): - vol = plainvols[volidx] - if vol is not biggestfs: - size = vol['initsize'] // sectorsize - else: - size = biggestsize // sectorsize - size += 2047 - (size % 2048) - end = curroffset + size - if end > sectors: - end = sectors - parted.run('mkpart primary {}s {}s'.format(curroffset, end)) - vol['targetdisk'] = get_partname(instdisk, volidx) - if vol['mount'] == '/': - newrootdev = vol['targetdisk'] - curroffset += size + 1 - if not lvmvols: - if swapsize: - swapsize = swapsize // sectorsize - swapsize += 2047 - (size % 2048) - end = curroffset + swapsize - if end > sectors: - end = sectors - parted.run('mkpart swap {}s {}s'.format(curroffset, end)) - newswapdev = get_partname(instdisk, volidx + 1) - subprocess.check_call(['mkswap', newswapdev]) - else: - parted.run('mkpart lvm {}s 100%'.format(curroffset)) - lvmpart = get_partname(instdisk, volidx + 1) - subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart]) - subprocess.check_call(['vgcreate', vgname, lvmpart]) - vginfo = subprocess.check_output(['vgdisplay', vgname, '--units', 'b']).decode('utf8') - vginfo = vginfo.split('\n') - pesize = 0 - pes = 0 - for infline in vginfo: - infline = infline.split() - if len(infline) >= 3 and infline[:2] == ['PE', 'Size']: - pesize = int(infline[2]) - if len(infline) >= 5 and infline[:2] == ['Free', 'PE']: - pes = int(infline[4]) - takeaway = swapsize // pesize - for volidx in lvmvols: - vol = lvmvols[volidx] - if vol is biggestfs: - continue - takeaway += vol['initsize'] // pesize - takeaway += 1 - biggestextents = pes - takeaway - for volidx in lvmvols: - vol = lvmvols[volidx] - if vol is biggestfs: - extents = biggestextents - else: - extents = vol['initsize'] // pesize - extents += 1 - if vol['mount'] == '/': - lvname = 'root' - - else: - lvname = vol['mount'].replace('/', '_') - subprocess.check_call(['lvcreate', '-l', '{}'.format(extents), '-y', '-n', lvname, vgname]) - vol['targetdisk'] = '/dev/{}/{}'.format(vgname, lvname) - if vol['mount'] == '/': - newrootdev = vol['targetdisk'] - if swapsize: - subprocess.check_call(['lvcreate', '-y', '-l', '{}'.format(swapsize // pesize), '-n', 'swap', vgname]) - subprocess.check_call(['mkswap', '/dev/{}/swap'.format(vgname)]) - newswapdev = '/dev/{}/swap'.format(vgname) - os.makedirs('/run/imginst/targ') - for vol in allvols: - with open(vol['targetdisk'], 'wb') as partition: - partition.write(b'\x00' * 1 * 1024 * 1024) - subprocess.check_call(['mkfs.{}'.format(vol['filesystem']), vol['targetdisk']]) - subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ']) - source = vol['mount'].replace('/', '_') - source = '/run/imginst/sources/' + source - if not os.path.exists(source): - source = '/run/imginst/sources/_' + vol['mount'] - blankfsstat = os.statvfs('/run/imginst/targ') - blankused = (blankfsstat.f_blocks - blankfsstat.f_bfree) * blankfsstat.f_bsize - sys.stdout.write('\nWriting {0}: '.format(vol['mount'])) - with subprocess.Popen(['cp', '-ax', source + '/.', '/run/imginst/targ']) as copier: - stillrunning = copier.poll() - lastprogress = 0.0 - while stillrunning is None: - currfsstat = os.statvfs('/run/imginst/targ') - currused = (currfsstat.f_blocks - currfsstat.f_bfree) * currfsstat.f_bsize - currused -= blankused - with open('/proc/meminfo') as meminf: - for line in meminf.read().split('\n'): - if line.startswith('Dirty:'): - _, dirty, _ = line.split() - dirty = int(dirty) * 1024 - progress = (currused - dirty) / vol['minsize'] - if progress < lastprogress: - progress = lastprogress - if progress > 0.99: - progress = 0.99 - lastprogress = progress - progress = progress * 100 - sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress)) - sys.stdout.flush() - time.sleep(0.5) - stillrunning = copier.poll() - if stillrunning != 0: - raise Exception("Error copying volume") - with subprocess.Popen(['sync']) as syncrun: - stillrunning = syncrun.poll() - while stillrunning is None: - with open('/proc/meminfo') as meminf: - for line in meminf.read().split('\n'): - if line.startswith('Dirty:'): - _, dirty, _ = line.split() - dirty = int(dirty) * 1024 - progress = (vol['minsize'] - dirty) / vol['minsize'] - if progress < lastprogress: - progress = lastprogress - if progress > 0.99: - progress = 0.99 - lastprogress = progress - progress = progress * 100 - sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress)) - sys.stdout.flush() - time.sleep(0.5) - stillrunning = syncrun.poll() - sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount'])) - sys.stdout.write('\n') - sys.stdout.flush() - if vol['mount'] == '/boot': - tbootuuid = subprocess.check_output(['blkid', vol['targetdisk']]) - if b'UUID="' in tbootuuid: - bootuuid = tbootuuid.split(b'UUID="', 1)[1].split(b'"')[0].decode('utf8') - - - - - subprocess.check_call(['umount', '/run/imginst/targ']) - for vol in allvols: - subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']]) - fixup('/run/imginst/targ', allvols) - - -if __name__ == '__main__': - try: - install_to_disk(os.environ['mountsrc']) - except Exception: - traceback.print_exc() - time.sleep(86400) - raise diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el10-diskless/profiles/default/scripts/image2disk.py new file mode 120000 index 00000000..f3aca670 --- /dev/null +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/image2disk.py @@ -0,0 +1 @@ +../../../../el9-diskless/profiles/default/scripts/image2disk.py \ No newline at end of file diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/installimage b/confluent_osdeploy/el10-diskless/profiles/default/scripts/installimage deleted file mode 100644 index c461173b..00000000 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/installimage +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -. /etc/confluent/functions -# the image will be used to deploy itself -# provide both access to image (for parsing metadata) -# and existing mounts of image (to take advantage of caching) -mount -o bind /sys /sysroot/sys -mount -o bind /dev /sysroot/dev -mount -o bind /proc /sysroot/proc -mount -o bind /run /sysroot/run - - -if [ ! -f /tmp/mountparts.sh ]; then - mkdir -p /sysroot/run/imginst/sources/_ - mount -o bind /mnt/remote /sysroot/run/imginst/sources/_ -else - for srcmount in $(cat /tmp/mountparts.sh | awk '{print $2}'); do - srcname=${srcmount#/dev/mapper/mproot} - srcdir=$(echo $srcmount | sed -e 's!/dev/mapper/mproot!/mnt/remote!' -e 's!_!/!g') - mkdir -p /sysroot/run/imginst/sources/$srcname - mount -o bind $srcdir /sysroot/run/imginst/sources/$srcname - done -fi -cd /sysroot/run -chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_python getinstalldisk" -chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_parts pre.d" -if [ ! -f /sysroot/tmp/installdisk ]; then - echo 'Unable to find a suitable installation target device, ssh to port 2222 to investigate' - while [ ! -f /sysroot/tmp/installdisk ]; do - sleep 1 - done -fi -lvm vgchange -a n -/sysroot/usr/sbin/wipefs -a /dev/$(cat /sysroot/tmp/installdisk) -udevadm control -e -if [ -f /sysroot/etc/lvm/devices/system.devices ]; then - rm /sysroot/etc/lvm/devices/system.devices -fi -chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon -chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py" -echo "Port 22" >> /etc/ssh/sshd_config -echo 'Match LocalPort 22' >> /etc/ssh/sshd_config -echo ' ChrootDirectory /sysroot/run/imginst/targ' >> /etc/ssh/sshd_config -kill -HUP $(cat /run/sshd.pid) -cp /sysroot/etc/pki/ca-trust/source/anchors/* /sysroot/run/imginst/targ/etc/pki/ca-trust/source/anchors/ -chroot /sysroot/run/imginst/targ update-ca-trust - -chroot /sysroot/run/imginst/targ bash -c "source /etc/confluent/functions; run_remote post.sh" -chroot /sysroot bash -c "umount \$(tac /proc/mounts|awk '{print \$2}'|grep ^/run/imginst/targ)" - diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/installimage b/confluent_osdeploy/el10-diskless/profiles/default/scripts/installimage new file mode 120000 index 00000000..64455ac9 --- /dev/null +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/installimage @@ -0,0 +1 @@ +../../../../el9-diskless/profiles/default/scripts/installimage \ No newline at end of file diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.custom b/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.custom deleted file mode 100644 index e69de29b..00000000 diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.custom b/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.custom new file mode 120000 index 00000000..b0dfa87a --- /dev/null +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.custom @@ -0,0 +1 @@ +../../../../el9-diskless/profiles/default/scripts/onboot.custom \ No newline at end of file diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.service b/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.service deleted file mode 100644 index f9235033..00000000 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Confluent onboot hook -Requires=network-online.target -After=network-online.target - -[Service] -ExecStart=/opt/confluent/bin/onboot.sh - -[Install] -WantedBy=multi-user.target - diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.service b/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.service new file mode 120000 index 00000000..da70c79c --- /dev/null +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.service @@ -0,0 +1 @@ +../../../../el9-diskless/profiles/default/scripts/onboot.service \ No newline at end of file diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.sh deleted file mode 100644 index 80f95870..00000000 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh - -# This script is executed on each boot as it is -# completed. It is best to edit the middle of the file as -# noted below so custom commands are executed before -# the script notifies confluent that install is fully complete. -ntpsrvs="" -nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') -confluent_apikey=$(cat /etc/confluent/confluent.apikey) -v4meth=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg|awk '{print $2}') -if [ "$v4meth" = "null" -o -z "$v4meth" ]; then - confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg|awk '{print $2}') -fi -if [ -z "$confluent_mgr" ]; then - confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}') -fi -confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}') -timedatectl set-timezone $(grep ^timezone: /etc/confluent/confluent.deploycfg|awk '{print $2}') -hostnamectl set-hostname $nodename - - -if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then - for ntpsrv in $(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //'); do - echo "server ${ntpsrv} iburst " >> /tmp/timeservers - done -fi - -if [ -f /tmp/timeservers ]; then - -ntpsrvs=$(cat /tmp/timeservers) - -sed -i "1,/^pool * /c\\ - -${ntpsrvs//$'\n'/\\$'\n'}" /etc/chrony.conf - - -systemctl restart chronyd - -rm -f /tmp/timeservers -fi - -export nodename confluent_mgr confluent_profile -. /etc/confluent/functions -mkdir -p /var/log/confluent -chmod 700 /var/log/confluent -exec >> /var/log/confluent/confluent-onboot.log -exec 2>> /var/log/confluent/confluent-onboot.log -chmod 600 /var/log/confluent/confluent-onboot.log -tail -f /var/log/confluent/confluent-onboot.log > /dev/console & -logshowpid=$! - -rpm --import /etc/pki/rpm-gpg/* - -run_remote_python add_local_repositories -run_remote_python syncfileclient -run_remote_python confignet - -run_remote onboot.custom -# onboot scripts may be placed into onboot.d, e.g. onboot.d/01-firstaction.sh, onboot.d/02-secondaction.sh -run_remote_parts onboot.d - -# Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/ -run_remote_config onboot.d - -#curl -X POST -d 'status: booted' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus -kill $logshowpid diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.sh new file mode 120000 index 00000000..c6b678db --- /dev/null +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/onboot.sh @@ -0,0 +1 @@ +../../../../el9-diskless/profiles/default/scripts/onboot.sh \ No newline at end of file diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/post.sh b/confluent_osdeploy/el10-diskless/profiles/default/scripts/post.sh deleted file mode 100644 index 914a12c3..00000000 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/post.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh - -# This script is executed 'chrooted' into a cloned disk target before rebooting -# -if [ -f /etc/dracut.conf.d/diskless.conf ]; then - rm /etc/dracut.conf.d/diskless.conf -fi -for kver in /lib/modules/*; do kver=$(basename $kver); kernel-install add $kver /boot/vmlinuz-$kver; done -nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') -confluent_apikey=$(cat /etc/confluent/confluent.apikey) -confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}') -confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg|awk '{print $2}') -if [ -z "$confluent_mgr" ] || [ "$confluent_mgr" == "null" ] || ! ping -c 1 $confluent_mgr >& /dev/null; then - confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}') -fi -confluent_websrv=$confluent_mgr -if [[ "$confluent_mgr" == *:* ]]; then - confluent_websrv="[$confluent_mgr]" -fi -export nodename confluent_mgr confluent_profile confluent_websrv -. /etc/confluent/functions -run_remote setupssh -mkdir -p /var/log/confluent -chmod 700 /var/log/confluent -exec >> /var/log/confluent/confluent-post.log -exec 2>> /var/log/confluent/confluent-post.log -chmod 600 /var/log/confluent/confluent-post.log -tail -f /var/log/confluent/confluent-post.log > /dev/console & -logshowpid=$! -curl -f https://$confluent_websrv/confluent-public/os/$confluent_profile/scripts/firstboot.service > /etc/systemd/system/firstboot.service -mkdir -p /opt/confluent/bin -curl -f https://$confluent_websrv/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /opt/confluent/bin/firstboot.sh -chmod +x /opt/confluent/bin/firstboot.sh -systemctl enable firstboot -selinuxpolicy=$(grep ^SELINUXTYPE /etc/selinux/config |awk -F= '{print $2}') -if [ ! -z "$selinuxpolicy" ]; then - setfiles /etc/selinux/${selinuxpolicy}/contexts/files/file_contexts /etc/ -fi -run_remote_python syncfileclient -run_remote post.custom -# post scripts may be placed into post.d, e.g. post.d/01-firstaction.sh, post.d/02-secondaction.sh -run_remote_parts post.d - -# Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/ -run_remote_config post.d - -# rebuild initrd, pick up new drivers if needed -dracut -f /boot/initramfs-$(uname -r).img $(uname -r) - -curl -sf -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_websrv/confluent-api/self/updatestatus - -kill $logshowpid - diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/post.sh b/confluent_osdeploy/el10-diskless/profiles/default/scripts/post.sh new file mode 120000 index 00000000..61642fa5 --- /dev/null +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/post.sh @@ -0,0 +1 @@ +../../../../el9-diskless/profiles/default/scripts/post.sh \ No newline at end of file diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el10-diskless/profiles/default/scripts/syncfileclient deleted file mode 100644 index 5f2efc5e..00000000 --- a/confluent_osdeploy/el10-diskless/profiles/default/scripts/syncfileclient +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/python3 -import random -import time -import subprocess -import importlib -import tempfile -import json -import os -import shutil -import pwd -import grp -import sys -from importlib.machinery import SourceFileLoader -try: - apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() -except FileNotFoundError: - apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module() - - -def partitionhostsline(line): - comment = '' - try: - cmdidx = line.index('#') - comment = line[cmdidx:] - line = line[:cmdidx].strip() - except ValueError: - pass - if not line: - return '', [], comment - ipaddr, names = line.split(maxsplit=1) - names = names.split() - return ipaddr, names, comment - -class HostMerger(object): - def __init__(self): - self.byip = {} - self.byname = {} - self.sourcelines = [] - self.targlines = [] - - def read_source(self, sourcefile): - with open(sourcefile, 'r') as hfile: - self.sourcelines = hfile.read().split('\n') - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - for x in range(len(self.sourcelines)): - line = self.sourcelines[x] - currip, names, comment = partitionhostsline(line) - if currip: - self.byip[currip] = x - for name in names: - self.byname[name] = x - - def read_target(self, targetfile): - with open(targetfile, 'r') as hfile: - lines = hfile.read().split('\n') - if not lines[-1]: - lines = lines[:-1] - for y in range(len(lines)): - line = lines[y] - currip, names, comment = partitionhostsline(line) - if currip in self.byip: - x = self.byip[currip] - if self.sourcelines[x] is None: - # have already consumed this enntry - continue - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - continue - for name in names: - if name in self.byname: - x = self.byname[name] - if self.sourcelines[x] is None: - break - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - break - else: - self.targlines.append(line) - - def write_out(self, targetfile): - while not self.targlines[-1]: - self.targlines = self.targlines[:-1] - if not self.targlines: - break - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - if not self.sourcelines: - break - with open(targetfile, 'w') as hosts: - for line in self.targlines: - hosts.write(line + '\n') - for line in self.sourcelines: - if line is not None: - hosts.write(line + '\n') - - -class CredMerger: - def __init__(self): - try: - with open('/etc/login.defs', 'r') as ldefs: - defs = ldefs.read().split('\n') - except FileNotFoundError: - defs = [] - lkup = {} - self.discardnames = {} - self.shadowednames = {} - for line in defs: - try: - line = line[:line.index('#')] - except ValueError: - pass - keyval = line.split() - if len(keyval) < 2: - continue - lkup[keyval[0]] = keyval[1] - self.uidmin = int(lkup.get('UID_MIN', 1000)) - self.uidmax = int(lkup.get('UID_MAX', 60000)) - self.gidmin = int(lkup.get('GID_MIN', 1000)) - self.gidmax = int(lkup.get('GID_MAX', 60000)) - self.shadowlines = None - - def read_passwd(self, source, targfile=False): - self.read_generic(source, self.uidmin, self.uidmax, targfile) - - def read_group(self, source, targfile=False): - self.read_generic(source, self.gidmin, self.gidmax, targfile) - - def read_generic(self, source, minid, maxid, targfile): - if targfile: - self.targdata = [] - else: - self.sourcedata = [] - with open(source, 'r') as inputfile: - for line in inputfile.read().split('\n'): - try: - name, _, uid, _ = line.split(':', 3) - uid = int(uid) - except ValueError: - continue - if targfile: - if uid < minid or uid > maxid: - self.targdata.append(line) - else: - self.discardnames[name] = 1 - else: - if name[0] in ('+', '#', '@'): - self.sourcedata.append(line) - elif uid >= minid and uid <= maxid: - self.sourcedata.append(line) - - def read_shadow(self, source): - self.shadowlines = [] - try: - with open(source, 'r') as inshadow: - for line in inshadow.read().split('\n'): - try: - name, _ = line.split(':' , 1) - except ValueError: - continue - if name in self.discardnames: - continue - self.shadowednames[name] = 1 - self.shadowlines.append(line) - except FileNotFoundError: - return - - def write_out(self, outfile): - with open(outfile, 'w') as targ: - for line in self.targdata: - targ.write(line + '\n') - for line in self.sourcedata: - targ.write(line + '\n') - if outfile == '/etc/passwd': - if self.shadowlines is None: - self.read_shadow('/etc/shadow') - with open('/etc/shadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':', 1) - if name[0] in ('+', '#', '@'): - continue - if name in self.shadowednames: - continue - shadout.write(name + ':!:::::::\n') - if outfile == '/etc/group': - if self.shadowlines is None: - self.read_shadow('/etc/gshadow') - with open('/etc/gshadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':' , 1) - if name in self.shadowednames: - continue - shadout.write(name + ':!::\n') - -def appendonce(basepath, filename): - with open(filename, 'rb') as filehdl: - thedata = filehdl.read() - targname = filename.replace(basepath, '') - try: - with open(targname, 'rb') as filehdl: - targdata = filehdl.read() - except IOError: - targdata = b'' - if thedata in targdata: - return - with open(targname, 'ab') as targhdl: - targhdl.write(thedata) - -def synchronize(): - tmpdir = tempfile.mkdtemp() - appendoncedir = tempfile.mkdtemp() - try: - ac = apiclient.HTTPSClient() - myips = [] - ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n') - for line in ipaddrs: - isa = line.split() - if len(isa) < 3 or isa[1] != b'UP': - continue - for addr in isa[2:]: - if addr.startswith(b'fe80::') or addr.startswith(b'169.254'): - continue - addr = addr.split(b'/')[0] - if not isinstance(addr, str): - addr = addr.decode('utf8') - myips.append(addr) - data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) - if status >= 300: - sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(rsp.decode('utf8')) - sys.stderr.write('\n') - sys.stderr.flush() - return status - if status == 202: - lastrsp = '' - while status != 204: - time.sleep(1+(2*random.random())) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') - if not isinstance(rsp, str): - rsp = rsp.decode('utf8') - if status == 200: - lastrsp = rsp - pendpasswd = os.path.join(tmpdir, 'etc/passwd') - if os.path.exists(pendpasswd): - cm = CredMerger() - cm.read_passwd(pendpasswd, targfile=False) - cm.read_passwd('/etc/passwd', targfile=True) - cm.write_out('/etc/passwd') - pendgroup = os.path.join(tmpdir, 'etc/group') - if os.path.exists(pendgroup): - cm = CredMerger() - cm.read_group(pendgroup, targfile=False) - cm.read_group('/etc/group', targfile=True) - cm.write_out('/etc/group') - pendhosts = os.path.join(tmpdir, 'etc/hosts') - if os.path.exists(pendhosts): - cm = HostMerger() - cm.read_source(pendhosts) - cm.read_target('/etc/hosts') - cm.write_out('/etc/hosts') - for dirn in os.walk(appendoncedir): - for filen in dirn[2]: - appendonce(appendoncedir, os.path.join(dirn[0], filen)) - if lastrsp: - lastrsp = json.loads(lastrsp) - opts = lastrsp.get('options', {}) - for fname in opts: - uid = -1 - gid = -1 - for opt in opts[fname]: - if opt == 'owner': - try: - uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid - except KeyError: - uid = opts[fname][opt]['id'] - elif opt == 'group': - try: - gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid - except KeyError: - gid = opts[fname][opt]['id'] - elif opt == 'permissions': - os.chmod(fname, int(opts[fname][opt], 8)) - if uid != -1 or gid != -1: - os.chown(fname, uid, gid) - return status - finally: - shutil.rmtree(tmpdir) - shutil.rmtree(appendoncedir) - - -if __name__ == '__main__': - status = 202 - while status not in (204, 200): - try: - status = synchronize() - except Exception as e: - sys.stderr.write(str(e)) - sys.stderr.write('\n') - sys.stderr.flush() - status = 300 - if status not in (204, 200): - time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/el10-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el10-diskless/profiles/default/scripts/syncfileclient new file mode 120000 index 00000000..219244e6 --- /dev/null +++ b/confluent_osdeploy/el10-diskless/profiles/default/scripts/syncfileclient @@ -0,0 +1 @@ +../../../../el9-diskless/profiles/default/scripts/syncfileclient \ No newline at end of file