From a3b768c70f2aac4437180ca7f428409795015ad2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 20 Nov 2025 16:44:24 -0500 Subject: [PATCH] Draft bluefield deploymeent facilities --- confluent_osdeploy/bluefield/bfb-autoinstall | 74 +++++++++++ .../bluefield/hostscripts/bfb-autoinstall | 74 +++++++++++ .../profiles/default/bluefield.cfg.template | 71 ++++++++++ .../bluefield/profiles/default/nodedeploy-bfb | 125 ++++++++++++++++++ 4 files changed, 344 insertions(+) create mode 100644 confluent_osdeploy/bluefield/bfb-autoinstall create mode 100644 confluent_osdeploy/bluefield/hostscripts/bfb-autoinstall create mode 100644 confluent_osdeploy/bluefield/profiles/default/bluefield.cfg.template create mode 100644 confluent_osdeploy/bluefield/profiles/default/nodedeploy-bfb diff --git a/confluent_osdeploy/bluefield/bfb-autoinstall b/confluent_osdeploy/bluefield/bfb-autoinstall new file mode 100644 index 00000000..32ff47ed --- /dev/null +++ b/confluent_osdeploy/bluefield/bfb-autoinstall @@ -0,0 +1,74 @@ +#!/usr/bin/python3 +import glob +import gzip +import base64 +import os +import subprocess +import sys +import tempfile + +def collect_certificates(tmpdir): + certdata = '' + for cacert in glob.glob(f'{tmpdir}/*.pem'): + with open(cacert, 'r') as f: + certdata += f.read() + return certdata + +def embed_certificates(incfg, certdata): + if not certdata: + raise Exception('No certificates found to embed') + incfg = incfg.replace('%CONFLUENTCERTCOLL%', certdata) + return incfg + +def embed_identity(incfg, identityjson): + incfg = incfg.replace('%IDENTJSON%', identityjson) + return incfg + +def embed_apiclient(incfg, apiclient): + with open(apiclient, 'r') as f: + apiclientdata = f.read() + compressed = gzip.compress(apiclientdata.encode()) + encoded = base64.b64encode(compressed).decode() + incfg = incfg.replace('%APICLIENTZ64%', encoded) + return incfg + +def embed_data(tmpdir, outfile): + templatefile = f'{tmpdir}/bfb.cfg.template' + with open(templatefile, 'r') as f: + incfg = f.read() + + certdata = collect_certificates(tmpdir) + incfg = embed_certificates(incfg, certdata) + + with open(f'{tmpdir}/identity.json', 'r') as f: + identityjson = f.read() + + incfg = embed_identity(incfg, identityjson) + + incfg = embed_apiclient(incfg, f'{tmpdir}/../apiclient') + + with open(outfile, 'w') as f: + f.write(incfg) + +def get_identity_json(node): + identity_file = f'/var/lib/confluent/private/site/identity_files/{node}.json' + try: + with open(identity_file, 'r') as f: + return f.read() + except FileNotFoundError: + return None + +if __name__ == '__main__': + if len(sys.argv) != 4: + print("Usage: bfb-autoinstall ") + sys.exit(1) + + node = sys.argv[1] + bfbfile = sys.argv[2] + rshim = sys.argv[3] + + os.chdir(os.path.dirname(os.path.abspath(__file__))) + currdir = os.getcwd() + tempdir = tempfile.mkdtemp(prefix=f'bfb-autoinstall-{node}-') + embed_data(f'{currdir}/{node}', f'{tempdir}/bfb.cfg') + subprocess.check_call(['bfb-install', '-b', bfbfile, '-c', f'{tempdir}/bfb.cfg', '-r', rshim]) diff --git a/confluent_osdeploy/bluefield/hostscripts/bfb-autoinstall b/confluent_osdeploy/bluefield/hostscripts/bfb-autoinstall new file mode 100644 index 00000000..32ff47ed --- /dev/null +++ b/confluent_osdeploy/bluefield/hostscripts/bfb-autoinstall @@ -0,0 +1,74 @@ +#!/usr/bin/python3 +import glob +import gzip +import base64 +import os +import subprocess +import sys +import tempfile + +def collect_certificates(tmpdir): + certdata = '' + for cacert in glob.glob(f'{tmpdir}/*.pem'): + with open(cacert, 'r') as f: + certdata += f.read() + return certdata + +def embed_certificates(incfg, certdata): + if not certdata: + raise Exception('No certificates found to embed') + incfg = incfg.replace('%CONFLUENTCERTCOLL%', certdata) + return incfg + +def embed_identity(incfg, identityjson): + incfg = incfg.replace('%IDENTJSON%', identityjson) + return incfg + +def embed_apiclient(incfg, apiclient): + with open(apiclient, 'r') as f: + apiclientdata = f.read() + compressed = gzip.compress(apiclientdata.encode()) + encoded = base64.b64encode(compressed).decode() + incfg = incfg.replace('%APICLIENTZ64%', encoded) + return incfg + +def embed_data(tmpdir, outfile): + templatefile = f'{tmpdir}/bfb.cfg.template' + with open(templatefile, 'r') as f: + incfg = f.read() + + certdata = collect_certificates(tmpdir) + incfg = embed_certificates(incfg, certdata) + + with open(f'{tmpdir}/identity.json', 'r') as f: + identityjson = f.read() + + incfg = embed_identity(incfg, identityjson) + + incfg = embed_apiclient(incfg, f'{tmpdir}/../apiclient') + + with open(outfile, 'w') as f: + f.write(incfg) + +def get_identity_json(node): + identity_file = f'/var/lib/confluent/private/site/identity_files/{node}.json' + try: + with open(identity_file, 'r') as f: + return f.read() + except FileNotFoundError: + return None + +if __name__ == '__main__': + if len(sys.argv) != 4: + print("Usage: bfb-autoinstall ") + sys.exit(1) + + node = sys.argv[1] + bfbfile = sys.argv[2] + rshim = sys.argv[3] + + os.chdir(os.path.dirname(os.path.abspath(__file__))) + currdir = os.getcwd() + tempdir = tempfile.mkdtemp(prefix=f'bfb-autoinstall-{node}-') + embed_data(f'{currdir}/{node}', f'{tempdir}/bfb.cfg') + subprocess.check_call(['bfb-install', '-b', bfbfile, '-c', f'{tempdir}/bfb.cfg', '-r', rshim]) diff --git a/confluent_osdeploy/bluefield/profiles/default/bluefield.cfg.template b/confluent_osdeploy/bluefield/profiles/default/bluefield.cfg.template new file mode 100644 index 00000000..5f7b6a08 --- /dev/null +++ b/confluent_osdeploy/bluefield/profiles/default/bluefield.cfg.template @@ -0,0 +1,71 @@ +function bfb_modify_os() { + echo 'ubuntu:!' | chpasswd -e + mkdir -p /mnt/opt/confluent/bin/ + cat > /mnt/opt/confluent/bin/confluentbootstrap.sh << 'END_OF_EMBED' +#!/bin/bash + cat > /usr/local/share/ca-certificates/confluent.crt << 'END_OF_CERTS' +%CONFLUENTCERTCOLL% +END_OF_CERTS + update-ca-certificates + mkdir -p /opt/confluent/bin /etc/confluent/ + cp /usr/local/share/ca-certificates/confluent.crt /etc/confluent/ca.pem + cat > /opt/confluent/bin/apiclient.gz.b64 << 'END_OF_CLIENT' +%APICLIENTZ64% +END_OF_CLIENT + base64 -d /opt/confluent/bin/apiclient.gz.b64 | gunzip > /opt/confluent/bin/apiclient + cat > /etc/confluent/ident.json << 'END_OF_IDENT' +%IDENTJSON% +END_OF_IDENT + python3 /opt/confluent/bin/apiclient -i /etc/confluent/ident.json /confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg + PROFILE=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}') + ROOTPASS=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg | awk '{print $2}'|grep -v null) + if [ -n "$ROOTPASS" ]; then + echo root:$ROOTPASS | chpasswd -e + echo "ubuntu:$ROOTPASS" | chpasswd -e + else + echo 'ubuntu:!' | chpasswd -e + fi + python3 /opt/confluent/bin/apiclient /confluent-public/os/$PROFILE/scripts/functions > /etc/confluent/functions + touch /etc/confluent/confluent.deploycfg + bash /etc/confluent/functions run_remote_python confignet + bash /etc/confluent/functions run_remote setupssh + for cert in /etc/ssh/ssh*-cert.pub; do + if [ -s $cert ]; then + echo HostCertificate $cert >> /etc/ssh/sshd_config.d/90-confluent.conf + fi + done + mkdir -p /var/log/confluent + chmod 700 /var/log/confluent + touch /var/log/confluent/confluent-firstboot.log + touch /var/log/confluent/confluent-post.log + chmod 600 /var/log/confluent/confluent-post.log + chmod 600 /var/log/confluent/confluent-firstboot.log + exec >> /var/log/confluent/confluent-post.log + exec 2>> /var/log/confluent/confluent-post.log + bash /etc/confluent/functions run_remote_python syncfileclient + bash /etc/confluent/functions run_remote_parts post.d + bash /etc/confluent/functions run_remote_config post.d + exec >> /var/log/confluent/confluent-firstboot.log + exec 2>> /var/log/confluent/confluent-firstboot.log + bash /etc/confluent/functions run_remote_parts firstboot.d + bash /etc/confluent/functions run_remote_config firstboot.d + python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus -d 'status: staged' + python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus -d 'status: complete' + systemctl disable confluentbootstrap + rm /etc/systemd/system/confluentbootstrap.service +END_OF_EMBED + chmod +x /mnt/opt/confluent/bin/confluentbootstrap.sh + cat > /mnt/etc/systemd/system/confluentbootstrap.service << EOS +[Unit] +Description=First Boot Process +Requires=network-online.target +After=network-online.target + +[Service] +ExecStart=/opt/confluent/bin/confluentbootstrap.sh + +[Install] +WantedBy=multi-user.target +EOS + chroot /mnt systemctl enable confluentbootstrap +} \ No newline at end of file diff --git a/confluent_osdeploy/bluefield/profiles/default/nodedeploy-bfb b/confluent_osdeploy/bluefield/profiles/default/nodedeploy-bfb new file mode 100644 index 00000000..33d04cb2 --- /dev/null +++ b/confluent_osdeploy/bluefield/profiles/default/nodedeploy-bfb @@ -0,0 +1,125 @@ +#!/usr/bin/python3 + +import os +import sys +import tempfile +import glob +import shutil +import shlex +import subprocess +import select + +sys.path.append('/opt/lib/confluent/python') + +import confluent.sortutil as sortutil +import confluent.client as client + + +def prep_outdir(node): + tmpdir = tempfile.mkdtemp() + for certfile in glob.glob('/var/lib/confluent/public/site/tls/*.pem'): + basename = os.path.basename(certfile) + destfile = os.path.join(tmpdir, basename) + shutil.copy2(certfile, destfile) + subprocess.check_call(shlex.split(f'confetty set /nodes/{node}/deployment/ident_image=create')) + shutil.copy2(f'/var/lib/confluent/private/identity_files/{node}.json', os.path.join(tmpdir, 'identity.json')) + return tmpdir + +def exec_bfb_install(host, nodetorshim, bfbfile, installprocs, pipedesc, all, poller): + remotedir = subprocess.check_output(shlex.split(f'ssh {host} mktemp -d /tmp/bfb.XXXXXX')).decode().strip() + bfbbasename = os.path.basename(bfbfile) + subprocess.check_call(shlex.split(f'rsync -avz --info=progress2 {bfbfile} {host}:{remotedir}/{bfbbasename}')) + subprocess.check_call(shlex.split(f'rsync -avc --info=progress2 /opt/lib/confluent/osdeploy/bluefield/hostscripts/ {host}:{remotedir}/')) + for node in nodetorshim: + rshim = nodetorshim[node] + nodeoutdir = prep_outdir(node) + nodeprofile = subprocess.check_output(shlex.split(f'nodeattrib {node} deployment.pendingprofile')).decode().strip().split(':', 2)[2].strip() + shutil.copy2(f'/var/lib/confluent/public/os/{nodeprofile}/bfb.cfg.template', os.path.join(nodeoutdir, 'bfb.cfg.template')) + subprocess.check_call(shlex.split(f'rsync -avz {nodeoutdir}/ {host}:{remotedir}/{node}/')) + shutil.rmtree(nodeoutdir) + run_cmdv(node, shlex.split(f'ssh {host} sh /etc/confluent/functions confluentpython {remotedir}/bfb-autoinstall {node} {remotedir}/{bfbbasename} {rshim}'), all, poller, pipedesc) + + +def run_cmdv(node, cmdv, all, poller, pipedesc): + try: + nopen = subprocess.Popen( + cmdv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError as e: + if e.errno == 2: + sys.stderr.write('{0}: Unable to find local executable file "{1}"\n'.format(node, cmdv[0])) + return + raise + pipedesc[nopen.stdout.fileno()] = {'node': node, 'popen': nopen, + 'type': 'stdout', 'file': nopen.stdout} + pipedesc[nopen.stderr.fileno()] = {'node': node, 'popen': nopen, + 'type': 'stderr', 'file': nopen.stderr} + all.add(nopen.stdout) + poller.register(nopen.stdout, select.EPOLLIN) + all.add(nopen.stderr) + poller.register(nopen.stderr, select.EPOLLIN) + +if __name__ == '__main__': + + + if len(sys.argv) < 3: + print(f'Usage: {sys.argv[0]} [ ...]') + sys.exit(1) + + host = sys.argv[1] + bfbfile = sys.argv[2] + nodetorshim = {} + for arg in sys.argv[3:]: + node, rshim = arg.split(':') + nodetorshim[node] = rshim + + installprocs = {} + pipedesc = {} + all = set() + poller = select.epoll() + + exec_bfb_install(host, nodetorshim, bfbfile, installprocs, pipedesc, all, poller) + rdy = poller.poll(10) + pendingexecs = [] + exitcode = 0 + while all: + pernodeout = {} + for r in rdy: + r = r[0] + desc = pipedesc[r] + r = desc['file'] + node = desc['node'] + data = True + singlepoller = select.epoll() + singlepoller.register(r, select.EPOLLIN) + while data and singlepoller.poll(0): + data = r.readline() + if data: + if desc['type'] == 'stdout': + if node not in pernodeout: + pernodeout[node] = [] + pernodeout[node].append(data) + else: + data = client.stringify(data) + sys.stderr.write('{0}: {1}'.format(node, data)) + sys.stderr.flush() + else: + pop = desc['popen'] + ret = pop.poll() + if ret is not None: + exitcode = exitcode | ret + all.discard(r) + poller.unregister(r) + r.close() + if desc['type'] == 'stdout' and pendingexecs: + node, cmdv = pendingexecs.popleft() + run_cmdv(node, cmdv, all, poller, pipedesc) + singlepoller.close() + for node in sortutil.natural_sort(pernodeout): + for line in pernodeout[node]: + line = client.stringify(line) + sys.stdout.write('{0}: {1}'.format(node, line)) + sys.stdout.flush() + if all: + rdy = poller.poll(10) + +