From 8947ab08dab002d2370cdfd08bfe03d8d813ffcd Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 13:04:45 -0400 Subject: [PATCH 01/34] Add file to show the rpm versions at build time of genesis --- genesis/buildgenesis.sh | 12 +++++++++++- genesis/confluent-genesis.spec | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/genesis/buildgenesis.sh b/genesis/buildgenesis.sh index 1d5998a8..014c4f25 100644 --- a/genesis/buildgenesis.sh +++ b/genesis/buildgenesis.sh @@ -5,11 +5,21 @@ chmod +x /usr/lib/dracut/modules.d/97genesis/install /usr/lib/dracut/modules.d/9 mkdir -p boot/initramfs mkdir -p boot/efi/boot dracut --no-early-microcode --xz -N -m "genesis base" -f boot/initramfs/distribution $(uname -r) +tdir=$(mktemp -d) +tfile=$(mktemp) +cp boot/initramfs/distribution $tdir +cd $tdir +xzcat distribution|cpio -dumi +rm distribution +find . -type f -exec rpm -qf /{} \; 2> /dev/null | grep -v 'not owned' | sort -u > $tfile +cd - +rm -rf $tdir +cp $tfile rpmlist cp -f /boot/vmlinuz-$(uname -r) boot/kernel cp /boot/efi/EFI/BOOT/BOOTX64.EFI boot/efi/boot cp /boot/efi/EFI/centos/grubx64.efi boot/efi/boot/grubx64.efi mkdir -p ~/rpmbuild/SOURCES/ -tar cf ~/rpmbuild/SOURCES/confluent-genesis.tar boot +tar cf ~/rpmbuild/SOURCES/confluent-genesis.tar boot rpmlist rpmbuild -bb confluent-genesis.spec rm -rf /usr/lib/dracut/modules.d/97genesis cd - diff --git a/genesis/confluent-genesis.spec b/genesis/confluent-genesis.spec index b1a24a6d..30111120 100644 --- a/genesis/confluent-genesis.spec +++ b/genesis/confluent-genesis.spec @@ -28,6 +28,7 @@ find . -type f -exec chmod o+r {} + find . -type f -exec chmod -x {} + %files +/opt/confluent/genesis/%{arch}/rpmlist /opt/confluent/genesis/%{arch}/boot/efi/boot/BOOTX64.EFI /opt/confluent/genesis/%{arch}/boot/efi/boot/grubx64.efi /opt/confluent/genesis/%{arch}/boot/initramfs/distribution From 128cb61eaa83ea2f9fb5350fcc3dae6b7b5e5f4c Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 13:11:43 -0400 Subject: [PATCH 02/34] Fix directory traversal --- genesis/buildgenesis.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/genesis/buildgenesis.sh b/genesis/buildgenesis.sh index 014c4f25..fbf82e60 100644 --- a/genesis/buildgenesis.sh +++ b/genesis/buildgenesis.sh @@ -8,11 +8,11 @@ dracut --no-early-microcode --xz -N -m "genesis base" -f boot/initramfs/distribu tdir=$(mktemp -d) tfile=$(mktemp) cp boot/initramfs/distribution $tdir -cd $tdir +pushd $tdir xzcat distribution|cpio -dumi rm distribution find . -type f -exec rpm -qf /{} \; 2> /dev/null | grep -v 'not owned' | sort -u > $tfile -cd - +popd rm -rf $tdir cp $tfile rpmlist cp -f /boot/vmlinuz-$(uname -r) boot/kernel From 2f8846ee75392f85a116384b67002678d1e95fc4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 13:14:02 -0400 Subject: [PATCH 03/34] Further change directory traversal --- genesis/buildgenesis.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/genesis/buildgenesis.sh b/genesis/buildgenesis.sh index fbf82e60..3c5c9931 100644 --- a/genesis/buildgenesis.sh +++ b/genesis/buildgenesis.sh @@ -1,4 +1,4 @@ -cd $(dirname $0) +pushd $(dirname $0) cp -a 97genesis /usr/lib/dracut/modules.d/ cat /usr/lib/dracut/modules.d/97genesis/install-* > /usr/lib/dracut/modules.d/97genesis/install chmod +x /usr/lib/dracut/modules.d/97genesis/install /usr/lib/dracut/modules.d/97genesis/installkernel @@ -22,7 +22,7 @@ mkdir -p ~/rpmbuild/SOURCES/ tar cf ~/rpmbuild/SOURCES/confluent-genesis.tar boot rpmlist rpmbuild -bb confluent-genesis.spec rm -rf /usr/lib/dracut/modules.d/97genesis -cd - +popd # getting src rpms would be nice, but centos isn't consistent.. # /usr/lib/dracut/skipcpio /opt/confluent/genesis/x86_64/boot/initramfs/distribution | xzcat | cpio -dumiv # rpm -qf $(find . -type f | sed -e 's/^.//') |sort -u|grep -v 'not owned' > ../rpmlist From 71a698f771a8f36adb6c787e12324479270fb807 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 13:52:36 -0400 Subject: [PATCH 04/34] Carry over license content from donor rpms --- genesis/buildgenesis.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/genesis/buildgenesis.sh b/genesis/buildgenesis.sh index 3c5c9931..190cbeec 100644 --- a/genesis/buildgenesis.sh +++ b/genesis/buildgenesis.sh @@ -15,12 +15,23 @@ find . -type f -exec rpm -qf /{} \; 2> /dev/null | grep -v 'not owned' | sort -u popd rm -rf $tdir cp $tfile rpmlist +cp confluent-genesis.spec confluent-genesis-out.spec +echo %license >> confluent-genesis-out.spec +for r in $(cat rpmlist); do + #rpm -qi $r | grep ^License|sed -e 's/^.*:/${r}:/' >> licenselist + for l in $(rpm -qL $r); do + lo=${l#/usr/share/} + mkdir -p licenses/$(dirname $lo) + cp $l $lo + echo $lo >> confluent-genesis-out.spec + done +done cp -f /boot/vmlinuz-$(uname -r) boot/kernel cp /boot/efi/EFI/BOOT/BOOTX64.EFI boot/efi/boot cp /boot/efi/EFI/centos/grubx64.efi boot/efi/boot/grubx64.efi mkdir -p ~/rpmbuild/SOURCES/ tar cf ~/rpmbuild/SOURCES/confluent-genesis.tar boot rpmlist -rpmbuild -bb confluent-genesis.spec +rpmbuild -bb confluent-genesis-out.spec rm -rf /usr/lib/dracut/modules.d/97genesis popd # getting src rpms would be nice, but centos isn't consistent.. From 4d218757f3b194d411df56ba124d4418764b5cc6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 14:02:08 -0400 Subject: [PATCH 05/34] Amend license carry over logic --- genesis/buildgenesis.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/genesis/buildgenesis.sh b/genesis/buildgenesis.sh index 190cbeec..4b1cf704 100644 --- a/genesis/buildgenesis.sh +++ b/genesis/buildgenesis.sh @@ -23,14 +23,14 @@ for r in $(cat rpmlist); do lo=${l#/usr/share/} mkdir -p licenses/$(dirname $lo) cp $l $lo - echo $lo >> confluent-genesis-out.spec + echo /opt/confluent/genesis/x86_64/$lo >> confluent-genesis-out.spec done done cp -f /boot/vmlinuz-$(uname -r) boot/kernel cp /boot/efi/EFI/BOOT/BOOTX64.EFI boot/efi/boot cp /boot/efi/EFI/centos/grubx64.efi boot/efi/boot/grubx64.efi mkdir -p ~/rpmbuild/SOURCES/ -tar cf ~/rpmbuild/SOURCES/confluent-genesis.tar boot rpmlist +tar cf ~/rpmbuild/SOURCES/confluent-genesis.tar boot rpmlist licenses rpmbuild -bb confluent-genesis-out.spec rm -rf /usr/lib/dracut/modules.d/97genesis popd From 5d03eb886332ce8cb80ff989ba9190d898e55e1d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 14:15:07 -0400 Subject: [PATCH 06/34] Try to normalize license paths --- genesis/buildgenesis.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/genesis/buildgenesis.sh b/genesis/buildgenesis.sh index 4b1cf704..ba480ce5 100644 --- a/genesis/buildgenesis.sh +++ b/genesis/buildgenesis.sh @@ -21,9 +21,10 @@ for r in $(cat rpmlist); do #rpm -qi $r | grep ^License|sed -e 's/^.*:/${r}:/' >> licenselist for l in $(rpm -qL $r); do lo=${l#/usr/share/} + lo=${l#licenses/} mkdir -p licenses/$(dirname $lo) - cp $l $lo - echo /opt/confluent/genesis/x86_64/$lo >> confluent-genesis-out.spec + cp $l licenses/$lo + echo /opt/confluent/genesis/%{arch}/$lo >> confluent-genesis-out.spec done done cp -f /boot/vmlinuz-$(uname -r) boot/kernel From 061d4217ff906fec07e1680e974b1b239411b66d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 14:46:50 -0400 Subject: [PATCH 07/34] Further amend license handling --- genesis/buildgenesis.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/genesis/buildgenesis.sh b/genesis/buildgenesis.sh index ba480ce5..c93aebe6 100644 --- a/genesis/buildgenesis.sh +++ b/genesis/buildgenesis.sh @@ -1,4 +1,5 @@ pushd $(dirname $0) +rm -rf licenses cp -a 97genesis /usr/lib/dracut/modules.d/ cat /usr/lib/dracut/modules.d/97genesis/install-* > /usr/lib/dracut/modules.d/97genesis/install chmod +x /usr/lib/dracut/modules.d/97genesis/install /usr/lib/dracut/modules.d/97genesis/installkernel @@ -21,10 +22,10 @@ for r in $(cat rpmlist); do #rpm -qi $r | grep ^License|sed -e 's/^.*:/${r}:/' >> licenselist for l in $(rpm -qL $r); do lo=${l#/usr/share/} - lo=${l#licenses/} + lo=${lo#licenses/} mkdir -p licenses/$(dirname $lo) cp $l licenses/$lo - echo /opt/confluent/genesis/%{arch}/$lo >> confluent-genesis-out.spec + echo /opt/confluent/genesis/%{arch}/licenses/$lo >> confluent-genesis-out.spec done done cp -f /boot/vmlinuz-$(uname -r) boot/kernel From f009f6b8c2abd9f47497fb7de18b6d37fcab178a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 15:18:27 -0400 Subject: [PATCH 08/34] Trim to appropriate content for the license --- LICENSE | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/LICENSE b/LICENSE index d6456956..f433b1a5 100644 --- a/LICENSE +++ b/LICENSE @@ -175,28 +175,3 @@ of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. From 2c058fe6340622be515ddd4dce810fb1ceb40c75 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 15:46:29 -0400 Subject: [PATCH 09/34] Properly mark license files as license --- genesis/buildgenesis.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/genesis/buildgenesis.sh b/genesis/buildgenesis.sh index c93aebe6..8791eb84 100644 --- a/genesis/buildgenesis.sh +++ b/genesis/buildgenesis.sh @@ -17,7 +17,6 @@ popd rm -rf $tdir cp $tfile rpmlist cp confluent-genesis.spec confluent-genesis-out.spec -echo %license >> confluent-genesis-out.spec for r in $(cat rpmlist); do #rpm -qi $r | grep ^License|sed -e 's/^.*:/${r}:/' >> licenselist for l in $(rpm -qL $r); do @@ -25,7 +24,7 @@ for r in $(cat rpmlist); do lo=${lo#licenses/} mkdir -p licenses/$(dirname $lo) cp $l licenses/$lo - echo /opt/confluent/genesis/%{arch}/licenses/$lo >> confluent-genesis-out.spec + echo %license /opt/confluent/genesis/%{arch}/licenses/$lo >> confluent-genesis-out.spec done done cp -f /boot/vmlinuz-$(uname -r) boot/kernel From 72343105195fb1289b51e97cd8db9f8703d8b87e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 16:05:22 -0400 Subject: [PATCH 10/34] Move licensing to files instead of rpm description for vtbuffer --- confluent_vtbufferd/buildrpm | 2 +- .../confluent_vtbufferd.spec.tmpl | 21 +++++-------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/confluent_vtbufferd/buildrpm b/confluent_vtbufferd/buildrpm index 333f6acb..8312c348 100755 --- a/confluent_vtbufferd/buildrpm +++ b/confluent_vtbufferd/buildrpm @@ -4,7 +4,7 @@ if [ "$NUMCOMMITS" != "$VERSION" ]; then VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3` fi mkdir -p dist/confluent_vtbufferd-$VERSION -cp *.c *.h Makefile dist/confluent_vtbufferd-$VERSION +cp ../LICENSE *.c *.h Makefile dist/confluent_vtbufferd-$VERSION cd dist tar czf confluent_vtbufferd-$VERSION.tar.gz confluent_vtbufferd-$VERSION cd - diff --git a/confluent_vtbufferd/confluent_vtbufferd.spec.tmpl b/confluent_vtbufferd/confluent_vtbufferd.spec.tmpl index 24e0681e..dcf022b8 100644 --- a/confluent_vtbufferd/confluent_vtbufferd.spec.tmpl +++ b/confluent_vtbufferd/confluent_vtbufferd.spec.tmpl @@ -17,21 +17,6 @@ Url: https://github.com/lenovo/confluent/ %description Service for managing in-memory VT emulation for confluent. -Contains third party open source code: - -Copyright (c) 2017 Rob King -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the copyright holder nor the - names of contributors may be used to endorse or promote products - derived from this software without specific prior written permission. %prep %setup -n %{name}-%{version} -n %{name}-%{version} @@ -40,8 +25,12 @@ modification, are permitted provided that the following conditions are met: make %install -mkdir -p $RPM_BUILD_ROOT/opt/confluent/bin +mkdir -p $RPM_BUILD_ROOT/opt/confluent/bin $RPM_BUILD_ROOT/opt/confluent/share/licenses/vtbufferd cp vtbufferd $RPM_BUILD_ROOT/opt/confluent/bin/ +cp COPYING.tmt $RPM_BUILD_ROOT/opt/confluent/share/licenses/vtbufferd +cp LICENSE $RPM_BUILD_ROOT/opt/confluent/share/licenses/vtbufferd %files /opt/confluent/bin/vtbufferd +%license /opt/confluent/share/licenses/vtbufferd/COPYING.tmt +%license /opt/confluent/share/licenses/vtbufferd/LICENSE From 66f2ba98ecf11f2ded927b827f1795e0b5aea41f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 16:15:49 -0400 Subject: [PATCH 11/34] Set up confluent licensisng in setupdist context --- confluent_client/setup.py.tmpl | 1 + confluent_server/makesetup | 1 + confluent_server/setup.py.tmpl | 1 + 3 files changed, 3 insertions(+) diff --git a/confluent_client/setup.py.tmpl b/confluent_client/setup.py.tmpl index 34199a2b..9a12fc58 100644 --- a/confluent_client/setup.py.tmpl +++ b/confluent_client/setup.py.tmpl @@ -17,6 +17,7 @@ setup( author_email='jjohnson2@lenovo.com', url='http://xcat.sf.net/', packages=['confluent'], + license_files=['LICENSE'], scripts=scriptlist, data_files=data_files, ) diff --git a/confluent_server/makesetup b/confluent_server/makesetup index 44e7293f..f20e2d25 100755 --- a/confluent_server/makesetup +++ b/confluent_server/makesetup @@ -9,3 +9,4 @@ sed -e "s/#VERSION#/$VERSION/" setup.py.tmpl > setup.py if [ -f confluent/client.py ]; then echo '__version__ = "'$VERSION'"' > confluent/__init__.py fi +cp ../LICENSE . diff --git a/confluent_server/setup.py.tmpl b/confluent_server/setup.py.tmpl index e81fb811..74c52835 100644 --- a/confluent_server/setup.py.tmpl +++ b/confluent_server/setup.py.tmpl @@ -19,6 +19,7 @@ setup( 'confluent/plugins/shell/', 'confluent/collective/', 'confluent/plugins/configuration/'], + license_files=['LICENSE'], install_requires=['paramiko', 'pycrypto>=2.6', 'confluent_client>=0.1.0', 'eventlet', 'dnspython', 'netifaces', 'pysnmp', 'pyparsing', 'pyghmi>=1.0.44'], From 6270d57729bfac7a9fef11f8995e96a033ce89c4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 16:31:51 -0400 Subject: [PATCH 12/34] Add tmt license terms to vtbufferd package --- confluent_vtbufferd/COPYING.tmt | 13 +++++++++++++ confluent_vtbufferd/buildrpm | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 confluent_vtbufferd/COPYING.tmt diff --git a/confluent_vtbufferd/COPYING.tmt b/confluent_vtbufferd/COPYING.tmt new file mode 100644 index 00000000..791ea159 --- /dev/null +++ b/confluent_vtbufferd/COPYING.tmt @@ -0,0 +1,13 @@ +Copyright (c) 2017 Rob King +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of contributors may be used to endorse or promote products + derived from this software without specific prior written permission. diff --git a/confluent_vtbufferd/buildrpm b/confluent_vtbufferd/buildrpm index 8312c348..270a06ad 100755 --- a/confluent_vtbufferd/buildrpm +++ b/confluent_vtbufferd/buildrpm @@ -4,7 +4,7 @@ if [ "$NUMCOMMITS" != "$VERSION" ]; then VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3` fi mkdir -p dist/confluent_vtbufferd-$VERSION -cp ../LICENSE *.c *.h Makefile dist/confluent_vtbufferd-$VERSION +cp ../LICENSE COPYING.tmt *.c *.h Makefile dist/confluent_vtbufferd-$VERSION cd dist tar czf confluent_vtbufferd-$VERSION.tar.gz confluent_vtbufferd-$VERSION cd - From 5538f99376c60acccd430e196bc36c9c73ed5460 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 16:47:19 -0400 Subject: [PATCH 13/34] Support older setuptools for license packaging --- confluent_client/setup.py.tmpl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/confluent_client/setup.py.tmpl b/confluent_client/setup.py.tmpl index 9a12fc58..a3e4e122 100644 --- a/confluent_client/setup.py.tmpl +++ b/confluent_client/setup.py.tmpl @@ -1,7 +1,9 @@ from setuptools import setup import os -data_files = [('/etc/profile.d', ['confluent_env.sh', 'confluent_env.csh'])] +data_files = [('/etc/profile.d', ['confluent_env.sh', 'confluent_env.csh']), + ('/opt/confluent/share/licenses/confluent_client/', ['LICENSE']) + ] try: scriptlist = ['bin/{0}'.format(d) for d in os.listdir('bin/')] data_files.append(('/opt/confluent/share/man/man1', ['man/man1/' + x for x in os.listdir('man/man1')])) @@ -17,7 +19,6 @@ setup( author_email='jjohnson2@lenovo.com', url='http://xcat.sf.net/', packages=['confluent'], - license_files=['LICENSE'], scripts=scriptlist, - data_files=data_files, + data_files=data_files + ['LICENSE'], ) From 9333c999c9522695c6d7286c0a58c44adef64556 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 16:48:31 -0400 Subject: [PATCH 14/34] Adjust setup.py licensing for older setuptools --- confluent_server/setup.py.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/setup.py.tmpl b/confluent_server/setup.py.tmpl index 74c52835..c0dbe674 100644 --- a/confluent_server/setup.py.tmpl +++ b/confluent_server/setup.py.tmpl @@ -19,13 +19,13 @@ setup( 'confluent/plugins/shell/', 'confluent/collective/', 'confluent/plugins/configuration/'], - license_files=['LICENSE'], install_requires=['paramiko', 'pycrypto>=2.6', 'confluent_client>=0.1.0', 'eventlet', 'dnspython', 'netifaces', 'pysnmp', 'pyparsing', 'pyghmi>=1.0.44'], scripts=['bin/confluent', 'bin/confluentdbutil', 'bin/collective', 'bin/osdeploy'], data_files=[('/etc/init.d', ['sysvinit/confluent']), ('/usr/lib/sysctl.d', ['sysctl/confluent.conf']), + ('/opt/confluent/share/licenses/confluent_server', ['LICENSE']), ('/usr/lib/systemd/system', ['systemd/confluent.service']), ('/opt/confluent/lib/python/confluent/plugins/console/', [])], From 69b3aca815d4d3b92cd20ab26767cf49e7807bfd Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 16:54:06 -0400 Subject: [PATCH 15/34] Remove errant copy of LICENSE file in client --- confluent_client/setup.py.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_client/setup.py.tmpl b/confluent_client/setup.py.tmpl index a3e4e122..c447ae31 100644 --- a/confluent_client/setup.py.tmpl +++ b/confluent_client/setup.py.tmpl @@ -20,5 +20,5 @@ setup( url='http://xcat.sf.net/', packages=['confluent'], scripts=scriptlist, - data_files=data_files + ['LICENSE'], + data_files=data_files, ) From 060f639ab3a8eefce4e210a1167060ec5e155985 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 25 Apr 2022 17:02:35 -0400 Subject: [PATCH 16/34] Flag license files appropriately --- confluent_client/confluent_client.spec.tmpl | 1 + confluent_server/confluent_server.spec.tmpl | 1 + 2 files changed, 2 insertions(+) diff --git a/confluent_client/confluent_client.spec.tmpl b/confluent_client/confluent_client.spec.tmpl index b7aa4c8c..fc643ddd 100644 --- a/confluent_client/confluent_client.spec.tmpl +++ b/confluent_client/confluent_client.spec.tmpl @@ -50,4 +50,5 @@ python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUI rm -rf $RPM_BUILD_ROOT %files -f INSTALLED_FILES +%license /opt/confluent/share/licenses/confluent_client/LICENSE %defattr(-,root,root) diff --git a/confluent_server/confluent_server.spec.tmpl b/confluent_server/confluent_server.spec.tmpl index ffa83295..1a821114 100644 --- a/confluent_server/confluent_server.spec.tmpl +++ b/confluent_server/confluent_server.spec.tmpl @@ -99,4 +99,5 @@ true rm -rf $RPM_BUILD_ROOT %files -f INSTALLED_FILES +%license /opt/confluent/share/licenses/confluent_license/LICENSE %defattr(-,root,root) From 46ffe3f5f26c6b5911e5303e30ec0e093b94e0ed Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 26 Apr 2022 08:00:13 -0400 Subject: [PATCH 17/34] Fix license directory name for confluent server --- confluent_server/confluent_server.spec.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent_server.spec.tmpl b/confluent_server/confluent_server.spec.tmpl index 1a821114..eef95ae0 100644 --- a/confluent_server/confluent_server.spec.tmpl +++ b/confluent_server/confluent_server.spec.tmpl @@ -99,5 +99,5 @@ true rm -rf $RPM_BUILD_ROOT %files -f INSTALLED_FILES -%license /opt/confluent/share/licenses/confluent_license/LICENSE +%license /opt/confluent/share/licenses/confluent_server/LICENSE %defattr(-,root,root) From e842c2ddc8adf4191e3a4c9fd73ce0cf93bfba08 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 26 Apr 2022 08:09:50 -0400 Subject: [PATCH 18/34] Add licensing to osdeploy package --- confluent_osdeploy/buildrpm | 2 +- confluent_osdeploy/confluent_osdeploy.spec.tmpl | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/confluent_osdeploy/buildrpm b/confluent_osdeploy/buildrpm index 84db3482..9d7ff45f 100755 --- a/confluent_osdeploy/buildrpm +++ b/confluent_osdeploy/buildrpm @@ -5,7 +5,7 @@ if [ "$NUMCOMMITS" != "$VERSION" ]; then fi sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy.spec.tmpl > confluent_osdeploy.spec cd .. -tar Jcvf confluent_osdeploy.tar.xz confluent_osdeploy +tar Jcvf confluent_osdeploy.tar.xz confluent_osdeploy LICENSE mv confluent_osdeploy.tar.xz ~/rpmbuild/SOURCES/ cd - mkdir -p el9bin/opt/confluent/bin diff --git a/confluent_osdeploy/confluent_osdeploy.spec.tmpl b/confluent_osdeploy/confluent_osdeploy.spec.tmpl index de5a247d..8cc0c95b 100644 --- a/confluent_osdeploy/confluent_osdeploy.spec.tmpl +++ b/confluent_osdeploy/confluent_osdeploy.spec.tmpl @@ -76,6 +76,8 @@ cp -a esxi7out esxi6out cp -a esxi7 esxi6 %install +mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/ +cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/ for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 esxi6 esxi7 coreos; do mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles @@ -98,3 +100,4 @@ find %{buildroot}/opt/confluent/lib/osdeploy/ -name .gitignore -exec rm -f {} + %files /opt/confluent/lib/osdeploy +%license /opt/confluent/share/licenses/confluent_osdeploy/LICENSE From 7da8a2ec4ca6d3c105d4b36964fec21901992cd6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 26 Apr 2022 08:16:12 -0400 Subject: [PATCH 19/34] Properly stage the license file for osdeploy --- confluent_osdeploy/buildrpm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/confluent_osdeploy/buildrpm b/confluent_osdeploy/buildrpm index 9d7ff45f..89b9ebbc 100755 --- a/confluent_osdeploy/buildrpm +++ b/confluent_osdeploy/buildrpm @@ -5,7 +5,8 @@ if [ "$NUMCOMMITS" != "$VERSION" ]; then fi sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy.spec.tmpl > confluent_osdeploy.spec cd .. -tar Jcvf confluent_osdeploy.tar.xz confluent_osdeploy LICENSE +cp ../LICENSE . +tar Jcvf confluent_osdeploy.tar.xz confluent_osdeploy mv confluent_osdeploy.tar.xz ~/rpmbuild/SOURCES/ cd - mkdir -p el9bin/opt/confluent/bin From a738be3b53ba7e00b114bacb32bcb5df914b6d8a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 26 Apr 2022 08:18:31 -0400 Subject: [PATCH 20/34] Add license file to imgutil packaging --- imgutil/buildrpm | 1 + imgutil/confluent_imgutil.spec.tmpl | 3 +++ 2 files changed, 4 insertions(+) diff --git a/imgutil/buildrpm b/imgutil/buildrpm index 27687d6b..87631ac0 100755 --- a/imgutil/buildrpm +++ b/imgutil/buildrpm @@ -5,6 +5,7 @@ if [ "$NUMCOMMITS" != "$VERSION" ]; then VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3` fi sed -e "s/#VERSION#/$VERSION/" confluent_imgutil.spec.tmpl > confluent_imgutil.spec +cp ../LICENSE . cd .. tar Jcvf confluent_imgutil.tar.xz imgutil mv confluent_imgutil.tar.xz ~/rpmbuild/SOURCES/ diff --git a/imgutil/confluent_imgutil.spec.tmpl b/imgutil/confluent_imgutil.spec.tmpl index 3b31b232..a5c93f42 100644 --- a/imgutil/confluent_imgutil.spec.tmpl +++ b/imgutil/confluent_imgutil.spec.tmpl @@ -21,6 +21,8 @@ mkdir -p opt/confluent/bin mv imgutil opt/confluent/bin/ chmod a+x opt/confluent/bin/imgutil mv ubuntu suse15 el7 el8 opt/confluent/lib/imgutil/ +mkdir -p opt/confluent/share/licenses/confluent_imgutil +cp LICENSE opt/confluent/share/licenses/confluent_imgutil %install cp -a opt %{buildroot}/ @@ -28,3 +30,4 @@ cp -a opt %{buildroot}/ %files /opt/confluent/bin/imgutil /opt/confluent/lib/imgutil +%license /opt/confluent/share/licenses/confluent_imgutil/LICENSE From 48fd49663776b779de29dde61470d2e74f809b9b Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 27 Apr 2022 15:09:40 -0400 Subject: [PATCH 21/34] Fix discovery uuid cross-reference The SSDP does an endian scramble of the smm uuid, fix it, and also update the core to prefer the processed enclosure.uuid attribute. --- confluent_server/confluent/discovery/core.py | 4 +++- confluent_server/confluent/discovery/handlers/xcc.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/discovery/core.py b/confluent_server/confluent/discovery/core.py index 7ca752a4..11f9b28b 100644 --- a/confluent_server/confluent/discovery/core.py +++ b/confluent_server/confluent/discovery/core.py @@ -947,7 +947,9 @@ def get_node_by_uuid_or_mac(uuidormac): def get_nodename_from_enclosures(cfg, info): nodename = None - cuuid = info.get('attributes', {}).get('chassis-uuid', [None])[0] + cuuid = info.get('enclosure.uuid', None) + if not cuuid: + cuuid = info.get('attributes', {}).get('chassis-uuid', [None])[0] if cuuid and cuuid in nodes_by_uuid: encl = nodes_by_uuid[cuuid] bay = info.get('enclosure.bay', None) diff --git a/confluent_server/confluent/discovery/handlers/xcc.py b/confluent_server/confluent/discovery/handlers/xcc.py index 537cf285..a4723e31 100644 --- a/confluent_server/confluent/discovery/handlers/xcc.py +++ b/confluent_server/confluent/discovery/handlers/xcc.py @@ -30,6 +30,15 @@ import struct getaddrinfo = eventlet.support.greendns.getaddrinfo +def fixuuid(baduuid): + # SMM dumps it out in hex + uuidprefix = (baduuid[:8], baduuid[9:13], baduuid[14:18]) + a = codecs.encode(struct.pack(' Date: Mon, 2 May 2022 17:04:13 -0400 Subject: [PATCH 22/34] Note addition of enclosure uuid to nodediscover This will help sort out dense invorenments more easily, particularly if no switch or chained. --- confluent_client/bin/nodediscover | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_client/bin/nodediscover b/confluent_client/bin/nodediscover index 84daa06d..a14b7358 100755 --- a/confluent_client/bin/nodediscover +++ b/confluent_client/bin/nodediscover @@ -48,6 +48,7 @@ columnmapping = { 'Advertised IP': 'otheripaddrs', 'Other IP': 'otheripaddrs', } +#TODO: add chassis uuid def print_disco(options, session, currmac, outhandler, columns): From 98d22ffabc797935c03bda1b22d21ce0b81efc19 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 3 May 2022 08:29:43 -0400 Subject: [PATCH 23/34] Add a utility to induce an inventory --- misc/forceinventory.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 misc/forceinventory.py diff --git a/misc/forceinventory.py b/misc/forceinventory.py new file mode 100644 index 00000000..c7f68195 --- /dev/null +++ b/misc/forceinventory.py @@ -0,0 +1,9 @@ +import pyghmi.ipmi.command as cmd +import sys +import os +# alternatively, the following ipmi raw sequence: +# 0x3a 0xc4 0x3 0x0 0x21 0x1 0x9d 0x2f 0x76 0x32 0x2f 0x69 0x62 0x6d 0x63 0x2f 0x75 0x65 0x66 0x69 0x2f 0x66 0x6f 0x72 0x63 0x65 0x2d 0x69 0x6e 0x76 0x65 0x6e 0x74 0x6f 0x72 0x79 0x11 0x1 + +c = cmd.Command(sys.argv[1], os.environ['XCCUSER'], os.environ['XCCPASS'], verifycallback=lambda x: True) +c.oem_init() +c._oem.immhandler.set_property('/v2/ibmc/uefi/force-inventory', 1) From 0301f5ca680aa1b4037c340e213a906c6b11e6e7 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 May 2022 06:44:07 -0400 Subject: [PATCH 24/34] Fix imgutil pack expecting addpackagelist The attribute only populates on a different subcommand, so only process if the attribute exists. --- imgutil/imgutil | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 500d0d18..a5706814 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -394,10 +394,11 @@ class OsHandler(object): except AttributeError: pkglist = '' self.addpkglists = [] - for plist in args.addpackagelist: - if os.path.exists(os.path.abspath(plist)): - plist = os.path.abspath(plist) - self.addpkglists.append(plist) + if hasattr(args, 'addpackagelist'): + for plist in args.addpackagelist: + if os.path.exists(os.path.abspath(plist)): + plist = os.path.abspath(plist) + self.addpkglists.append(plist) if pkglist: if os.path.exists(os.path.abspath(pkglist)): pkglist = os.path.abspath(pkglist) From b2feb62d8a94452bc904ba7c0a67def08281d1c6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 May 2022 09:25:49 -0400 Subject: [PATCH 25/34] Add ubuntu22.04 profile Ubuntu 22.04 makes some changes, notably removing the custom-installation hooks. Change to injecting our modifications more directly to where the custom-installation hooks used to be. --- .../initramfs/conf/conf.d/confluent | 8 + .../initramfs/custom-installation/post.sh | 7 + .../initramfs/custom-installation/pre.sh | 6 + .../scripts/casper-bottom/99confluent | 75 +++++ .../initramfs/scripts/init-premount/confluent | 31 ++ .../profiles/default/autoinstall/meta-data | 0 .../profiles/default/autoinstall/user-data | 17 ++ .../profiles/default/initprofile.sh | 7 + .../ubuntu22.04/profiles/default/profile.yaml | 3 + .../default/scripts/firstboot.d/.gitignore | 0 .../profiles/default/scripts/firstboot.sh | 21 ++ .../profiles/default/scripts/functions | 196 +++++++++++++ .../profiles/default/scripts/getinstalldisk | 88 ++++++ .../default/scripts/post.d/.gitignore | 0 .../profiles/default/scripts/post.sh | 85 ++++++ .../profiles/default/scripts/pre.sh | 29 ++ .../profiles/default/scripts/syncfileclient | 272 ++++++++++++++++++ 17 files changed, 845 insertions(+) create mode 100644 confluent_osdeploy/ubuntu22.04/initramfs/conf/conf.d/confluent create mode 100755 confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/post.sh create mode 100755 confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/pre.sh create mode 100644 confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent create mode 100755 confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/meta-data create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/profile.yaml create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.d/.gitignore create mode 100755 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/functions create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.d/.gitignore create mode 100755 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh create mode 100755 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/conf/conf.d/confluent b/confluent_osdeploy/ubuntu22.04/initramfs/conf/conf.d/confluent new file mode 100644 index 00000000..8d11e9f5 --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/initramfs/conf/conf.d/confluent @@ -0,0 +1,8 @@ +if ! grep console= /proc/cmdline > /dev/null; then + /opt/confluent/bin/autocons > /custom-installation/autocons.info + cons=$(cat /custom-installation/autocons.info) + if [ ! -z "$cons" ]; then + echo "Auto-detected serial console: $cons" > ${cons%,*} + fi +fi +echo /scripts/init-premount/confluent >> /scripts/init-premount/ORDER diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/post.sh b/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/post.sh new file mode 100755 index 00000000..5bd43bc6 --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/post.sh @@ -0,0 +1,7 @@ +#!/bin/bash +deploycfg=/custom-installation/confluent/confluent.deploycfg +confluent_mgr=$(grep ^deploy_server $deploycfg|awk '{print $2}') +confluent_profile=$(grep ^profile: $deploycfg|awk '{print $2}') +export deploycfg confluent_mgr confluent_profile +curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/post.sh > /tmp/post.sh +. /tmp/post.sh diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/pre.sh b/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/pre.sh new file mode 100755 index 00000000..70def99c --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/pre.sh @@ -0,0 +1,6 @@ +#!/bin/bash +deploycfg=/custom-installation/confluent/confluent.deploycfg +confluent_mgr=$(grep ^deploy_server $deploycfg|awk '{print $2}') +confluent_profile=$(grep ^profile: $deploycfg|awk '{print $2}') +curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/pre.sh > /tmp/pre.sh +. /tmp/pre.sh diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent b/confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent new file mode 100644 index 00000000..fc68c7fb --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent @@ -0,0 +1,75 @@ +cp -a /custom-installation/ /root/custom-installation +cd /root +cat /tls/*.0 >> /root/etc/ssl/certs/ca-certificates.crt +mkdir -p /root/custom-installation/ssh +mkdir -p /root/custom-installation/tls +cp /ssh/* /root/custom-installation/ssh +cp /tls/* /root/custom-installation/tls +NODENAME=$(grep ^NODENAME: /custom-installation/confluent/confluent.info|awk '{print $2}') +MGR=$(grep ^EXTMGRINFO: /custom-installation/confluent/confluent.info |awk -F'|' '{print $1 " " $3}'|grep " 1$" | head -n 1 | awk '{print $2}') +MGR=$(grep ^MANAGER: /custom-installation/confluent/confluent.info|head -n 1| awk '{print $2}') +MGTIFACE=$(grep $MGR /custom-installation/confluent/confluent.info | grep ^EXTMGRINFO: | head -n 1 | awk -F'|' '{print $2}') +oum=$(umask) +umask 077 +chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey +MGR=[$MGR] +deploycfg=/root/custom-installation/confluent/confluent.deploycfg +if [ -z "$MGTIFACE" ]; then + chroot . usr/bin/curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg +else + chroot . usr/bin/curl -f -H "CONFLUENT_MGTIFACE: $MGTIFACE" -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg +fi +umask $oum +nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1) +nic=$(ip link |grep ^$nic:|awk '{print $2}') +DEVICE=${nic%:} +ipv4m=$(grep ^ipv4_method $deploycfg|awk '{print$2}') +. /scripts/functions +if [ "$ipv4m" = "dhcp" ]; then + IP=dhcp + configure_networking +elif [ "$ipv4m" = "static" ]; then + v4addr=$(grep ^ipv4_address: $deploycfg) + v4addr=${v4addr#ipv4_address: } + v4gw=$(grep ^ipv4_gateway: $deploycfg) + v4gw=${v4gw#ipv4_gateway: } + if [ "$v4gw" = "null" ]; then + v4gw="" + fi + v4nm=$(grep ipv4_netmask: $deploycfg) + v4nm=${v4nm#ipv4_netmask: } + dnsdomain=$(grep ^dnsdomain: $deploycfg) + dnsdomain=${dnsdomain#dnsdomain: } + if [ "$dnsdomain" = "null" ]; then dnsdomain=""; fi + dns=$(grep -A1 ^nameservers: $deploycfg|head -n 2|tail -n 1|sed -e 's/^- //'|sed -e "s/''//") + { + echo "DEVICE='$DEVICE'" + echo "PROTO='none'" + echo "IPV4PROTO='none'" + echo "IPV4ADDR='$v4addr'" + echo "IPV4NETMASK='$v4nm'" + echo "IPV4BROADCAST='$v4nm'" + echo "IPV4GATEWAY='$v4gw'" + echo "IPV4DNS1='$dns'" + echo "HOSTNAME='$NODENAME'" + echo "DNSDOMAIN='$dnsdomain'" + echo "DOMAINSEARCH='$dnsdomain'" + } > "/run/net-$DEVICE.conf" + configure_networking +else + IP=off +fi +ipv4s=$(grep ^deploy_server $deploycfg|awk '{print $2}') +osprofile=$(cat /custom-installation/confluent/osprofile) +fcmdline="$(cat /custom-installation/confluent/cmdline.orig) autoinstall ds=nocloud-net;s=https://${ipv4s}/confluent-public/os/${osprofile}/autoinstall/" +if [ -f /custom-installation/autocons.info ]; then + cons=$(cat /custom-installation/autocons.info) +fi +if [ ! -z "$cons" ]; then + echo "Installation will proceed on graphics console, autoconsole not supported during autoinstall phase" > ${cons%,*} + echo "Progress can be checked by using ssh to access and running the screendump command" > ${cons%,*} + echo ${cons%,*} > /root/tmp/autoconsdev + #fcmdline="$fcmdline console=${cons#/dev/}" +fi +echo $fcmdline > /custom-installation/confluent/fakecmdline +/scripts/casper-bottom/58server_network diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent b/confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent new file mode 100755 index 00000000..ffc13c1b --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent @@ -0,0 +1,31 @@ +cd /sys/class/net +for nic in *; do + ip link set $nic up +done +mkdir -p /custom-installation +cp -a /opt/confluent /custom-installation +touch /custom-installation/confluent/confluent.info +while ! grep NODENAME /custom-installation/confluent/confluent.info; do + /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info +done +MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]" +osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline) +cat /proc/cmdline > /custom-installation/confluent/cmdline.orig +if [ -f /custom-installation/autocons.info ]; then + cons=$(cat /custom-installation/autocons.info) +fi +if [ ! -z "$cons" ]; then + echo "Preparing to deploy $osprofile from $MGR" > ${cons%,*} +fi +echo "Preparing to deploy $osprofile from $MGR" +echo $osprofile > /custom-installation/confluent/osprofile +echo URL=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf +fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso" +if [ ! -z "$cons" ]; then + fcmdline="$fcmdline console=${cons#/dev/}" +fi +echo $fcmdline > /custom-installation/confluent/fakecmdline +mount -o bind /custom-installation/confluent/fakecmdline /proc/cmdline +echo '/scripts/casper-bottom/99confluent "$@"' >> /scripts/casper-bottom/ORDER + + diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/meta-data b/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/meta-data new file mode 100644 index 00000000..e69de29b diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data b/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data new file mode 100644 index 00000000..5b6c9894 --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data @@ -0,0 +1,17 @@ +#cloud-config +autoinstall: + version: 1 + early-commands: + - /custom-installation/pre.sh + late-commands: + - /custom-installation/post.sh + ssh: + install-server: true + storage: + layout: + name: lvm + match: + path: "%%INSTALLDISK%%" + user-data: + runcmd: + - /etc/confluent/firstboot.sh diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh new file mode 100644 index 00000000..20e12471 --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh @@ -0,0 +1,7 @@ +#!/bin/sh +sed -i 's/label: ubuntu/label: Ubuntu/' $2/profile.yaml && \ +ln -s $1/casper/vmlinuz $2/boot/kernel && \ +ln -s $1/casper/initrd $2/boot/initramfs/distribution && \ +mkdir -p $2/boot/efi/boot && \ +ln -s $1/EFI/boot/* $2/boot/efi/boot + diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/profile.yaml b/confluent_osdeploy/ubuntu22.04/profiles/default/profile.yaml new file mode 100644 index 00000000..b76cdfbf --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/profile.yaml @@ -0,0 +1,3 @@ +label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile) +kernelargs: quiet osprofile=%%PROFILE%% +#installedargs: example # These arguments would be added to the installed system diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.d/.gitignore b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.d/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh new file mode 100755 index 00000000..d90d5f7f --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh @@ -0,0 +1,21 @@ +#!/bin/bash +echo "Confluent first boot is running" +cp -a /etc/confluent/ssh/* /etc/ssh/ +systemctl restart sshd +rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}') +if [ ! -z "$rootpw" -a "$rootpw" != "null" ]; then + echo root:$rootpw | chpasswd -e +fi +nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}') +confluent_apikey=$(cat /etc/confluent/confluent.apikey) +confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg |awk '{print $2}') +while ! ping -c 1 $confluent_mgr >& /dev/null; do + sleep 1 +done +hostnamectl set-hostname $(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}') +touch /etc/cloud/cloud-init.disabled +source /etc/confluent/functions + +run_remote_parts firstboot.d +run_remote_config firstboot.d +curl --capath /etc/confluent/tls -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/functions b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/functions new file mode 100644 index 00000000..d70c63db --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/functions @@ -0,0 +1,196 @@ +#!/bin/bash +function test_mgr() { + if curl -s https://${1}/confluent-api/ > /dev/null; then + return 0 + fi + return 1 +} + +function confluentpython() { + if [ -x /usr/libexec/platform-python ]; then + /usr/libexec/platform-python $* + elif [ -x /usr/bin/python3 ]; then + /usr/bin/python3 $* + elif [ -x /usr/bin/python ]; then + /usr/bin/python $* + elif [ -x /usr/bin/python2 ]; then + /usr/bin/python2 $* + fi +} + +function set_confluent_vars() { + if [ -z "$nodename" ]; then + nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}') + fi + if [[ "$confluent_mgr" == *"%"* ]]; then + confluent_mgr="" + fi + if [ -z "$confluent_mgr" ]; then + confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') + if ! test_mgr $confluent_mgr; then + confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') + if [[ "$confluent_mgr" = *":"* ]]; then + confluent_mgr="[$confluent_mgr]" + fi + fi + if ! test_mgr $confluent_mgr; then + BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//') + OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//') + for confluent_mgr in $BESTMGRS $OKMGRS; do + if [[ $confluent_mgr == *":"* ]]; then + confluent_mgr="[$confluent_mgr]" + fi + if test_mgr $confluent_mgr; then + break + fi + done + fi + fi + if [ -z "$confluent_profile" ]; then + confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //') + fi +} + +fetch_remote() { + curlargs="" + if [ -f /etc/confluent/ca.pem ]; then + curlargs=" --cacert /etc/confluent/ca.pem" + fi + set_confluent_vars + mkdir -p $(dirname $1) + curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1 + if [ $? != 0 ]; then echo $1 failed to download; return 1; fi +} + +source_remote_parts() { + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + apiclient=/opt/confluent/bin/apiclient + if [ -f /etc/confluent/apiclient ]; then + apiclient=/etc/confluent/apiclient + fi + scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //') + for script in $scriptlist; do + source_remote $1/$script + done + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir +} + +run_remote_parts() { + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + apiclient=/opt/confluent/bin/apiclient + if [ -f /etc/confluent/apiclient ]; then + apiclient=/etc/confluent/apiclient + fi + scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //') + for script in $scriptlist; do + run_remote $1/$script + done + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir +} + +source_remote() { + set_confluent_vars + unsettmpdir=0 + echo + echo '---------------------------------------------------------------------------' + echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ + if [ -z "$confluentscripttmpdir" ]; then + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + unsettmpdir=1 + fi + echo Sourcing from $confluentscripttmpdir + cd $confluentscripttmpdir + fetch_remote $1 + if [ $? != 0 ]; then echo $1 failed to download; return 1; fi + chmod +x $1 + cmd=$1 + shift + source ./$cmd + cd - > /dev/null + if [ "$unsettmpdir" = 1 ]; then + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir + unsettmpdir=0 + fi + rm -rf $confluentscripttmpdir + return $retcode +} + +run_remote() { + requestedcmd="'$*'" + unsettmpdir=0 + set_confluent_vars + echo + echo '---------------------------------------------------------------------------' + echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ + if [ -z "$confluentscripttmpdir" ]; then + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + unsettmpdir=1 + fi + echo Executing in $confluentscripttmpdir + cd $confluentscripttmpdir + fetch_remote $1 + if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi + chmod +x $1 + cmd=$1 + if [ -x /usr/bin/chcon ]; then + chcon system_u:object_r:bin_t:s0 $cmd + fi + shift + ./$cmd $* + retcode=$? + if [ $retcode -ne 0 ]; then + echo "$requestedcmd exited with code $retcode" + fi + cd - > /dev/null + if [ "$unsettmpdir" = 1 ]; then + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir + unsettmpdir=0 + fi + return $retcode +} + +run_remote_python() { + echo + set_confluent_vars + if [ -f /etc/confluent/ca.pem ]; then + curlargs=" --cacert /etc/confluent/ca.pem" + fi + echo '---------------------------------------------------------------------------' + echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/ + confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX) + echo Executing in $confluentscripttmpdir + cd $confluentscripttmpdir + mkdir -p $(dirname $1) + curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1 + if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi + confluentpython $* + retcode=$? + echo "'$*' exited with code $retcode" + cd - > /dev/null + rm -rf $confluentscripttmpdir + unset confluentscripttmpdir + return $retcode +} + +run_remote_config() { + echo + set_confluent_vars + apiclient=/opt/confluent/bin/apiclient + if [ -f /etc/confluent/apiclient ]; then + apiclient=/etc/confluent/apiclient + fi + echo '---------------------------------------------------------------------------' + echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile + confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {} + confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204 + echo + echo 'Completed remote configuration' + echo '---------------------------------------------------------------------------' + return +} +#If invoked as a command, use the arguments to actually run a function +(return 0 2>/dev/null) || $1 "${@:2}" diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk new file mode 100644 index 00000000..4af31c0b --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk @@ -0,0 +1,88 @@ +import subprocess +import os + +class DiskInfo(object): + def __init__(self, devname): + self.name = devname + self.wwn = None + self.path = None + self.model = '' + self.size = 0 + self.driver = None + self.mdcontainer = '' + devnode = '/dev/{0}'.format(devname) + qprop = subprocess.check_output( + ['udevadm', 'info', '--query=property', devnode]) + if not isinstance(qprop, str): + qprop = qprop.decode('utf8') + for prop in qprop.split('\n'): + if '=' not in prop: + continue + k, v = prop.split('=', 1) + if k == 'DEVTYPE' and v != 'disk': + raise Exception('Not a disk') + elif k == 'DM_NAME': + raise Exception('Device Mapper') + elif k == 'ID_MODEL': + self.model = v + elif k == 'DEVPATH': + self.path = v + elif k == 'ID_WWN': + self.wwn = v + elif k == 'MD_CONTAINER': + self.mdcontainer = v + attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode]) + if not isinstance(attrs, str): + attrs = attrs.decode('utf8') + for attr in attrs.split('\n'): + if '==' not in attr: + continue + k, v = attr.split('==', 1) + k = k.strip() + if k == 'ATTRS{size}': + self.size = v.replace('"', '') + elif (k == 'DRIVERS' and not self.driver + and v not in ('"sd"', '""')): + self.driver = v.replace('"', '') + if not self.driver and 'imsm' not in self.mdcontainer: + raise Exception("No driver detected") + + @property + def priority(self): + if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'): + return 0 + if 'imsm' in self.mdcontainer: + return 1 + if self.driver == 'ahci': + return 2 + if self.driver.startswith('megaraid'): + return 3 + if self.driver.startswith('mpt'): + return 4 + return 99 + + def __repr__(self): + return repr({ + 'name': self.name, + 'path': self.path, + 'wwn': self.wwn, + 'driver': self.driver, + 'size': self.size, + 'model': self.model, + }) + + +def main(): + disks = [] + for disk in sorted(os.listdir('/sys/class/block')): + try: + disk = DiskInfo(disk) + disks.append(disk) + except Exception as e: + print("Skipping {0}: {1}".format(disk, str(e))) + nd = [x.name for x in sorted(disks, key=lambda x: x.priority)] + if nd: + open('/tmp/installdisk', 'w').write(nd[0]) + +if __name__ == '__main__': + main() diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.d/.gitignore b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.d/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh new file mode 100755 index 00000000..6c99735c --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh @@ -0,0 +1,85 @@ +#!/bin/bash +cp -a /root/.ssh /target/root/ +mkdir -p /target/etc/confluent/ssh/sshd_config.d/ +chmod 700 /target/etc/confluent +cp /custom-installation/confluent/* /target/etc/confluent/ +cp -a /custom-installation/tls /target/etc/confluent/ +chmod go-rwx /etc/confluent/* +for i in /custom-installation/ssh/*.ca; do + echo '@cert-authority *' $(cat $i) >> /target/etc/ssh/ssh_known_hosts +done + +cp -a /etc/ssh/ssh_host* /target/etc/confluent/ssh/ +cp -a /etc/ssh/sshd_config.d/confluent.conf /target/etc/confluent/ssh/sshd_config.d/ +sshconf=/target/etc/ssh/ssh_config +if [ -d /target/etc/ssh/ssh_config.d/ ]; then + sshconf=/target/etc/ssh/ssh_config.d/01-confluent.conf +fi +echo 'Host *' >> $sshconf +echo ' HostbasedAuthentication yes' >> $sshconf +echo ' EnableSSHKeysign yes' >> $sshconf +echo ' HostbasedKeyTypes *ed25519*' >> $sshconf + +curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /target/etc/confluent/firstboot.sh +curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /target/etc/confluent/functions +source /target/etc/confluent/functions +chmod +x /target/etc/confluent/firstboot.sh +cp /tmp/allnodes /target/root/.shosts +cp /tmp/allnodes /target/etc/ssh/shosts.equiv +if grep ^ntpservers: /target/etc/confluent/confluent.deploycfg > /dev/null; then + ntps=$(sed -n '/^ntpservers:/,/^[^-]/p' /target/etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //' | paste -sd ' ') + sed -i "s/#NTP=/NTP=$ntps/" /target/etc/systemd/timesyncd.conf +fi +textcons=$(grep ^textconsole: /target/etc/confluent/confluent.deploycfg |awk '{print $2}') +updategrub=0 +if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then + cons="" + if [ -f /custom-installation/autocons.info ]; then + cons=$(cat /custom-installation/autocons.info) + fi + if [ ! -z "$cons" ]; then + sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 console='${cons#/dev/}'"/' /target/etc/default/grub + updategrub=1 + fi +fi +kargs=$(curl https://$confluent_mgr/confluent-public/os/$confluent_profile/profile.yaml | grep ^installedargs: | sed -e 's/#.*//') +if [ ! -z "$kargs" ]; then + sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /target/etc/default/grub +fi +mkdir -p /opt/confluent/bin +mkdir -p /etc/confluent +cp -a /target/etc/confluent/* /etc/confluent +mkdir -p /target/opt/confluent/bin +cp /custom-installation/confluent/bin/apiclient /opt/confluent/bin/ +cp /custom-installation/confluent/bin/apiclient /target/opt/confluent/bin + +mount -o bind /dev /target/dev +mount -o bind /proc /target/proc +mount -o bind /sys /target/sys +if [ 1 = $updategrub ]; then + chroot /target update-grub +fi +echo "Port 22" >> /etc/ssh/sshd_config +echo "Port 2222" >> /etc/ssh/sshd_config +echo "Match LocalPort 22" >> /etc/ssh/sshd_config +echo " ChrootDirectory /target" >> /etc/ssh/sshd_config +kill -HUP $(cat /run/sshd.pid) +if [ -e /sys/firmware/efi ]; then + bootnum=$(chroot /target efibootmgr | grep ubuntu | sed -e 's/ .*//' -e 's/\*//' -e s/Boot//) + if [ ! -z "$bootnum" ]; then + currboot=$(chroot /target efibootmgr | grep ^BootOrder: | awk '{print $2}') + nextboot=$(echo $currboot| awk -F, '{print $1}') + [ "$nextboot" = "$bootnum" ] || chroot /target efibootmgr -o $bootnum,$currboot + chroot /target efibootmgr -D + fi +fi +cat /target/etc/confluent/tls/*.pem > /target/etc/confluent/ca.pem +cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem +chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient" +chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d" +source /target/etc/confluent/functions + +run_remote_config post + +umount /target/sys /target/dev /target/proc + diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh new file mode 100755 index 00000000..ddfe598b --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -0,0 +1,29 @@ +#!/bin/bash +deploycfg=/custom-installation/confluent/confluent.deploycfg + +cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //') +if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then + echo "****Encrypted boot requested, but not implemented for this OS, halting install" > /dev/console + [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but not implemented for this OS,halting install" >> $(cat /tmp/autoconsdev)) + while :; do sleep 86400; done +fi + + +cat /custom-installation/ssh/*pubkey > /root/.ssh/authorized_keys +nodename=$(grep ^NODENAME: /custom-installation/confluent/confluent.info|awk '{print $2}') +apikey=$(cat /custom-installation/confluent/confluent.apikey) +for pubkey in /etc/ssh/ssh_host*key.pub; do + certfile=${pubkey/.pub/-cert.pub} + keyfile=${pubkey%.pub} + curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" -d @$pubkey https://$confluent_mgr/confluent-api/self/sshcert > $certfile + echo HostKey $keyfile >> /etc/ssh/sshd_config.d/confluent.conf + echo HostCertificate $certfile >> /etc/ssh/sshd_config.d/confluent.conf +done +echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/confluent.conf +echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.conf +echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf +systemctl restart sshd +curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$confluent_mgr/confluent-api/self/nodelist > /tmp/allnodes +curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /custom-installation/getinstalldisk +python3 /custom-installation/getinstalldisk +sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient new file mode 100644 index 00000000..3fdd1f08 --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient @@ -0,0 +1,272 @@ +#!/usr/bin/python +import importlib +import tempfile +import json +import os +import shutil +import pwd +import grp +from importlib.machinery import SourceFileLoader +try: + apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() +except FileNotFoundError: + apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module() + + +def partitionhostsline(line): + comment = '' + try: + cmdidx = line.index('#') + comment = line[cmdidx:] + line = line[:cmdidx].strip() + except ValueError: + pass + if not line: + return '', [], comment + ipaddr, names = line.split(maxsplit=1) + names = names.split() + return ipaddr, names, comment + +class HostMerger(object): + def __init__(self): + self.byip = {} + self.byname = {} + self.sourcelines = [] + self.targlines = [] + + def read_source(self, sourcefile): + with open(sourcefile, 'r') as hfile: + self.sourcelines = hfile.read().split('\n') + while not self.sourcelines[-1]: + self.sourcelines = self.sourcelines[:-1] + for x in range(len(self.sourcelines)): + line = self.sourcelines[x] + currip, names, comment = partitionhostsline(line) + if currip: + self.byip[currip] = x + for name in names: + self.byname[name] = x + + def read_target(self, targetfile): + with open(targetfile, 'r') as hfile: + lines = hfile.read().split('\n') + if not lines[-1]: + lines = lines[:-1] + for y in range(len(lines)): + line = lines[y] + currip, names, comment = partitionhostsline(line) + if currip in self.byip: + x = self.byip[currip] + if self.sourcelines[x] is None: + # have already consumed this enntry + continue + self.targlines.append(self.sourcelines[x]) + self.sourcelines[x] = None + continue + for name in names: + if name in self.byname: + x = self.byname[name] + if self.sourcelines[x] is None: + break + self.targlines.append(self.sourcelines[x]) + self.sourcelines[x] = None + break + else: + self.targlines.append(line) + + def write_out(self, targetfile): + while not self.targlines[-1]: + self.targlines = self.targlines[:-1] + if not self.targlines: + break + while not self.sourcelines[-1]: + self.sourcelines = self.sourcelines[:-1] + if not self.sourcelines: + break + with open(targetfile, 'w') as hosts: + for line in self.targlines: + hosts.write(line + '\n') + for line in self.sourcelines: + if line is not None: + hosts.write(line + '\n') + + +class CredMerger: + def __init__(self): + try: + with open('/etc/login.defs', 'r') as ldefs: + defs = ldefs.read().split('\n') + except FileNotFoundError: + defs = [] + lkup = {} + self.discardnames = {} + self.shadowednames = {} + for line in defs: + try: + line = line[:line.index('#')] + except ValueError: + pass + keyval = line.split() + if len(keyval) < 2: + continue + lkup[keyval[0]] = keyval[1] + self.uidmin = int(lkup.get('UID_MIN', 1000)) + self.uidmax = int(lkup.get('UID_MAX', 60000)) + self.gidmin = int(lkup.get('GID_MIN', 1000)) + self.gidmax = int(lkup.get('GID_MAX', 60000)) + self.shadowlines = None + + def read_passwd(self, source, targfile=False): + self.read_generic(source, self.uidmin, self.uidmax, targfile) + + def read_group(self, source, targfile=False): + self.read_generic(source, self.gidmin, self.gidmax, targfile) + + def read_generic(self, source, minid, maxid, targfile): + if targfile: + self.targdata = [] + else: + self.sourcedata = [] + with open(source, 'r') as inputfile: + for line in inputfile.read().split('\n'): + try: + name, _, uid, _ = line.split(':', 3) + uid = int(uid) + except ValueError: + continue + if targfile: + if uid < minid or uid > maxid: + self.targdata.append(line) + else: + self.discardnames[name] = 1 + else: + if name[0] in ('+', '#', '@'): + self.sourcedata.append(line) + elif uid >= minid and uid <= maxid: + self.sourcedata.append(line) + + def read_shadow(self, source): + self.shadowlines = [] + try: + with open(source, 'r') as inshadow: + for line in inshadow.read().split('\n'): + try: + name, _ = line.split(':' , 1) + except ValueError: + continue + if name in self.discardnames: + continue + self.shadowednames[name] = 1 + self.shadowlines.append(line) + except FileNotFoundError: + return + + def write_out(self, outfile): + with open(outfile, 'w') as targ: + for line in self.targdata: + targ.write(line + '\n') + for line in self.sourcedata: + targ.write(line + '\n') + if outfile == '/etc/passwd': + if self.shadowlines is None: + self.read_shadow('/etc/shadow') + with open('/etc/shadow', 'w') as shadout: + for line in self.shadowlines: + shadout.write(line + '\n') + for line in self.sourcedata: + name, _ = line.split(':', 1) + if name[0] in ('+', '#', '@'): + continue + if name in self.shadowednames: + continue + shadout.write(name + ':!:::::::\n') + if outfile == '/etc/group': + if self.shadowlines is None: + self.read_shadow('/etc/gshadow') + with open('/etc/gshadow', 'w') as shadout: + for line in self.shadowlines: + shadout.write(line + '\n') + for line in self.sourcedata: + name, _ = line.split(':' , 1) + if name in self.shadowednames: + continue + shadout.write(name + ':!::\n') + +def appendonce(basepath, filename): + with open(filename, 'rb') as filehdl: + thedata = filehdl.read() + targname = filename.replace(basepath, '') + try: + with open(targname, 'rb') as filehdl: + targdata = filehdl.read() + except IOError: + targdata = b'' + if thedata in targdata: + return + with open(targname, 'ab') as targhdl: + targhdl.write(thedata) + +def synchronize(): + tmpdir = tempfile.mkdtemp() + appendoncedir = tempfile.mkdtemp() + try: + ac = apiclient.HTTPSClient() + data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir}) + status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status == 202: + lastrsp = '' + while status != 204: + status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') + if not isinstance(rsp, str): + rsp = rsp.decode('utf8') + if status == 200: + lastrsp = rsp + pendpasswd = os.path.join(tmpdir, 'etc/passwd') + if os.path.exists(pendpasswd): + cm = CredMerger() + cm.read_passwd(pendpasswd, targfile=False) + cm.read_passwd('/etc/passwd', targfile=True) + cm.write_out('/etc/passwd') + pendgroup = os.path.join(tmpdir, 'etc/group') + if os.path.exists(pendgroup): + cm = CredMerger() + cm.read_group(pendgroup, targfile=False) + cm.read_group('/etc/group', targfile=True) + cm.write_out('/etc/group') + pendhosts = os.path.join(tmpdir, 'etc/hosts') + if os.path.exists(pendhosts): + cm = HostMerger() + cm.read_source(pendhosts) + cm.read_target('/etc/hosts') + cm.write_out('/etc/hosts') + for dirn in os.walk(appendoncedir): + for filen in dirn[2]: + appendonce(appendoncedir, os.path.join(dirn[0], filen)) + if lastrsp: + lastrsp = json.loads(lastrsp) + opts = lastrsp.get('options', {}) + for fname in opts: + uid = -1 + gid = -1 + for opt in opts[fname]: + if opt == 'owner': + try: + uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid + except KeyError: + uid = opts[fname][opt]['id'] + elif opt == 'group': + try: + gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid + except KeyError: + gid = opts[fname][opt]['id'] + elif opt == 'permissions': + os.chmod(fname, int(opts[fname][opt], 8)) + if uid != -1 or gid != -1: + os.chown(fname, uid, gid) + finally: + shutil.rmtree(tmpdir) + shutil.rmtree(appendoncedir) + + +if __name__ == '__main__': + synchronize() From e2e4014db3cbfa835e0ca771fc320151479c5021 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 May 2022 11:01:59 -0400 Subject: [PATCH 26/34] Make ubuntu case insensitive on import Ubuntu changes their minds about case of some files. --- confluent_server/confluent/osimage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index 59e8fc9b..b42084dc 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -197,7 +197,7 @@ def extract_entries(entries, flags=0, callback=None, totalsize=None, extractlist for entry in entries: if str(entry).endswith('TRANS.TBL'): continue - if extractlist and str(entry) not in extractlist: + if extractlist and str(entry).lower() not in extractlist: continue write_header(write_p, entry._entry_p) read_p = entry._archive_p @@ -369,7 +369,7 @@ def check_ubuntu(isoinfo): return {'name': 'ubuntu-{0}-{1}'.format(ver, arch), 'method': EXTRACT|COPY, 'extractlist': ['casper/vmlinuz', 'casper/initrd', - 'EFI/BOOT/BOOTx64.EFI', 'EFI/BOOT/grubx64.efi' + 'efi/boot/bootx64.efi', 'efi/boot/grubx64.efi' ], 'copyto': 'install.iso', 'category': 'ubuntu{0}'.format(major)} From bf37c05d71847ef0751409523ba3775f3f676ec7 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 May 2022 11:31:03 -0400 Subject: [PATCH 27/34] Add ubuntu 22.04 to packaging --- confluent_osdeploy/confluent_osdeploy.spec.tmpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/confluent_osdeploy/confluent_osdeploy.spec.tmpl b/confluent_osdeploy/confluent_osdeploy.spec.tmpl index 8cc0c95b..21f80b47 100644 --- a/confluent_osdeploy/confluent_osdeploy.spec.tmpl +++ b/confluent_osdeploy/confluent_osdeploy.spec.tmpl @@ -30,7 +30,7 @@ cp start_root urlmount ../stateless-bin/ cd .. ln -s el8 el9 ln -s el8-diskless el9-diskless -for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 coreos el9; do +for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 coreos el9; do mkdir ${os}out cd ${os}out if [ -d ../${os}bin ]; then @@ -44,7 +44,7 @@ for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 coreos el9; do mv ../addons.cpio . cd .. done -for os in el7 el8 suse15 el9 ubuntu20.04; do +for os in el7 el8 suse15 el9 ubuntu20.04 ubuntu22.04; do mkdir ${os}disklessout cd ${os}disklessout if [ -d ../${os}bin ]; then @@ -78,7 +78,7 @@ cp -a esxi7 esxi6 %install mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/ cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/ -for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 esxi6 esxi7 coreos; do +for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu22.04 esxi6 esxi7 coreos; do mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs From 62b27b1fcc4e71dc5a2f056ae576b50ca3be85e2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 May 2022 11:42:14 -0400 Subject: [PATCH 28/34] Back off diskless profile for ubuntu 22.04, not yet done --- confluent_osdeploy/confluent_osdeploy.spec.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/confluent_osdeploy.spec.tmpl b/confluent_osdeploy/confluent_osdeploy.spec.tmpl index 21f80b47..983792ba 100644 --- a/confluent_osdeploy/confluent_osdeploy.spec.tmpl +++ b/confluent_osdeploy/confluent_osdeploy.spec.tmpl @@ -44,7 +44,7 @@ for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 coreos el9; do mv ../addons.cpio . cd .. done -for os in el7 el8 suse15 el9 ubuntu20.04 ubuntu22.04; do +for os in el7 el8 suse15 el9 ubuntu20.04; do mkdir ${os}disklessout cd ${os}disklessout if [ -d ../${os}bin ]; then From 548111f4d18c75b389fdb9e2cac846d926e7cf96 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 May 2022 12:11:24 -0400 Subject: [PATCH 29/34] Fix executable status of the confluent casper hook --- .../ubuntu22.04/initramfs/scripts/casper-bottom/99confluent | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent b/confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent old mode 100644 new mode 100755 From d3129847b46a4f74a7eaa3e53c56deba830abd39 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 May 2022 14:16:51 -0400 Subject: [PATCH 30/34] Add recognition of Alma 9 and (presumably) rocky 9 --- confluent_server/confluent/osimage.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index b42084dc..f79e4593 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -250,6 +250,11 @@ def check_rocky(isoinfo): arch = entry.split('.')[-2] cat = 'el8' break + if 'rocky-release-9' in entry: + ver = entry.split('-')[2] + arch = entry.split('.')[-2] + cat = 'el9' + break else: return None if arch == 'noarch' and '.discinfo' in isoinfo[1]: @@ -270,6 +275,11 @@ def check_alma(isoinfo): arch = entry.split('.')[-2] cat = 'el8' break + elif 'almalinux-release-9' in entry: + ver = entry.split('-')[2] + arch = entry.split('.')[-2] + cat = 'el9' + break else: return None if arch == 'noarch' and '.discinfo' in isoinfo[1]: From ffadb9cc3a08087d80ad421d4d1d42781a2ecbdf Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 May 2022 14:23:41 -0400 Subject: [PATCH 31/34] Add squashfs dependency It has been confusing when imgutil cannot pack by default --- imgutil/confluent_imgutil.spec.tmpl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/imgutil/confluent_imgutil.spec.tmpl b/imgutil/confluent_imgutil.spec.tmpl index a5c93f42..76862999 100644 --- a/imgutil/confluent_imgutil.spec.tmpl +++ b/imgutil/confluent_imgutil.spec.tmpl @@ -7,6 +7,16 @@ URL: https://hpc.lenovo.com/ Source: confluent_imgutil.tar.xz BuildArch: noarch BuildRoot: /tmp/ +%if "%{dist}" == ".el8" +Requires squashfs-tools +%else +%if "%{dist}" == ".el9" +Requires squashfs-tools +%else +Requires squashfs +%endif +%endif + %description Utility for generating, modifying, and packing confluent deployment From 754cdfdd78f438b9e22b659f8395d410d2b5b0ee Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 May 2022 14:35:03 -0400 Subject: [PATCH 32/34] Fix rpm spec syntax for imgutil --- imgutil/confluent_imgutil.spec.tmpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/imgutil/confluent_imgutil.spec.tmpl b/imgutil/confluent_imgutil.spec.tmpl index 76862999..268b63f6 100644 --- a/imgutil/confluent_imgutil.spec.tmpl +++ b/imgutil/confluent_imgutil.spec.tmpl @@ -8,12 +8,12 @@ Source: confluent_imgutil.tar.xz BuildArch: noarch BuildRoot: /tmp/ %if "%{dist}" == ".el8" -Requires squashfs-tools +Requires: squashfs-tools %else %if "%{dist}" == ".el9" -Requires squashfs-tools +Requires: squashfs-tools %else -Requires squashfs +Requires: squashfs %endif %endif From 86891eb2e5c6571a95297a0c6660ba4248b579d9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 5 May 2022 09:26:55 -0400 Subject: [PATCH 33/34] Rework resolv watcher Handle symlinks better and do not trigger overly eagerly --- confluent_server/confluent/sockapi.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/sockapi.py b/confluent_server/confluent/sockapi.py index 53f76ed2..341f1973 100644 --- a/confluent_server/confluent/sockapi.py +++ b/confluent_server/confluent/sockapi.py @@ -502,8 +502,17 @@ class SockApi(object): def watch_resolv(self): while True: watcher = libc.inotify_init1(os.O_NONBLOCK) - if libc.inotify_add_watch(watcher, b'/etc/resolv.conf', 0xcda) <= -1: - break + resolvpath = '/etc/resolv.conf' + while True: + try: + resolvpath = os.readlink(resolvpath) + except Exception: + break + if not isinstance(resolvpath, bytes): + resolvpath = resolvpath.encode('utf8') + if libc.inotify_add_watch(watcher, resolvpath, 0xcc2) <= -1: + eventlet.sleep(15) + continue select.select((watcher,), (), (), 86400) try: os.read(watcher, 1024) From 2925c291cf1be12ce76345ae13ae4cca567e0f18 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 5 May 2022 09:28:09 -0400 Subject: [PATCH 34/34] Increase ipmi concurrency 128 can be a bit limiting, try 512 for improved responsiveness --- confluent_server/confluent/plugins/hardwaremanagement/ipmi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py index 22cbec0a..b2e5615c 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py @@ -110,7 +110,7 @@ def exithandler(): atexit.register(exithandler) -_ipmiworkers = greenpool.GreenPool(128) +_ipmiworkers = greenpool.GreenPool(512) _ipmithread = None _ipmiwaiters = []