From 41ecfeccf4733bf0309dceada8a93369cf03ea0c Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 15 Oct 2026 17:27:39 +0200 Subject: [PATCH 1/6] Drop pasted shell session from the iPXE spec The spec carried a copy of a terminal session after %changelog: a shell prompt followed by a second full copy of the spec. Also fix the debug_package macro name typo. --- ipxe/confluent_ipxe.spec | 44 +--------------------------------------- 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/ipxe/confluent_ipxe.spec b/ipxe/confluent_ipxe.spec index b80bef4c..01ded551 100644 --- a/ipxe/confluent_ipxe.spec +++ b/ipxe/confluent_ipxe.spec @@ -12,7 +12,7 @@ BuildArch: noarch %description iPXE boot roms for chainloading systems -%define debug_pagkage %{nil} +%define debug_package %{nil} %prep %setup -n ipxeboot @@ -38,45 +38,3 @@ cp arm64-sb/snponly.efi %{buildroot}/opt/confluent/lib/ipxe/ipxe-aarch64.efi %changelog * Tue Sep 22 2026 Confluent - 2.0.0-1 - Initial -[jjohnson2@fourge ~]$ cat ~/rpmbuild/SPECS/confluent_ipxe.spec -Name: confluent_ipxe -Version: 2.0.0 -Release: 1 -Summary: iPXE network boot roms for confluent - -License: GPL -Source: https://github.com/ipxe/ipxe/releases/latest/download/ipxeboot.tar.gz -Obsoletes: confluent_ipxe-aarch64 < 2.0.0 -Provides: confluent_ipxe-aarch64 = %{version}-%{release} -BuildArch: noarch - -%description -iPXE boot roms for chainloading systems - -%define debug_pagkage %{nil} -%prep -%setup -n ipxeboot - - -%install -mkdir -p %{buildroot}/opt/confluent/lib/ipxe -cp x86_64/undionly.kpxe %{buildroot}/opt/confluent/lib/ipxe/ipxe.kkpxe -cp x86_64-sb/shimx64.efi %{buildroot}/opt/confluent/lib/ipxe/ipxe-shim.efi -cp x86_64-sb/snponly.efi %{buildroot}/opt/confluent/lib/ipxe/ipxe.efi -cp arm64-sb/shimaa64.efi %{buildroot}/opt/confluent/lib/ipxe/ipxe-aarch64-shim.efi -cp arm64-sb/snponly.efi %{buildroot}/opt/confluent/lib/ipxe/ipxe-aarch64.efi - -%files -/opt/confluent/lib/ipxe/ipxe.kkpxe -/opt/confluent/lib/ipxe/ipxe.efi -/opt/confluent/lib/ipxe/ipxe-shim.efi -/opt/confluent/lib/ipxe/ipxe-aarch64-shim.efi -/opt/confluent/lib/ipxe/ipxe-aarch64.efi - - - - -%changelog -* Tue Sep 22 2026 Confluent - 2.0.0-1 -- Initial - From b86e0aa27f89f5165cd70dc85b42845e1b9d8913 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 15 Oct 2026 17:46:17 +0200 Subject: [PATCH 2/6] Make webauthn and eficompressor optional server dependencies Neither is packaged by EPEL or Ubuntu, so the hard dependency made the server uninstallable there without --nodeps or a stub package. Both are already optional in the code: httpapi drops passkey support when webauthn does not import, and the Lenovo IMM2 BIOS settings path reports the missing module as unsupported functionality. Recommend them instead, as the SUSE branch already does for webauthn. See xcat2/confluent#192. --- confluent_server/builddeb | 6 ++++-- confluent_server/confluent_server.spec.tmpl | 9 +++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/confluent_server/builddeb b/confluent_server/builddeb index 69d7ee01..a4b2cada 100755 --- a/confluent_server/builddeb +++ b/confluent_server/builddeb @@ -35,10 +35,12 @@ shopt -s extglob cd deb_dist/!(*.orig)/ if [ "$OPKGNAME" = "confluent-server" ]; then if grep noble /etc/os-release; then - sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-cryptography, python3-websocket, python3-msgpack, python3-aiohttp, python3-pyparsing, python3-asyncssh, python3-pysnmp-lextudio, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil, python3-numpy, python3-pillow/' debian/control + sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-cryptography, python3-websocket, python3-msgpack, python3-aiohttp, python3-pyparsing, python3-asyncssh, python3-pysnmp-lextudio, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil, python3-numpy, python3-pillow/' debian/control else - sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-cryptography, python3-websocket, python3-msgpack, python3-aiohttp, python3-pyparsing, python3-asyncssh, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil, python3-crypt-r, python3-numpy, python3-pillow/' debian/control + sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-cryptography, python3-websocket, python3-msgpack, python3-aiohttp, python3-pyparsing, python3-asyncssh, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil, python3-crypt-r, python3-numpy, python3-pillow/' debian/control fi + # only BIOS settings on IMM2-era Lenovo BMCs need it, and Ubuntu does not package it + sed -i '/^Depends:/a Recommends: python3-eficompressor' debian/control echo 'confluent_client confluent-client' >> debian/pydist-overrides fi diff --git a/confluent_server/confluent_server.spec.tmpl b/confluent_server/confluent_server.spec.tmpl index dc87e8e8..056be11e 100644 --- a/confluent_server/confluent_server.spec.tmpl +++ b/confluent_server/confluent_server.spec.tmpl @@ -24,10 +24,15 @@ Requires: confluent_vtbufferd Recommends: python3-legacycrypt %endif %if "%{dist}" == ".el9" -Requires: python3-asyncssh, python3-cryptography, confluent_client == %{version}, python3-pyparsing, python3-webauthn, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL python3-msgpack python3-libarchive-c python3-yaml python3-yarl python3-aiohttp openssl iproute +Requires: python3-asyncssh, python3-cryptography, confluent_client == %{version}, python3-pyparsing, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL python3-msgpack python3-libarchive-c python3-yaml python3-yarl python3-aiohttp openssl iproute +# httpapi drops passkey support without webauthn, and only BIOS settings on +# IMM2-era Lenovo BMCs need eficompressor. Neither is in EPEL. +Recommends: python3-webauthn python3-eficompressor %else %if "%{dist}" == ".el10" -Requires: python3-asyncssh, python3-cryptography, confluent_client == %{version}, python3-pyparsing, python3-webauthn python3-psutil, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml,python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL python3-msgpack python3-libarchive-c python3-yaml openssl iproute python3-yarl python3-aiohttp +Requires: python3-asyncssh, python3-cryptography, confluent_client == %{version}, python3-pyparsing, python3-psutil, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml,python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL python3-msgpack python3-libarchive-c python3-yaml openssl iproute python3-yarl python3-aiohttp +# httpapi drops passkey support without webauthn, which is not in EPEL +Recommends: python3-webauthn %else %if 0%{?suse_version} # python3-dbm is split out there, PyYAML keeps its upstream capitalisation From b1ed925e408fc2d4fba4472bd677885bab3c53e5 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 15 Oct 2026 18:03:42 +0200 Subject: [PATCH 3/6] Fall back to libcrypt when no crypt module imports Python 3.13 dropped crypt from the stdlib, and openSUSE Leap 16 packages neither legacycrypt nor crypt_r, so the server did not start there without a pip install. Both of those only call crypt(3) from libcrypt, which every distribution ships, so do that through ctypes as the last resort, serialized because crypt(3) returns a static buffer. Its hashes match the stdlib module byte for byte. Move the import chain both callers carried into confluent.pwcrypt. The legacycrypt Recommends is no longer needed. --- .../confluent/config/configmanager.py | 10 +-- confluent_server/confluent/pwcrypt.py | 66 +++++++++++++++++++ confluent_server/confluent/selfservice.py | 10 +-- confluent_server/confluent_server.spec.tmpl | 6 -- 4 files changed, 68 insertions(+), 24 deletions(-) create mode 100644 confluent_server/confluent/pwcrypt.py diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 71ff5cef..1957c303 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -69,15 +69,7 @@ import confluent.netutil as netutil import confluent.exceptions as exc import confluent.log import copy -try: - import crypt -except ImportError: - # crypt left the stdlib in 3.13. Both of these reach the same libcrypt - # call and give the same hash; distros package one, the other, or neither. - try: - import legacycrypt as crypt - except ImportError: - import crypt_r as crypt +import confluent.pwcrypt as crypt try: import cPickle except ModuleNotFoundError: diff --git a/confluent_server/confluent/pwcrypt.py b/confluent_server/confluent/pwcrypt.py new file mode 100644 index 00000000..18560c9d --- /dev/null +++ b/confluent_server/confluent/pwcrypt.py @@ -0,0 +1,66 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2026 MEGWARE Computer Vertrieb und Service GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# crypt(3) for hashing and checking the crypted.* attributes. crypt left the +# stdlib in 3.13, so prefer it, then the packaged replacements (legacycrypt, +# crypt_r), and finally call libcrypt directly, which is all the others do. +# Every Linux distribution ships libcrypt, while some package none of the +# Python modules (openSUSE Leap 16). + +try: + from crypt import crypt +except ImportError: + try: + from legacycrypt import crypt + except ImportError: + try: + from crypt_r import crypt + except ImportError: + crypt = None + +if crypt is None: + import ctypes + import ctypes.util + import threading + + def _load_libcrypt(): + # libxcrypt is libcrypt.so.2 where it is built without the glibc + # compatible API, libcrypt.so.1 otherwise + names = [ctypes.util.find_library('crypt'), 'libcrypt.so.2', + 'libcrypt.so.1'] + for name in names: + if not name: + continue + try: + lib = ctypes.CDLL(name, use_errno=True) + except OSError: + continue + lib.crypt.argtypes = (ctypes.c_char_p, ctypes.c_char_p) + lib.crypt.restype = ctypes.c_char_p + return lib + raise ImportError('no crypt module and no libcrypt found') + + _libcrypt = _load_libcrypt() + # crypt(3) returns a pointer to a static buffer + _lock = threading.Lock() + + def crypt(word, salt): + with _lock: + ret = _libcrypt.crypt(word.encode('utf8'), salt.encode('utf8')) + if ret is None: + errno = ctypes.get_errno() + raise OSError(errno, 'crypt failed') + return ret.decode('utf8') diff --git a/confluent_server/confluent/selfservice.py b/confluent_server/confluent/selfservice.py index e65d91a4..bd57b29b 100644 --- a/confluent_server/confluent/selfservice.py +++ b/confluent_server/confluent/selfservice.py @@ -15,15 +15,7 @@ import confluent.discovery.core as disco import base64 import hmac import hashlib -try: - import crypt -except ImportError: - # crypt left the stdlib in 3.13. Both of these reach the same libcrypt - # call and give the same hash; distros package one, the other, or neither. - try: - import legacycrypt as crypt - except ImportError: - import crypt_r as crypt +import confluent.pwcrypt as crypt import json import os import tempfile diff --git a/confluent_server/confluent_server.spec.tmpl b/confluent_server/confluent_server.spec.tmpl index 056be11e..42c723fb 100644 --- a/confluent_server/confluent_server.spec.tmpl +++ b/confluent_server/confluent_server.spec.tmpl @@ -17,12 +17,6 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot Prefix: %{_prefix} BuildArch: noarch Requires: confluent_vtbufferd -%if 0%{?python3_version_nodots} >= 313 -# 3.13 removed crypt from the stdlib, and stored credentials are hashed with -# it. Recommends rather than Requires: some 3.13 distros package no candidate -# at all, where a hard dependency would make the rpm uninstallable. -Recommends: python3-legacycrypt -%endif %if "%{dist}" == ".el9" Requires: python3-asyncssh, python3-cryptography, confluent_client == %{version}, python3-pyparsing, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL python3-msgpack python3-libarchive-c python3-yaml python3-yarl python3-aiohttp openssl iproute # httpapi drops passkey support without webauthn, and only BIOS settings on From f25d121fc50ccd823927ab2527f197ff81b27325 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 15 Oct 2026 18:04:50 +0200 Subject: [PATCH 4/6] Version the Python package rpms with ~dev as well confluent_client and confluent_server rpms took the PEP 440 version setup.py needs, 4.0.1.dev5+g..., which rpm sorts after 4.0.1: a released package did not upgrade a development build of it. Stamp the rpm with the ~dev form the other packages already use and keep .dev only for setup.py and the sdist name. The debs already sort right, since stdeb rewrites .dev to ~dev. --- README.md | 4 ++-- confluent_client/confluent_client.spec.tmpl | 3 ++- confluent_common/confluent_common.spec.tmpl | 9 +++++++-- confluent_server/buildrpm | 5 ++++- confluent_server/confluent_server.spec.tmpl | 3 ++- mkversion | 5 +++-- 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c08d94c7..27d110b0 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,8 @@ Want to help? Submit a [Pull Request](https://github.com/xcat2/confluent/pulls). The top-level `VERSION` file names the release the current branch is working toward. Build scripts call `./mkversion`, which turns it into the version stamped on packages: the tag itself on a release -tag (`4.0.1`), otherwise a development version (`4.0.1.dev5+gdeadbee`, or `4.0.1~dev5+gdeadbee` for -the packages that have no `setup.py`). +tag (`4.0.1`), otherwise a development version (`4.0.1~dev5+gdeadbee`). Packages sort it before the +release it leads up to. Only `setup.py` gets the PEP 440 spelling, `4.0.1.dev5+gdeadbee`. `mkversion` also derives a version from the newest tag reachable from HEAD and uses whichever is higher, so forgetting to bump `VERSION` after tagging the current branch cannot walk the version diff --git a/confluent_client/confluent_client.spec.tmpl b/confluent_client/confluent_client.spec.tmpl index 9de73bd7..82e19b34 100644 --- a/confluent_client/confluent_client.spec.tmpl +++ b/confluent_client/confluent_client.spec.tmpl @@ -1,7 +1,8 @@ %define name confluent_client %define version #VERSION# +# the sdist carries the PEP 440 spelling of the version (.dev, not ~dev) %define fversion %{lua: -sv, _ = string.gsub("#VERSION#", "[~]", "-") +sv, _ = string.gsub("#VERSION#", "[~]", ".") print(sv) } %define release 1 diff --git a/confluent_common/confluent_common.spec.tmpl b/confluent_common/confluent_common.spec.tmpl index 94a1ea73..683f0cc3 100644 --- a/confluent_common/confluent_common.spec.tmpl +++ b/confluent_common/confluent_common.spec.tmpl @@ -1,12 +1,17 @@ %define name confluent_common %define version #VERSION# +# the sdist carries the PEP 440 spelling of the version (.dev, not ~dev) +%define fversion %{lua: +sv, _ = string.gsub("#VERSION#", "[~]", ".") +print(sv) +} %define release 1 Summary: common content for confluent client and server Name: %{name} Version: %{version} Release: %{release} -Source0: %{name}-%{version}.tar.gz +Source0: %{name}-%{fversion}.tar.gz License: UNKNOWN Group: Development/Libraries BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot @@ -19,7 +24,7 @@ Url: http://xcat.sf.net/ This provides the modules common for both client and server %prep -%setup -n %{name}-%{version} -n %{name}-%{version} +%setup -n %{name}-%{fversion} %build python2 setup.py build diff --git a/confluent_server/buildrpm b/confluent_server/buildrpm index 4554e502..0f7e48ab 100755 --- a/confluent_server/buildrpm +++ b/confluent_server/buildrpm @@ -3,7 +3,10 @@ cd `dirname $0` if [ -x ./makeman ]; then ./makeman fi -VERSION=`./makesetup` || exit 1 +./makesetup > /dev/null || exit 1 +# setup.py needs the PEP 440 .dev form, the rpm the ~dev one, which sorts +# before the release it leads up to +VERSION=`../mkversion --tilde` || exit 1 PKGNAME=$(basename $(pwd)) python3 setup.py sdist > /dev/null 2>&1 cp dist/*.tar.gz ~/rpmbuild/SOURCES diff --git a/confluent_server/confluent_server.spec.tmpl b/confluent_server/confluent_server.spec.tmpl index 42c723fb..ddfb9443 100644 --- a/confluent_server/confluent_server.spec.tmpl +++ b/confluent_server/confluent_server.spec.tmpl @@ -1,7 +1,8 @@ %define name confluent_server %define version #VERSION# +# the sdist carries the PEP 440 spelling of the version (.dev, not ~dev) %define fversion %{lua: -sv, _ = string.gsub("#VERSION#", "[~]", "-") +sv, _ = string.gsub("#VERSION#", "[~]", ".") print(sv) } %define release 1 diff --git a/mkversion b/mkversion index d6f85b63..fec5c300 100755 --- a/mkversion +++ b/mkversion @@ -1,8 +1,9 @@ #!/bin/sh # Print the version to stamp on packages built from this checkout. # -# --tilde use "~dev" rather than ".dev" (packages with no setup.py; "~" sorts -# before the release, ".dev" after, but PEP 440 requires ".dev") +# --tilde use "~dev" rather than ".dev": the package version, as "~" sorts +# before the release and ".dev" after it. Only setup.py takes ".dev", +# which PEP 440 requires. # # The root VERSION file names the release this branch is working toward. Release # tags do not always live on master (3.15.2 through 3.15.6 are on branch 3.15), so From 6c48cb4fad9db533132fd94e98c41b9a935869bd Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 15 Oct 2026 17:51:15 +0200 Subject: [PATCH 5/6] Build and publish nightly packages Build the rpms for EL9, EL10 and SUSE 16 and the debs for Ubuntu noble and resolute, each for x86_64 and aarch64, install every set on a clean container and publish them as a rolling nightly pre-release. Each release asset is one target's bundle: all of its packages, repository metadata and an INSTALL.txt. Besides the in-tree components this packages confluent_ipxe from the iPXE release roms and the browserfs and vinz helpers from their own repositories. The osdeploy utils are built where buildrpm builds them, el7 for the x86_64 el8bin set, so they still run on the oldest targets. Genesis is left out for now. Noarch packages are built once, in the x86_64 legs, and the aarch64 legs build only what is architecture specific. Build tools are installed without weak dependencies, while the smoke tests keep the distribution defaults, since they stand in for a user's install. Scheduled runs skip the build when neither confluent nor the helper repositories moved since the last published nightly. Pull requests that touch the packaging build and test without publishing. --- .github/scripts/build-debs.sh | 48 ++++ .github/scripts/build-go-rpm.sh | 35 +++ .github/scripts/build-ipxe.sh | 13 + .github/scripts/build-osdeploy.sh | 40 +++ .github/scripts/build-rpms.sh | 38 +++ .github/scripts/build-utils.sh | 38 +++ .github/scripts/mkbundle.sh | 91 ++++++ .github/scripts/smoketest.sh | 78 +++++ .github/workflows/nightly.yml | 458 ++++++++++++++++++++++++++++++ .github/workflows/release.yml | 2 + README.md | 8 + 11 files changed, 849 insertions(+) create mode 100755 .github/scripts/build-debs.sh create mode 100755 .github/scripts/build-go-rpm.sh create mode 100755 .github/scripts/build-ipxe.sh create mode 100755 .github/scripts/build-osdeploy.sh create mode 100755 .github/scripts/build-rpms.sh create mode 100755 .github/scripts/build-utils.sh create mode 100755 .github/scripts/mkbundle.sh create mode 100755 .github/scripts/smoketest.sh create mode 100644 .github/workflows/nightly.yml diff --git a/.github/scripts/build-debs.sh b/.github/scripts/build-debs.sh new file mode 100755 index 00000000..efe8d1cf --- /dev/null +++ b/.github/scripts/build-debs.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Build the confluent debs in an Ubuntu container, like build/arm/*/buildapt.sh: +# every component with a builddeb script, plus alien conversions of the rpms +# that have no deb packaging of their own (osdeploy, ipxe, browserfs, vinz). +# With --arch-only, only the architecture-specific ones: confluent_vtbufferd +# and the conversions of rpms that are not noarch. +# +# Usage: build-debs.sh [--arch-only] [] +set -e + +ARCHONLY= +if [ "$1" = "--arch-only" ]; then + ARCHONLY=1 + shift +fi +OUT=$(readlink -f "$1") +RPMS=${2:+$(readlink -f "$2")} +[ -n "$OUT" ] || { echo "usage: $0 [--arch-only] [rpmdir]" >&2; exit 1; } + +cd "$(dirname "$0")/../.." +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get -y --no-install-recommends install git build-essential debhelper devscripts dh-python fakeroot \ + python3-all python3-setuptools python3-stdeb pandoc patch alien +# stdeb on noble still imports the SafeConfigParser python 3.12 removed +if (cd / && patch -p1 --dry-run -s < "$OLDPWD/build/arm/noble/stdeb.patch") > /dev/null 2>&1; then + (cd / && patch -p1 < "$OLDPWD/build/arm/noble/stdeb.patch") +fi +git config --global --add safe.directory '*' +./mkversion + +mkdir -p "$OUT" +# vtbufferd first: it recreates /tmp/confluent, which client and server reuse +./confluent_vtbufferd/builddeb "$OUT" +if [ -z "$ARCHONLY" ]; then + ./confluent_client/builddeb "$OUT" + ./confluent_server/builddeb "$OUT" + ./imgutil/builddeb "$OUT" +fi + +if [ -n "$RPMS" ]; then + skip=(! -name '*.src.rpm') + [ -n "$ARCHONLY" ] && skip+=(! -name '*.noarch.rpm') + cd "$OUT" + find "$RPMS" -name '*.rpm' "${skip[@]}" -print0 | + xargs -0 -r -n1 fakeroot alien --to-deb --scripts --keep-version +fi +ls -la "$OUT" diff --git a/.github/scripts/build-go-rpm.sh b/.github/scripts/build-go-rpm.sh new file mode 100755 index 00000000..bdf88627 --- /dev/null +++ b/.github/scripts/build-go-rpm.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Build the rpm of one of the Go helpers confluent runs but keeps in separate +# repositories (browserfs, vinz), for the native architecture only. +# +# Mirrors the buildrpm script in those repositories, which needs git tags for +# its version. They carry none yet, so fall back to 0~dev+g, +# which sorts below any tagged release. +# +# Usage: build-go-rpm.sh [ref] +set -e + +NAME=$1 +URL=$2 +REF=${3:-HEAD} +WORK=$(mktemp -d) + +git clone -q "$URL" "$WORK/$NAME" +cd "$WORK/$NAME" +git checkout -q "$REF" +if desc=$(git describe 2>/dev/null); then + VERSION=${desc%%-*} + NUMCOMMITS=$(echo "$desc" | cut -d- -f2) + if [ "$NUMCOMMITS" != "$VERSION" ]; then + VERSION=$(echo "$VERSION" | awk -F. -v OFS=. '{$NF=$NF+1; print}') + VERSION=$VERSION~dev$NUMCOMMITS+$(echo "$desc" | cut -d- -f3) + fi +else + VERSION=0~dev$(git rev-list --count HEAD)+g$(git rev-parse --short HEAD) +fi +echo "$NAME $VERSION ($(git rev-parse HEAD))" + +mkdir -p ~/rpmbuild/SOURCES +(cd .. && tar cf ~/rpmbuild/SOURCES/confluent-$NAME.tar --exclude=.git "$NAME") +sed -e "s/#VERSION#/$VERSION/" $NAME.spec.tmpl > $NAME.spec +rpmbuild -ba $NAME.spec diff --git a/.github/scripts/build-ipxe.sh b/.github/scripts/build-ipxe.sh new file mode 100755 index 00000000..0d795669 --- /dev/null +++ b/.github/scripts/build-ipxe.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Build the confluent_ipxe rpm. The spec repackages the prebuilt boot roms +# that iPXE publishes with each release; rpmbuild does not fetch remote +# sources itself, so download the tarball of the release the spec names. +set -e + +cd "$(dirname "$0")/../.." +SPEC=ipxe/confluent_ipxe.spec +VERSION=$(rpmspec -q --srpm --qf '%{version}' $SPEC) +mkdir -p ~/rpmbuild/SOURCES +curl -fsSL -o ~/rpmbuild/SOURCES/ipxeboot.tar.gz \ + "https://github.com/ipxe/ipxe/releases/download/v$VERSION/ipxeboot.tar.gz" +rpmbuild -ba $SPEC diff --git a/.github/scripts/build-osdeploy.sh b/.github/scripts/build-osdeploy.sh new file mode 100755 index 00000000..c71efb1d --- /dev/null +++ b/.github/scripts/build-osdeploy.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Build the confluent_osdeploy rpm the way confluent_osdeploy/buildrpm and +# buildrpm-aarch64 do, but from utils binaries built by earlier CI jobs +# (build-utils.sh) instead of compiling them in local podman containers. +# +# Usage: build-osdeploy.sh +set -e + +ARCH=$1 +EL8UTILS=$(readlink -f "$2") +EL9UTILS=$(readlink -f "$3") + +cd "$(dirname "$0")/../.." +VERSION=$(./mkversion --tilde) + +case "$ARCH" in +x86_64) SPEC=confluent_osdeploy.spec ;; +aarch64) SPEC=confluent_osdeploy-aarch64.spec ;; +*) echo "unknown arch $ARCH" >&2; exit 1 ;; +esac +sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy/$SPEC.tmpl > confluent_osdeploy/$SPEC + +mkdir -p ~/rpmbuild/SOURCES +cp LICENSE confluent_osdeploy/ +tar Jcf ~/rpmbuild/SOURCES/confluent_osdeploy.tar.xz confluent_osdeploy + +for gen in el8 el9; do + utils=$EL8UTILS + [ "$gen" = el9 ] && utils=$EL9UTILS + rm -rf ${gen}bin + mkdir -p ${gen}bin/opt/confluent/bin ${gen}bin/stateless-bin + (cd "$utils" && cp confluent_imginfo copernicus clortho autocons "$OLDPWD"/${gen}bin/opt/confluent/bin/) + (cd "$utils" && cp start_root urlmount "$OLDPWD"/${gen}bin/stateless-bin/) + # artifact downloads lose the execute bit + chmod 755 ${gen}bin/opt/confluent/bin/* ${gen}bin/stateless-bin/* + tar Jcf ~/rpmbuild/SOURCES/confluent_${gen}bin.tar.xz ${gen}bin/ + rm -rf ${gen}bin +done + +rpmbuild -ba confluent_osdeploy/$SPEC diff --git a/.github/scripts/build-rpms.sh b/.github/scripts/build-rpms.sh new file mode 100755 index 00000000..92981385 --- /dev/null +++ b/.github/scripts/build-rpms.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Build the confluent rpms whose spec depends on the target distribution: +# confluent_vtbufferd (arch-specific) and, unless --arch-only is given, the +# noarch client, server and imgutil. Runs inside an EL or openSUSE container. +# +# Usage: build-rpms.sh [--arch-only] +set -e + +cd "$(dirname "$0")/../.." + +if command -v zypper > /dev/null; then + zypper -n install --no-recommends rpm-build gcc make git-core tar xz pandoc systemd python3 python3-setuptools + # the buildrpm scripts stage into ~/rpmbuild, SUSE defaults to /usr/src/packages + echo "%_topdir $HOME/rpmbuild" >> ~/.rpmmacros +else + dnf -y --setopt=install_weak_deps=False install epel-release + dnf -y --setopt=install_weak_deps=False install rpm-build gcc make git-core tar xz pandoc systemd python3 python3-setuptools +fi +git config --global --add safe.directory "$PWD" +./mkversion +mkdir -p ~/rpmbuild/SOURCES ~/rpmbuild/SPECS + +# the buildrpm scripts hide rpmbuild's output, so on failure rerun the spec +# they wrote to show why +build() { + "$1/buildrpm" && return 0 + spec=~/rpmbuild/SPECS/$(basename "$1").spec + [ -f "$spec" ] && rpmbuild -ba "$spec" + return 1 +} + +build confluent_vtbufferd +if [ "$1" != "--arch-only" ]; then + build confluent_client + build confluent_server + # imgutil/buildrpm calls rpmbuild directly, so its output is visible + ./imgutil/buildrpm +fi diff --git a/.github/scripts/build-utils.sh b/.github/scripts/build-utils.sh new file mode 100755 index 00000000..4bbac8f8 --- /dev/null +++ b/.github/scripts/build-utils.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Build the confluent_osdeploy utils inside a distro container, standing in for +# the el7build/el8builder/el9build podman images confluent_osdeploy/buildrpm +# uses. The el8bin set is linked against the oldest glibc any target runs +# (el7 on x86_64, el8 on aarch64), so it must be built on that distro. +# +# Usage: build-utils.sh (run from the repository root) +set -e + +OUT=$1 +[ -n "$OUT" ] || { echo "usage: $0 " >&2; exit 1; } + +. /etc/os-release +case "$VERSION_ID" in +7*) + # CentOS 7 is EOL: its mirrors are gone, only the vault remains + sed -i -e 's/^mirrorlist=/#mirrorlist=/' \ + -e 's|^#\?baseurl=http://mirror.centos.org|baseurl=https://vault.centos.org|' \ + /etc/yum.repos.d/CentOS-*.repo + yum -y install gcc make binutils libcurl-devel fuse-devel + ;; +8*) + dnf -y --setopt=install_weak_deps=False install dnf-plugins-core + dnf config-manager --set-enabled powertools + dnf -y --setopt=install_weak_deps=False install gcc make binutils libcurl-devel fuse-devel libxcrypt-devel + ;; +*) + dnf -y --setopt=install_weak_deps=False install dnf-plugins-core + dnf config-manager --set-enabled crb + dnf -y --setopt=install_weak_deps=False install gcc make binutils libcurl-devel fuse-devel libxcrypt-devel + ;; +esac + +make -C confluent_osdeploy/utils +mkdir -p "$OUT" +cd confluent_osdeploy/utils +cp confluent_imginfo copernicus clortho autocons start_root urlmount "$OUT"/ +make clean diff --git a/.github/scripts/mkbundle.sh b/.github/scripts/mkbundle.sh new file mode 100755 index 00000000..964afc44 --- /dev/null +++ b/.github/scripts/mkbundle.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Assemble the nightly bundle for one target: the packages, repository +# metadata so the directory also works as a dnf/zypper/apt repository, and +# an INSTALL.txt. Produces confluent-nightly-.tar.gz in . +# +# Usage: mkbundle.sh ... +# target is - +set -e + +TARGET=$1 +VERSION=$2 +OUTDIR=$(readlink -f "$3") +shift 3 + +NAME=confluent-nightly-$TARGET +DIR=$OUTDIR/$NAME +REPO=${GITHUB_REPOSITORY:-xcat2/confluent} +SHA=${GITHUB_SHA:-$(git rev-parse HEAD)} +rm -rf "$DIR" +mkdir -p "$DIR" +cp "$@" "$DIR"/ +echo "$VERSION" > "$DIR"/VERSION + +cat > "$DIR"/INSTALL.txt <> "$DIR"/INSTALL.txt <> "$DIR"/INSTALL.txt < /dev/null | gzip -9 > Packages.gz) + cat >> "$DIR"/INSTALL.txt < /etc/apt/sources.list.d/confluent-nightly.list + apt update && apt install confluent-server +EOF + ;; +*) + echo "unknown target $TARGET" >&2 + exit 1 + ;; +esac + +cat >> "$DIR"/INSTALL.txt < +set -x + +PKGDIR=$(readlink -f "$1") +FAIL=0 +. /etc/os-release + +case "$ID" in +almalinux|rocky|rhel|centos) + dnf -y install epel-release + dnf config-manager --set-enabled crb + dnf -y install procps-ng util-linux + echo "::group::install packages" + dnf -y install "$PKGDIR"/*.rpm || FAIL=1 + echo "::endgroup::" + ;; +opensuse-leap|sles) + zypper -n install procps util-linux + echo "::group::install packages" + # nightly rpms are unsigned + zypper -n --no-gpg-checks install --allow-unsigned-rpm "$PKGDIR"/*.rpm || FAIL=1 + echo "::endgroup::" + ;; +ubuntu|debian) + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get -y install procps util-linux + echo "::group::install packages" + apt-get -y install "$PKGDIR"/*.deb || FAIL=1 + echo "::endgroup::" + ;; +*) + echo "unsupported distribution $ID" >&2 + exit 1 + ;; +esac + +echo "::group::installed content" +test -d /opt/confluent/lib/osdeploy/el10/profiles || FAIL=1 +test -f /opt/confluent/lib/ipxe/ipxe.efi || FAIL=1 +test -x /opt/confluent/bin/browserfs || FAIL=1 +test -x /opt/confluent/bin/vinz || FAIL=1 +test -x /opt/confluent/bin/vtbufferd || FAIL=1 +test -x /opt/confluent/bin/imgutil || FAIL=1 +echo "::endgroup::" + +echo "::group::import and daemon smoke test" +export PYTHONPATH=/opt/confluent/lib/python +python3 -c "import confluent.main; print('IMPORT confluent.main: OK')" || FAIL=1 +python3 -c "import confluent.snmputil; print('IMPORT confluent.snmputil: OK')" || FAIL=1 +python3 -c "import aiohmi.redfish.command; print('IMPORT aiohmi.redfish: OK')" || FAIL=1 +# packaging sets up non-root operation (/etc/confluent owned by confluent), +# and there is no systemd in the container to start it, so run it directly +mkdir -p /var/run/confluent +chown confluent:confluent /var/run/confluent +timeout 60 runuser -u confluent -- /opt/confluent/bin/confluent -f > /tmp/confluentd.log 2>&1 & +for _ in $(seq 30); do + [ -S /var/run/confluent/api.sock ] && break + sleep 1 +done +if [ -S /var/run/confluent/api.sock ] && /opt/confluent/bin/confetty show /; then + echo "DAEMON_RUNNING=yes" +else + echo "DAEMON_RUNNING=no" + FAIL=1 +fi +cat /tmp/confluentd.log +pkill -f /opt/confluent/bin/confluent +echo "::endgroup::" + +exit $FAIL diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000..edf6fc21 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,458 @@ +name: Nightly + +# Builds the confluent rpms (EL9, EL10, SUSE 16) and debs (Ubuntu noble, +# resolute) for x86_64 and aarch64, installs each bundle on a clean container, +# and publishes the bundles as the rolling "nightly" pre-release. Pull +# requests that touch the packaging build and test, but publish nothing. + +on: + schedule: + - cron: '23 2 * * *' + workflow_dispatch: + inputs: + publish: + description: Publish the result as the nightly pre-release + type: boolean + default: true + force: + description: Build even if nothing changed since the last nightly + type: boolean + default: false + pull_request: + paths: + - .github/workflows/nightly.yml + - .github/scripts/** + - '**/buildrpm' + - '**/buildrpm-aarch64' + - '**/builddeb' + - '**/makesetup' + - '**/setup.py.tmpl' + - '**/*.spec' + - '**/*.spec.tmpl' + - confluent_osdeploy/utils/** + - build/arm/** + - mkversion + - VERSION + +permissions: + contents: read + +concurrency: + group: nightly-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + BROWSERFS_REPO: https://github.com/xcat2/browserfs + VINZ_REPO: https://github.com/xcat2/vinz + +defaults: + run: + shell: bash + +jobs: + prepare: + name: prepare + runs-on: ubuntu-26.04 + outputs: + build: ${{ steps.check.outputs.build }} + version: ${{ steps.check.outputs.version }} + key: ${{ steps.check.outputs.key }} + browserfs: ${{ steps.check.outputs.browserfs }} + vinz: ${{ steps.check.outputs.vinz }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Version and change check + id: check + env: + GH_TOKEN: ${{ github.token }} + FORCE: ${{ inputs.force }} + # The release notes carry the commits of the last published build, so + # a scheduled run with nothing new to build stops here. + run: | + version=$(./mkversion --tilde) + browserfs=$(git ls-remote "$BROWSERFS_REPO" HEAD | cut -f1) + vinz=$(git ls-remote "$VINZ_REPO" HEAD | cut -f1) + key="confluent@$GITHUB_SHA browserfs@$browserfs vinz@$vinz" + # no release yet on the first run, and bash runs with pipefail here + last=$({ gh release view nightly --repo "$GITHUB_REPOSITORY" --json body -q .body 2>/dev/null || true; } | + sed -n 's/^$/\1/p') + build=true + if [ "$GITHUB_EVENT_NAME" != pull_request ] && [ "$FORCE" != true ] && [ "$key" = "$last" ]; then + echo "::notice::Nothing changed since the last nightly ($last)" + build=false + fi + { + echo "build=$build" + echo "version=$version" + echo "key=$key" + echo "browserfs=$browserfs" + echo "vinz=$vinz" + } | tee -a "$GITHUB_OUTPUT" + + utils: + name: osdeploy utils (${{ matrix.set }}-${{ matrix.arch }}) + needs: prepare + if: needs.prepare.outputs.build == 'true' + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + # The el8bin set must run on the oldest target userspace, so it is + # built where confluent_osdeploy/buildrpm builds it: el7 for x86_64, + # el8 for aarch64. CentOS 7 is too old for the node-based actions, + # so these run the image by hand instead of as a job container. + include: + - { set: el8, arch: x86_64, image: "centos:7", runner: ubuntu-26.04 } + - { set: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 } + - { set: el8, arch: aarch64, image: "almalinux:8", runner: ubuntu-26.04-arm } + - { set: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm } + steps: + - uses: actions/checkout@v7 + - name: Build utils + run: docker run --rm -v "$PWD:/src" -w /src ${{ matrix.image }} bash .github/scripts/build-utils.sh /src/utilsbin + - uses: actions/upload-artifact@v7 + with: + name: utils-${{ matrix.set }}-${{ matrix.arch }} + path: utilsbin/ + retention-days: 3 + + rpms: + name: RPMs (${{ matrix.dist }}-${{ matrix.arch }}) + needs: prepare + if: needs.prepare.outputs.build == 'true' + runs-on: ${{ matrix.runner }} + container: ${{ matrix.image }} + strategy: + fail-fast: false + matrix: + # noarch rpms are built in the x86_64 leg only, and shared by both + include: + - { dist: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 } + - { dist: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm } + - { dist: el10, arch: x86_64, image: "almalinux:10", runner: ubuntu-26.04 } + - { dist: el10, arch: aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm } + - { dist: suse16, arch: x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 } + - { dist: suse16, arch: aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm } + steps: + - name: Install git + # without git in the container, checkout downloads a tarball and + # mkversion has no history to describe + run: | + if command -v zypper > /dev/null; then + zypper -n install --no-recommends git-core tar gzip + else + dnf -y --setopt=install_weak_deps=False install git-core + fi + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Build RPMs + run: bash .github/scripts/build-rpms.sh ${{ matrix.arch == 'aarch64' && '--arch-only' || '' }} + - uses: actions/upload-artifact@v7 + with: + name: rpms-${{ matrix.dist }}-${{ matrix.arch }} + path: ~/rpmbuild/RPMS/ + retention-days: 3 + + osdeploy: + name: osdeploy RPM (${{ matrix.arch }}) + needs: utils + runs-on: ubuntu-26.04 + container: almalinux:9 + strategy: + fail-fast: false + matrix: + arch: [x86_64, aarch64] + steps: + - name: Install build dependencies + run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core tar xz cpio + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/download-artifact@v8 + with: + name: utils-el8-${{ matrix.arch }} + path: el8utils + - uses: actions/download-artifact@v8 + with: + name: utils-el9-${{ matrix.arch }} + path: el9utils + - name: Build osdeploy RPM + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + bash .github/scripts/build-osdeploy.sh ${{ matrix.arch }} el8utils el9utils + - uses: actions/upload-artifact@v7 + with: + name: rpms-osdeploy-${{ matrix.arch }} + path: ~/rpmbuild/RPMS/ + retention-days: 3 + + ipxe: + name: iPXE RPM + needs: prepare + if: needs.prepare.outputs.build == 'true' + runs-on: ubuntu-26.04 + container: almalinux:9 + steps: + - name: Install build dependencies + run: dnf -y --setopt=install_weak_deps=False install rpm-build tar gzip + - uses: actions/checkout@v7 + - name: Build iPXE RPM + run: bash .github/scripts/build-ipxe.sh + - uses: actions/upload-artifact@v7 + with: + name: rpms-ipxe + path: ~/rpmbuild/RPMS/ + retention-days: 3 + + go: + name: browserfs and vinz RPMs (${{ matrix.arch }}) + needs: prepare + if: needs.prepare.outputs.build == 'true' + runs-on: ${{ matrix.runner }} + container: almalinux:10 + strategy: + fail-fast: false + matrix: + # static (CGO_ENABLED=0) binaries, so one build serves every distro + include: + - { arch: x86_64, runner: ubuntu-26.04 } + - { arch: aarch64, runner: ubuntu-26.04-arm } + steps: + - name: Install build dependencies + run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core golang tar + - uses: actions/checkout@v7 + - name: Build RPMs + run: | + bash .github/scripts/build-go-rpm.sh browserfs "$BROWSERFS_REPO" ${{ needs.prepare.outputs.browserfs }} + bash .github/scripts/build-go-rpm.sh vinz "$VINZ_REPO" ${{ needs.prepare.outputs.vinz }} + - uses: actions/upload-artifact@v7 + with: + name: rpms-go-${{ matrix.arch }} + path: ~/rpmbuild/RPMS/ + retention-days: 3 + + debs: + name: debs (${{ matrix.release }}-${{ matrix.arch }}) + needs: [osdeploy, ipxe, go] + runs-on: ${{ matrix.runner }} + container: ubuntu:${{ matrix.release }} + strategy: + fail-fast: false + matrix: + release: [noble, resolute] + arch: [amd64, arm64] + include: + # alien only converts rpms for the architecture it runs on + - { arch: amd64, rpmarch: x86_64, runner: ubuntu-26.04 } + - { arch: arm64, rpmarch: aarch64, runner: ubuntu-26.04-arm } + steps: + - name: Install git + # ca-certificates is only a Recommends of git, and checkout clones over https + run: apt-get update && apt-get -y --no-install-recommends install git ca-certificates + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + # noarch, so converted in the amd64 leg only, like the noarch rpms + - uses: actions/download-artifact@v8 + if: matrix.arch == 'amd64' + with: + pattern: rpms-osdeploy-* + path: alien + merge-multiple: true + - uses: actions/download-artifact@v8 + if: matrix.arch == 'amd64' + with: + name: rpms-ipxe + path: alien + - uses: actions/download-artifact@v8 + with: + name: rpms-go-${{ matrix.rpmarch }} + path: alien + - name: Build debs + run: bash .github/scripts/build-debs.sh ${{ matrix.arch == 'arm64' && '--arch-only' || '' }} debout alien + - uses: actions/upload-artifact@v7 + with: + name: debs-${{ matrix.release }}-${{ matrix.arch }} + path: debout/*.deb + retention-days: 3 + if-no-files-found: error + + bundle: + name: assemble bundles + needs: [prepare, rpms, osdeploy, ipxe, go, debs] + runs-on: ubuntu-26.04 + steps: + - uses: actions/checkout@v7 + - name: Install tools + run: sudo apt-get update && sudo apt-get -y --no-install-recommends install createrepo-c dpkg-dev + - uses: actions/download-artifact@v8 + with: + path: art + - name: Compose bundles + env: + VERSION: ${{ needs.prepare.outputs.version }} + run: | + set -e + pick() { find "$@" -name '*.rpm' ! -name '*.src.rpm'; } + for dist in el9 el10 suse16; do + for arch in x86_64 aarch64; do + mapfile -t files < <( + pick "art/rpms-$dist-x86_64" -name '*.noarch.rpm' + pick "art/rpms-$dist-$arch" -name "*.$arch.rpm" + pick art/rpms-osdeploy-* art/rpms-ipxe "art/rpms-go-$arch") + bash .github/scripts/mkbundle.sh "$dist-$arch" "$VERSION" bundles "${files[@]}" + done + done + # Architecture: all debs come from the amd64 leg only + pickdeb() { # + for f in "$1"/*.deb; do + [ "$(dpkg-deb -f "$f" Architecture)" = "$2" ] && echo "$f" + done + return 0 + } + for release in noble resolute; do + for arch in amd64 arm64; do + mapfile -t files < <( + pickdeb "art/debs-$release-amd64" all + pickdeb "art/debs-$release-$arch" "$arch") + bash .github/scripts/mkbundle.sh "$release-$arch" "$VERSION" bundles "${files[@]}" + done + done + - name: Write run summary + env: + VERSION: ${{ needs.prepare.outputs.version }} + run: | + { + echo "## confluent nightly $VERSION" + echo + for b in bundles/*.tar.gz; do + echo "
$(basename "$b")" + echo + tar tzf "$b" | sed -n 's|^[^/]*/\([^/]*\.\(rpm\|deb\)\)$|- \1|p' | sort + echo + echo "
" + done + } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v7 + with: + name: bundles + path: bundles/*.tar.gz + retention-days: 7 + if-no-files-found: error + + smoketest: + name: smoketest (${{ matrix.target }}) + needs: bundle + runs-on: ${{ matrix.runner }} + container: ${{ matrix.image }} + strategy: + fail-fast: false + matrix: + include: + - { target: el9-x86_64, image: "almalinux:9", runner: ubuntu-26.04 } + - { target: el9-aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm } + - { target: el10-x86_64, image: "almalinux:10", runner: ubuntu-26.04 } + - { target: el10-aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm } + - { target: suse16-x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 } + - { target: suse16-aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm } + - { target: noble-amd64, image: "ubuntu:noble", runner: ubuntu-26.04 } + - { target: noble-arm64, image: "ubuntu:noble", runner: ubuntu-26.04-arm } + - { target: resolute-amd64, image: "ubuntu:resolute", runner: ubuntu-26.04 } + - { target: resolute-arm64, image: "ubuntu:resolute", runner: ubuntu-26.04-arm } + steps: + - name: Install tar + run: | + if command -v zypper > /dev/null; then + zypper -n install --no-recommends tar gzip + elif command -v dnf > /dev/null; then + dnf -y --setopt=install_weak_deps=False install tar gzip + fi + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: bundles + path: bundles + - name: Install and smoke test + run: | + tar xzf bundles/confluent-nightly-${{ matrix.target }}.tar.gz + bash .github/scripts/smoketest.sh confluent-nightly-${{ matrix.target }} + + publish: + name: publish nightly release + needs: [prepare, smoketest] + if: >- + needs.prepare.outputs.build == 'true' && + (github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && inputs.publish)) + runs-on: ubuntu-26.04 + permissions: + contents: write + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + VERSION: ${{ needs.prepare.outputs.version }} + KEY: ${{ needs.prepare.outputs.key }} + BROWSERFS: ${{ needs.prepare.outputs.browserfs }} + VINZ: ${{ needs.prepare.outputs.vinz }} + steps: + - uses: actions/download-artifact@v8 + with: + name: bundles + path: bundles + - name: Write release notes + run: | + url=https://github.com/$GITHUB_REPOSITORY + cat > notes.md <> notes.md + done + cat >> notes.md < + EOF + cat notes.md + - name: Publish + # Move the tag and replace the assets in place, so the release and its + # download URLs never disappear between nights. + run: | + set -e + if gh release view nightly > /dev/null 2>&1; then + gh api -X PATCH "repos/$GH_REPO/git/refs/tags/nightly" -f sha="$GITHUB_SHA" -F force=true > /dev/null + gh release edit nightly --title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false + for old in $(gh release view nightly --json assets -q '.assets[].name'); do + [ -f "bundles/$old" ] || gh release delete-asset nightly "$old" -y + done + gh release upload nightly bundles/*.tar.gz --clobber + else + # a stale tag without a release would keep pointing at its old commit + gh api -X DELETE "repos/$GH_REPO/git/refs/tags/nightly" > /dev/null 2>&1 || true + gh release create nightly bundles/*.tar.gz --target "$GITHUB_SHA" \ + --title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4243ed10..a71c9a3c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,8 @@ on: push: tags: - '*' + # the rolling pre-release nightly.yml maintains + - '!nightly' permissions: contents: write diff --git a/README.md b/README.md index 27d110b0..f07d297e 100644 --- a/README.md +++ b/README.md @@ -48,3 +48,11 @@ Cutting a new series: wrong series. Patch releases land on branch `X.Y` and are tagged `X.Y.Z` there; no `VERSION` edit is needed. + +## Nightly builds + +`.github/workflows/nightly.yml` builds the rpms (EL9, EL10, SUSE 16) and debs (Ubuntu 24.04, 26.04) +for x86_64 and aarch64 every night that something changed, installs each set on a clean container, +and publishes them as the `nightly` pre-release. Every bundle there holds all packages for one +target and an `INSTALL.txt`. Pull requests that touch the packaging run the same build and tests +without publishing. Genesis is not built there yet. From bb7563ec5129d818198cf2009928f671e3e4dbfa Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 15 Oct 2026 19:41:41 +0200 Subject: [PATCH 6/6] Add status badges to the README Replace the static license badge with a live one, and add badges for CI, the nightly build, the latest release and the documentation. The docs badge carries the Octicons book icon (MIT, GitHub Inc.) inline, as Simple Icons has no generic one. --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f07d297e..162c85e8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ # Confluent -![Python 3](https://img.shields.io/badge/python-3-blue.svg) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/xcat2/confluent/blob/master/LICENSE) +[![CI](https://github.com/xcat2/confluent/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/xcat2/confluent/actions/workflows/ci.yml?query=branch%3Amaster) +[![Nightly](https://github.com/xcat2/confluent/actions/workflows/nightly.yml/badge.svg?branch=master)](https://github.com/xcat2/confluent/actions/workflows/nightly.yml?query=branch%3Amaster) +[![Release](https://img.shields.io/github/v/release/xcat2/confluent?sort=semver)](https://github.com/xcat2/confluent/releases/latest) +[![Docs](https://img.shields.io/badge/docs-confluent--docs-blueviolet?logo=data:image/svg%2Bxml;base64,PHN2ZyBmaWxsPSJ3aGl0ZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB3aWR0aD0iMTYiIGhlaWdodD0iMTYiIHZpZXdCb3g9IjAgMCAxNiAxNiI%2BPHBhdGggZD0iTTAgMS43NUEuNzUuNzUgMCAwIDEgLjc1IDFoNC4yNTNjMS4yMjcgMCAyLjMxNy41OSAzIDEuNTAxQTMuNzQzIDMuNzQzIDAgMCAxIDExLjAwNiAxaDQuMjQ1YS43NS43NSAwIDAgMSAuNzUuNzV2MTAuNWEuNzUuNzUgMCAwIDEtLjc1Ljc1aC00LjUwN2EyLjI1IDIuMjUgMCAwIDAtMS41OTEuNjU5bC0uNjIyLjYyMWEuNzUuNzUgMCAwIDEtMS4wNiAwbC0uNjIyLS42MjFBMi4yNSAyLjI1IDAgMCAwIDUuMjU4IDEzSC43NWEuNzUuNzUgMCAwIDEtLjc1LS43NVptNy4yNTEgMTAuMzI0LjAwNC01LjA3My0uMDAyLTIuMjUzQTIuMjUgMi4yNSAwIDAgMCA1LjAwMyAyLjVIMS41djloMy43NTdhMy43NSAzLjc1IDAgMCAxIDEuOTk0LjU3NFpNOC43NTUgNC43NWwtLjAwNCA3LjMyMmEzLjc1MiAzLjc1MiAwIDAgMSAxLjk5Mi0uNTcySDE0LjV2LTloLTMuNDk1YTIuMjUgMi4yNSAwIDAgMC0yLjI1IDIuMjVaIi8%2BPC9zdmc%2B)](https://xcat2.github.io/confluent-docs/) +![Python 3](https://img.shields.io/badge/python-3-3776ab?logo=python&logoColor=white) +[![License](https://img.shields.io/github/license/xcat2/confluent)](LICENSE) Confluent is a software package to handle essential bootstrap and operation of scale-out server configurations. It supports stateful and stateless deployments for various operating systems.