From 6c48cb4fad9db533132fd94e98c41b9a935869bd Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 15 Oct 2026 17:51:15 +0200 Subject: [PATCH 1/2] Build and publish nightly packages Build the rpms for EL9, EL10 and SUSE 16 and the debs for Ubuntu noble and resolute, each for x86_64 and aarch64, install every set on a clean container and publish them as a rolling nightly pre-release. Each release asset is one target's bundle: all of its packages, repository metadata and an INSTALL.txt. Besides the in-tree components this packages confluent_ipxe from the iPXE release roms and the browserfs and vinz helpers from their own repositories. The osdeploy utils are built where buildrpm builds them, el7 for the x86_64 el8bin set, so they still run on the oldest targets. Genesis is left out for now. Noarch packages are built once, in the x86_64 legs, and the aarch64 legs build only what is architecture specific. Build tools are installed without weak dependencies, while the smoke tests keep the distribution defaults, since they stand in for a user's install. Scheduled runs skip the build when neither confluent nor the helper repositories moved since the last published nightly. Pull requests that touch the packaging build and test without publishing. --- .github/scripts/build-debs.sh | 48 ++++ .github/scripts/build-go-rpm.sh | 35 +++ .github/scripts/build-ipxe.sh | 13 + .github/scripts/build-osdeploy.sh | 40 +++ .github/scripts/build-rpms.sh | 38 +++ .github/scripts/build-utils.sh | 38 +++ .github/scripts/mkbundle.sh | 91 ++++++ .github/scripts/smoketest.sh | 78 +++++ .github/workflows/nightly.yml | 458 ++++++++++++++++++++++++++++++ .github/workflows/release.yml | 2 + README.md | 8 + 11 files changed, 849 insertions(+) create mode 100755 .github/scripts/build-debs.sh create mode 100755 .github/scripts/build-go-rpm.sh create mode 100755 .github/scripts/build-ipxe.sh create mode 100755 .github/scripts/build-osdeploy.sh create mode 100755 .github/scripts/build-rpms.sh create mode 100755 .github/scripts/build-utils.sh create mode 100755 .github/scripts/mkbundle.sh create mode 100755 .github/scripts/smoketest.sh create mode 100644 .github/workflows/nightly.yml diff --git a/.github/scripts/build-debs.sh b/.github/scripts/build-debs.sh new file mode 100755 index 00000000..efe8d1cf --- /dev/null +++ b/.github/scripts/build-debs.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Build the confluent debs in an Ubuntu container, like build/arm/*/buildapt.sh: +# every component with a builddeb script, plus alien conversions of the rpms +# that have no deb packaging of their own (osdeploy, ipxe, browserfs, vinz). +# With --arch-only, only the architecture-specific ones: confluent_vtbufferd +# and the conversions of rpms that are not noarch. +# +# Usage: build-debs.sh [--arch-only] [] +set -e + +ARCHONLY= +if [ "$1" = "--arch-only" ]; then + ARCHONLY=1 + shift +fi +OUT=$(readlink -f "$1") +RPMS=${2:+$(readlink -f "$2")} +[ -n "$OUT" ] || { echo "usage: $0 [--arch-only] [rpmdir]" >&2; exit 1; } + +cd "$(dirname "$0")/../.." +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get -y --no-install-recommends install git build-essential debhelper devscripts dh-python fakeroot \ + python3-all python3-setuptools python3-stdeb pandoc patch alien +# stdeb on noble still imports the SafeConfigParser python 3.12 removed +if (cd / && patch -p1 --dry-run -s < "$OLDPWD/build/arm/noble/stdeb.patch") > /dev/null 2>&1; then + (cd / && patch -p1 < "$OLDPWD/build/arm/noble/stdeb.patch") +fi +git config --global --add safe.directory '*' +./mkversion + +mkdir -p "$OUT" +# vtbufferd first: it recreates /tmp/confluent, which client and server reuse +./confluent_vtbufferd/builddeb "$OUT" +if [ -z "$ARCHONLY" ]; then + ./confluent_client/builddeb "$OUT" + ./confluent_server/builddeb "$OUT" + ./imgutil/builddeb "$OUT" +fi + +if [ -n "$RPMS" ]; then + skip=(! -name '*.src.rpm') + [ -n "$ARCHONLY" ] && skip+=(! -name '*.noarch.rpm') + cd "$OUT" + find "$RPMS" -name '*.rpm' "${skip[@]}" -print0 | + xargs -0 -r -n1 fakeroot alien --to-deb --scripts --keep-version +fi +ls -la "$OUT" diff --git a/.github/scripts/build-go-rpm.sh b/.github/scripts/build-go-rpm.sh new file mode 100755 index 00000000..bdf88627 --- /dev/null +++ b/.github/scripts/build-go-rpm.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Build the rpm of one of the Go helpers confluent runs but keeps in separate +# repositories (browserfs, vinz), for the native architecture only. +# +# Mirrors the buildrpm script in those repositories, which needs git tags for +# its version. They carry none yet, so fall back to 0~dev+g, +# which sorts below any tagged release. +# +# Usage: build-go-rpm.sh [ref] +set -e + +NAME=$1 +URL=$2 +REF=${3:-HEAD} +WORK=$(mktemp -d) + +git clone -q "$URL" "$WORK/$NAME" +cd "$WORK/$NAME" +git checkout -q "$REF" +if desc=$(git describe 2>/dev/null); then + VERSION=${desc%%-*} + NUMCOMMITS=$(echo "$desc" | cut -d- -f2) + if [ "$NUMCOMMITS" != "$VERSION" ]; then + VERSION=$(echo "$VERSION" | awk -F. -v OFS=. '{$NF=$NF+1; print}') + VERSION=$VERSION~dev$NUMCOMMITS+$(echo "$desc" | cut -d- -f3) + fi +else + VERSION=0~dev$(git rev-list --count HEAD)+g$(git rev-parse --short HEAD) +fi +echo "$NAME $VERSION ($(git rev-parse HEAD))" + +mkdir -p ~/rpmbuild/SOURCES +(cd .. && tar cf ~/rpmbuild/SOURCES/confluent-$NAME.tar --exclude=.git "$NAME") +sed -e "s/#VERSION#/$VERSION/" $NAME.spec.tmpl > $NAME.spec +rpmbuild -ba $NAME.spec diff --git a/.github/scripts/build-ipxe.sh b/.github/scripts/build-ipxe.sh new file mode 100755 index 00000000..0d795669 --- /dev/null +++ b/.github/scripts/build-ipxe.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Build the confluent_ipxe rpm. The spec repackages the prebuilt boot roms +# that iPXE publishes with each release; rpmbuild does not fetch remote +# sources itself, so download the tarball of the release the spec names. +set -e + +cd "$(dirname "$0")/../.." +SPEC=ipxe/confluent_ipxe.spec +VERSION=$(rpmspec -q --srpm --qf '%{version}' $SPEC) +mkdir -p ~/rpmbuild/SOURCES +curl -fsSL -o ~/rpmbuild/SOURCES/ipxeboot.tar.gz \ + "https://github.com/ipxe/ipxe/releases/download/v$VERSION/ipxeboot.tar.gz" +rpmbuild -ba $SPEC diff --git a/.github/scripts/build-osdeploy.sh b/.github/scripts/build-osdeploy.sh new file mode 100755 index 00000000..c71efb1d --- /dev/null +++ b/.github/scripts/build-osdeploy.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Build the confluent_osdeploy rpm the way confluent_osdeploy/buildrpm and +# buildrpm-aarch64 do, but from utils binaries built by earlier CI jobs +# (build-utils.sh) instead of compiling them in local podman containers. +# +# Usage: build-osdeploy.sh +set -e + +ARCH=$1 +EL8UTILS=$(readlink -f "$2") +EL9UTILS=$(readlink -f "$3") + +cd "$(dirname "$0")/../.." +VERSION=$(./mkversion --tilde) + +case "$ARCH" in +x86_64) SPEC=confluent_osdeploy.spec ;; +aarch64) SPEC=confluent_osdeploy-aarch64.spec ;; +*) echo "unknown arch $ARCH" >&2; exit 1 ;; +esac +sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy/$SPEC.tmpl > confluent_osdeploy/$SPEC + +mkdir -p ~/rpmbuild/SOURCES +cp LICENSE confluent_osdeploy/ +tar Jcf ~/rpmbuild/SOURCES/confluent_osdeploy.tar.xz confluent_osdeploy + +for gen in el8 el9; do + utils=$EL8UTILS + [ "$gen" = el9 ] && utils=$EL9UTILS + rm -rf ${gen}bin + mkdir -p ${gen}bin/opt/confluent/bin ${gen}bin/stateless-bin + (cd "$utils" && cp confluent_imginfo copernicus clortho autocons "$OLDPWD"/${gen}bin/opt/confluent/bin/) + (cd "$utils" && cp start_root urlmount "$OLDPWD"/${gen}bin/stateless-bin/) + # artifact downloads lose the execute bit + chmod 755 ${gen}bin/opt/confluent/bin/* ${gen}bin/stateless-bin/* + tar Jcf ~/rpmbuild/SOURCES/confluent_${gen}bin.tar.xz ${gen}bin/ + rm -rf ${gen}bin +done + +rpmbuild -ba confluent_osdeploy/$SPEC diff --git a/.github/scripts/build-rpms.sh b/.github/scripts/build-rpms.sh new file mode 100755 index 00000000..92981385 --- /dev/null +++ b/.github/scripts/build-rpms.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Build the confluent rpms whose spec depends on the target distribution: +# confluent_vtbufferd (arch-specific) and, unless --arch-only is given, the +# noarch client, server and imgutil. Runs inside an EL or openSUSE container. +# +# Usage: build-rpms.sh [--arch-only] +set -e + +cd "$(dirname "$0")/../.." + +if command -v zypper > /dev/null; then + zypper -n install --no-recommends rpm-build gcc make git-core tar xz pandoc systemd python3 python3-setuptools + # the buildrpm scripts stage into ~/rpmbuild, SUSE defaults to /usr/src/packages + echo "%_topdir $HOME/rpmbuild" >> ~/.rpmmacros +else + dnf -y --setopt=install_weak_deps=False install epel-release + dnf -y --setopt=install_weak_deps=False install rpm-build gcc make git-core tar xz pandoc systemd python3 python3-setuptools +fi +git config --global --add safe.directory "$PWD" +./mkversion +mkdir -p ~/rpmbuild/SOURCES ~/rpmbuild/SPECS + +# the buildrpm scripts hide rpmbuild's output, so on failure rerun the spec +# they wrote to show why +build() { + "$1/buildrpm" && return 0 + spec=~/rpmbuild/SPECS/$(basename "$1").spec + [ -f "$spec" ] && rpmbuild -ba "$spec" + return 1 +} + +build confluent_vtbufferd +if [ "$1" != "--arch-only" ]; then + build confluent_client + build confluent_server + # imgutil/buildrpm calls rpmbuild directly, so its output is visible + ./imgutil/buildrpm +fi diff --git a/.github/scripts/build-utils.sh b/.github/scripts/build-utils.sh new file mode 100755 index 00000000..4bbac8f8 --- /dev/null +++ b/.github/scripts/build-utils.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Build the confluent_osdeploy utils inside a distro container, standing in for +# the el7build/el8builder/el9build podman images confluent_osdeploy/buildrpm +# uses. The el8bin set is linked against the oldest glibc any target runs +# (el7 on x86_64, el8 on aarch64), so it must be built on that distro. +# +# Usage: build-utils.sh (run from the repository root) +set -e + +OUT=$1 +[ -n "$OUT" ] || { echo "usage: $0 " >&2; exit 1; } + +. /etc/os-release +case "$VERSION_ID" in +7*) + # CentOS 7 is EOL: its mirrors are gone, only the vault remains + sed -i -e 's/^mirrorlist=/#mirrorlist=/' \ + -e 's|^#\?baseurl=http://mirror.centos.org|baseurl=https://vault.centos.org|' \ + /etc/yum.repos.d/CentOS-*.repo + yum -y install gcc make binutils libcurl-devel fuse-devel + ;; +8*) + dnf -y --setopt=install_weak_deps=False install dnf-plugins-core + dnf config-manager --set-enabled powertools + dnf -y --setopt=install_weak_deps=False install gcc make binutils libcurl-devel fuse-devel libxcrypt-devel + ;; +*) + dnf -y --setopt=install_weak_deps=False install dnf-plugins-core + dnf config-manager --set-enabled crb + dnf -y --setopt=install_weak_deps=False install gcc make binutils libcurl-devel fuse-devel libxcrypt-devel + ;; +esac + +make -C confluent_osdeploy/utils +mkdir -p "$OUT" +cd confluent_osdeploy/utils +cp confluent_imginfo copernicus clortho autocons start_root urlmount "$OUT"/ +make clean diff --git a/.github/scripts/mkbundle.sh b/.github/scripts/mkbundle.sh new file mode 100755 index 00000000..964afc44 --- /dev/null +++ b/.github/scripts/mkbundle.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Assemble the nightly bundle for one target: the packages, repository +# metadata so the directory also works as a dnf/zypper/apt repository, and +# an INSTALL.txt. Produces confluent-nightly-.tar.gz in . +# +# Usage: mkbundle.sh ... +# target is - +set -e + +TARGET=$1 +VERSION=$2 +OUTDIR=$(readlink -f "$3") +shift 3 + +NAME=confluent-nightly-$TARGET +DIR=$OUTDIR/$NAME +REPO=${GITHUB_REPOSITORY:-xcat2/confluent} +SHA=${GITHUB_SHA:-$(git rev-parse HEAD)} +rm -rf "$DIR" +mkdir -p "$DIR" +cp "$@" "$DIR"/ +echo "$VERSION" > "$DIR"/VERSION + +cat > "$DIR"/INSTALL.txt <> "$DIR"/INSTALL.txt <> "$DIR"/INSTALL.txt < /dev/null | gzip -9 > Packages.gz) + cat >> "$DIR"/INSTALL.txt < /etc/apt/sources.list.d/confluent-nightly.list + apt update && apt install confluent-server +EOF + ;; +*) + echo "unknown target $TARGET" >&2 + exit 1 + ;; +esac + +cat >> "$DIR"/INSTALL.txt < +set -x + +PKGDIR=$(readlink -f "$1") +FAIL=0 +. /etc/os-release + +case "$ID" in +almalinux|rocky|rhel|centos) + dnf -y install epel-release + dnf config-manager --set-enabled crb + dnf -y install procps-ng util-linux + echo "::group::install packages" + dnf -y install "$PKGDIR"/*.rpm || FAIL=1 + echo "::endgroup::" + ;; +opensuse-leap|sles) + zypper -n install procps util-linux + echo "::group::install packages" + # nightly rpms are unsigned + zypper -n --no-gpg-checks install --allow-unsigned-rpm "$PKGDIR"/*.rpm || FAIL=1 + echo "::endgroup::" + ;; +ubuntu|debian) + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get -y install procps util-linux + echo "::group::install packages" + apt-get -y install "$PKGDIR"/*.deb || FAIL=1 + echo "::endgroup::" + ;; +*) + echo "unsupported distribution $ID" >&2 + exit 1 + ;; +esac + +echo "::group::installed content" +test -d /opt/confluent/lib/osdeploy/el10/profiles || FAIL=1 +test -f /opt/confluent/lib/ipxe/ipxe.efi || FAIL=1 +test -x /opt/confluent/bin/browserfs || FAIL=1 +test -x /opt/confluent/bin/vinz || FAIL=1 +test -x /opt/confluent/bin/vtbufferd || FAIL=1 +test -x /opt/confluent/bin/imgutil || FAIL=1 +echo "::endgroup::" + +echo "::group::import and daemon smoke test" +export PYTHONPATH=/opt/confluent/lib/python +python3 -c "import confluent.main; print('IMPORT confluent.main: OK')" || FAIL=1 +python3 -c "import confluent.snmputil; print('IMPORT confluent.snmputil: OK')" || FAIL=1 +python3 -c "import aiohmi.redfish.command; print('IMPORT aiohmi.redfish: OK')" || FAIL=1 +# packaging sets up non-root operation (/etc/confluent owned by confluent), +# and there is no systemd in the container to start it, so run it directly +mkdir -p /var/run/confluent +chown confluent:confluent /var/run/confluent +timeout 60 runuser -u confluent -- /opt/confluent/bin/confluent -f > /tmp/confluentd.log 2>&1 & +for _ in $(seq 30); do + [ -S /var/run/confluent/api.sock ] && break + sleep 1 +done +if [ -S /var/run/confluent/api.sock ] && /opt/confluent/bin/confetty show /; then + echo "DAEMON_RUNNING=yes" +else + echo "DAEMON_RUNNING=no" + FAIL=1 +fi +cat /tmp/confluentd.log +pkill -f /opt/confluent/bin/confluent +echo "::endgroup::" + +exit $FAIL diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000..edf6fc21 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,458 @@ +name: Nightly + +# Builds the confluent rpms (EL9, EL10, SUSE 16) and debs (Ubuntu noble, +# resolute) for x86_64 and aarch64, installs each bundle on a clean container, +# and publishes the bundles as the rolling "nightly" pre-release. Pull +# requests that touch the packaging build and test, but publish nothing. + +on: + schedule: + - cron: '23 2 * * *' + workflow_dispatch: + inputs: + publish: + description: Publish the result as the nightly pre-release + type: boolean + default: true + force: + description: Build even if nothing changed since the last nightly + type: boolean + default: false + pull_request: + paths: + - .github/workflows/nightly.yml + - .github/scripts/** + - '**/buildrpm' + - '**/buildrpm-aarch64' + - '**/builddeb' + - '**/makesetup' + - '**/setup.py.tmpl' + - '**/*.spec' + - '**/*.spec.tmpl' + - confluent_osdeploy/utils/** + - build/arm/** + - mkversion + - VERSION + +permissions: + contents: read + +concurrency: + group: nightly-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + BROWSERFS_REPO: https://github.com/xcat2/browserfs + VINZ_REPO: https://github.com/xcat2/vinz + +defaults: + run: + shell: bash + +jobs: + prepare: + name: prepare + runs-on: ubuntu-26.04 + outputs: + build: ${{ steps.check.outputs.build }} + version: ${{ steps.check.outputs.version }} + key: ${{ steps.check.outputs.key }} + browserfs: ${{ steps.check.outputs.browserfs }} + vinz: ${{ steps.check.outputs.vinz }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Version and change check + id: check + env: + GH_TOKEN: ${{ github.token }} + FORCE: ${{ inputs.force }} + # The release notes carry the commits of the last published build, so + # a scheduled run with nothing new to build stops here. + run: | + version=$(./mkversion --tilde) + browserfs=$(git ls-remote "$BROWSERFS_REPO" HEAD | cut -f1) + vinz=$(git ls-remote "$VINZ_REPO" HEAD | cut -f1) + key="confluent@$GITHUB_SHA browserfs@$browserfs vinz@$vinz" + # no release yet on the first run, and bash runs with pipefail here + last=$({ gh release view nightly --repo "$GITHUB_REPOSITORY" --json body -q .body 2>/dev/null || true; } | + sed -n 's/^$/\1/p') + build=true + if [ "$GITHUB_EVENT_NAME" != pull_request ] && [ "$FORCE" != true ] && [ "$key" = "$last" ]; then + echo "::notice::Nothing changed since the last nightly ($last)" + build=false + fi + { + echo "build=$build" + echo "version=$version" + echo "key=$key" + echo "browserfs=$browserfs" + echo "vinz=$vinz" + } | tee -a "$GITHUB_OUTPUT" + + utils: + name: osdeploy utils (${{ matrix.set }}-${{ matrix.arch }}) + needs: prepare + if: needs.prepare.outputs.build == 'true' + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + # The el8bin set must run on the oldest target userspace, so it is + # built where confluent_osdeploy/buildrpm builds it: el7 for x86_64, + # el8 for aarch64. CentOS 7 is too old for the node-based actions, + # so these run the image by hand instead of as a job container. + include: + - { set: el8, arch: x86_64, image: "centos:7", runner: ubuntu-26.04 } + - { set: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 } + - { set: el8, arch: aarch64, image: "almalinux:8", runner: ubuntu-26.04-arm } + - { set: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm } + steps: + - uses: actions/checkout@v7 + - name: Build utils + run: docker run --rm -v "$PWD:/src" -w /src ${{ matrix.image }} bash .github/scripts/build-utils.sh /src/utilsbin + - uses: actions/upload-artifact@v7 + with: + name: utils-${{ matrix.set }}-${{ matrix.arch }} + path: utilsbin/ + retention-days: 3 + + rpms: + name: RPMs (${{ matrix.dist }}-${{ matrix.arch }}) + needs: prepare + if: needs.prepare.outputs.build == 'true' + runs-on: ${{ matrix.runner }} + container: ${{ matrix.image }} + strategy: + fail-fast: false + matrix: + # noarch rpms are built in the x86_64 leg only, and shared by both + include: + - { dist: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 } + - { dist: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm } + - { dist: el10, arch: x86_64, image: "almalinux:10", runner: ubuntu-26.04 } + - { dist: el10, arch: aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm } + - { dist: suse16, arch: x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 } + - { dist: suse16, arch: aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm } + steps: + - name: Install git + # without git in the container, checkout downloads a tarball and + # mkversion has no history to describe + run: | + if command -v zypper > /dev/null; then + zypper -n install --no-recommends git-core tar gzip + else + dnf -y --setopt=install_weak_deps=False install git-core + fi + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Build RPMs + run: bash .github/scripts/build-rpms.sh ${{ matrix.arch == 'aarch64' && '--arch-only' || '' }} + - uses: actions/upload-artifact@v7 + with: + name: rpms-${{ matrix.dist }}-${{ matrix.arch }} + path: ~/rpmbuild/RPMS/ + retention-days: 3 + + osdeploy: + name: osdeploy RPM (${{ matrix.arch }}) + needs: utils + runs-on: ubuntu-26.04 + container: almalinux:9 + strategy: + fail-fast: false + matrix: + arch: [x86_64, aarch64] + steps: + - name: Install build dependencies + run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core tar xz cpio + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/download-artifact@v8 + with: + name: utils-el8-${{ matrix.arch }} + path: el8utils + - uses: actions/download-artifact@v8 + with: + name: utils-el9-${{ matrix.arch }} + path: el9utils + - name: Build osdeploy RPM + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + bash .github/scripts/build-osdeploy.sh ${{ matrix.arch }} el8utils el9utils + - uses: actions/upload-artifact@v7 + with: + name: rpms-osdeploy-${{ matrix.arch }} + path: ~/rpmbuild/RPMS/ + retention-days: 3 + + ipxe: + name: iPXE RPM + needs: prepare + if: needs.prepare.outputs.build == 'true' + runs-on: ubuntu-26.04 + container: almalinux:9 + steps: + - name: Install build dependencies + run: dnf -y --setopt=install_weak_deps=False install rpm-build tar gzip + - uses: actions/checkout@v7 + - name: Build iPXE RPM + run: bash .github/scripts/build-ipxe.sh + - uses: actions/upload-artifact@v7 + with: + name: rpms-ipxe + path: ~/rpmbuild/RPMS/ + retention-days: 3 + + go: + name: browserfs and vinz RPMs (${{ matrix.arch }}) + needs: prepare + if: needs.prepare.outputs.build == 'true' + runs-on: ${{ matrix.runner }} + container: almalinux:10 + strategy: + fail-fast: false + matrix: + # static (CGO_ENABLED=0) binaries, so one build serves every distro + include: + - { arch: x86_64, runner: ubuntu-26.04 } + - { arch: aarch64, runner: ubuntu-26.04-arm } + steps: + - name: Install build dependencies + run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core golang tar + - uses: actions/checkout@v7 + - name: Build RPMs + run: | + bash .github/scripts/build-go-rpm.sh browserfs "$BROWSERFS_REPO" ${{ needs.prepare.outputs.browserfs }} + bash .github/scripts/build-go-rpm.sh vinz "$VINZ_REPO" ${{ needs.prepare.outputs.vinz }} + - uses: actions/upload-artifact@v7 + with: + name: rpms-go-${{ matrix.arch }} + path: ~/rpmbuild/RPMS/ + retention-days: 3 + + debs: + name: debs (${{ matrix.release }}-${{ matrix.arch }}) + needs: [osdeploy, ipxe, go] + runs-on: ${{ matrix.runner }} + container: ubuntu:${{ matrix.release }} + strategy: + fail-fast: false + matrix: + release: [noble, resolute] + arch: [amd64, arm64] + include: + # alien only converts rpms for the architecture it runs on + - { arch: amd64, rpmarch: x86_64, runner: ubuntu-26.04 } + - { arch: arm64, rpmarch: aarch64, runner: ubuntu-26.04-arm } + steps: + - name: Install git + # ca-certificates is only a Recommends of git, and checkout clones over https + run: apt-get update && apt-get -y --no-install-recommends install git ca-certificates + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + # noarch, so converted in the amd64 leg only, like the noarch rpms + - uses: actions/download-artifact@v8 + if: matrix.arch == 'amd64' + with: + pattern: rpms-osdeploy-* + path: alien + merge-multiple: true + - uses: actions/download-artifact@v8 + if: matrix.arch == 'amd64' + with: + name: rpms-ipxe + path: alien + - uses: actions/download-artifact@v8 + with: + name: rpms-go-${{ matrix.rpmarch }} + path: alien + - name: Build debs + run: bash .github/scripts/build-debs.sh ${{ matrix.arch == 'arm64' && '--arch-only' || '' }} debout alien + - uses: actions/upload-artifact@v7 + with: + name: debs-${{ matrix.release }}-${{ matrix.arch }} + path: debout/*.deb + retention-days: 3 + if-no-files-found: error + + bundle: + name: assemble bundles + needs: [prepare, rpms, osdeploy, ipxe, go, debs] + runs-on: ubuntu-26.04 + steps: + - uses: actions/checkout@v7 + - name: Install tools + run: sudo apt-get update && sudo apt-get -y --no-install-recommends install createrepo-c dpkg-dev + - uses: actions/download-artifact@v8 + with: + path: art + - name: Compose bundles + env: + VERSION: ${{ needs.prepare.outputs.version }} + run: | + set -e + pick() { find "$@" -name '*.rpm' ! -name '*.src.rpm'; } + for dist in el9 el10 suse16; do + for arch in x86_64 aarch64; do + mapfile -t files < <( + pick "art/rpms-$dist-x86_64" -name '*.noarch.rpm' + pick "art/rpms-$dist-$arch" -name "*.$arch.rpm" + pick art/rpms-osdeploy-* art/rpms-ipxe "art/rpms-go-$arch") + bash .github/scripts/mkbundle.sh "$dist-$arch" "$VERSION" bundles "${files[@]}" + done + done + # Architecture: all debs come from the amd64 leg only + pickdeb() { # + for f in "$1"/*.deb; do + [ "$(dpkg-deb -f "$f" Architecture)" = "$2" ] && echo "$f" + done + return 0 + } + for release in noble resolute; do + for arch in amd64 arm64; do + mapfile -t files < <( + pickdeb "art/debs-$release-amd64" all + pickdeb "art/debs-$release-$arch" "$arch") + bash .github/scripts/mkbundle.sh "$release-$arch" "$VERSION" bundles "${files[@]}" + done + done + - name: Write run summary + env: + VERSION: ${{ needs.prepare.outputs.version }} + run: | + { + echo "## confluent nightly $VERSION" + echo + for b in bundles/*.tar.gz; do + echo "
$(basename "$b")" + echo + tar tzf "$b" | sed -n 's|^[^/]*/\([^/]*\.\(rpm\|deb\)\)$|- \1|p' | sort + echo + echo "
" + done + } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v7 + with: + name: bundles + path: bundles/*.tar.gz + retention-days: 7 + if-no-files-found: error + + smoketest: + name: smoketest (${{ matrix.target }}) + needs: bundle + runs-on: ${{ matrix.runner }} + container: ${{ matrix.image }} + strategy: + fail-fast: false + matrix: + include: + - { target: el9-x86_64, image: "almalinux:9", runner: ubuntu-26.04 } + - { target: el9-aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm } + - { target: el10-x86_64, image: "almalinux:10", runner: ubuntu-26.04 } + - { target: el10-aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm } + - { target: suse16-x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 } + - { target: suse16-aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm } + - { target: noble-amd64, image: "ubuntu:noble", runner: ubuntu-26.04 } + - { target: noble-arm64, image: "ubuntu:noble", runner: ubuntu-26.04-arm } + - { target: resolute-amd64, image: "ubuntu:resolute", runner: ubuntu-26.04 } + - { target: resolute-arm64, image: "ubuntu:resolute", runner: ubuntu-26.04-arm } + steps: + - name: Install tar + run: | + if command -v zypper > /dev/null; then + zypper -n install --no-recommends tar gzip + elif command -v dnf > /dev/null; then + dnf -y --setopt=install_weak_deps=False install tar gzip + fi + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: bundles + path: bundles + - name: Install and smoke test + run: | + tar xzf bundles/confluent-nightly-${{ matrix.target }}.tar.gz + bash .github/scripts/smoketest.sh confluent-nightly-${{ matrix.target }} + + publish: + name: publish nightly release + needs: [prepare, smoketest] + if: >- + needs.prepare.outputs.build == 'true' && + (github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && inputs.publish)) + runs-on: ubuntu-26.04 + permissions: + contents: write + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + VERSION: ${{ needs.prepare.outputs.version }} + KEY: ${{ needs.prepare.outputs.key }} + BROWSERFS: ${{ needs.prepare.outputs.browserfs }} + VINZ: ${{ needs.prepare.outputs.vinz }} + steps: + - uses: actions/download-artifact@v8 + with: + name: bundles + path: bundles + - name: Write release notes + run: | + url=https://github.com/$GITHUB_REPOSITORY + cat > notes.md <> notes.md + done + cat >> notes.md < + EOF + cat notes.md + - name: Publish + # Move the tag and replace the assets in place, so the release and its + # download URLs never disappear between nights. + run: | + set -e + if gh release view nightly > /dev/null 2>&1; then + gh api -X PATCH "repos/$GH_REPO/git/refs/tags/nightly" -f sha="$GITHUB_SHA" -F force=true > /dev/null + gh release edit nightly --title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false + for old in $(gh release view nightly --json assets -q '.assets[].name'); do + [ -f "bundles/$old" ] || gh release delete-asset nightly "$old" -y + done + gh release upload nightly bundles/*.tar.gz --clobber + else + # a stale tag without a release would keep pointing at its old commit + gh api -X DELETE "repos/$GH_REPO/git/refs/tags/nightly" > /dev/null 2>&1 || true + gh release create nightly bundles/*.tar.gz --target "$GITHUB_SHA" \ + --title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4243ed10..a71c9a3c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,8 @@ on: push: tags: - '*' + # the rolling pre-release nightly.yml maintains + - '!nightly' permissions: contents: write diff --git a/README.md b/README.md index 27d110b0..f07d297e 100644 --- a/README.md +++ b/README.md @@ -48,3 +48,11 @@ Cutting a new series: wrong series. Patch releases land on branch `X.Y` and are tagged `X.Y.Z` there; no `VERSION` edit is needed. + +## Nightly builds + +`.github/workflows/nightly.yml` builds the rpms (EL9, EL10, SUSE 16) and debs (Ubuntu 24.04, 26.04) +for x86_64 and aarch64 every night that something changed, installs each set on a clean container, +and publishes them as the `nightly` pre-release. Every bundle there holds all packages for one +target and an `INSTALL.txt`. Pull requests that touch the packaging run the same build and tests +without publishing. Genesis is not built there yet. From bb7563ec5129d818198cf2009928f671e3e4dbfa Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 15 Oct 2026 19:41:41 +0200 Subject: [PATCH 2/2] Add status badges to the README Replace the static license badge with a live one, and add badges for CI, the nightly build, the latest release and the documentation. The docs badge carries the Octicons book icon (MIT, GitHub Inc.) inline, as Simple Icons has no generic one. --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f07d297e..162c85e8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ # Confluent -![Python 3](https://img.shields.io/badge/python-3-blue.svg) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/xcat2/confluent/blob/master/LICENSE) +[![CI](https://github.com/xcat2/confluent/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/xcat2/confluent/actions/workflows/ci.yml?query=branch%3Amaster) +[![Nightly](https://github.com/xcat2/confluent/actions/workflows/nightly.yml/badge.svg?branch=master)](https://github.com/xcat2/confluent/actions/workflows/nightly.yml?query=branch%3Amaster) +[![Release](https://img.shields.io/github/v/release/xcat2/confluent?sort=semver)](https://github.com/xcat2/confluent/releases/latest) +[![Docs](https://img.shields.io/badge/docs-confluent--docs-blueviolet?logo=data:image/svg%2Bxml;base64,PHN2ZyBmaWxsPSJ3aGl0ZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB3aWR0aD0iMTYiIGhlaWdodD0iMTYiIHZpZXdCb3g9IjAgMCAxNiAxNiI%2BPHBhdGggZD0iTTAgMS43NUEuNzUuNzUgMCAwIDEgLjc1IDFoNC4yNTNjMS4yMjcgMCAyLjMxNy41OSAzIDEuNTAxQTMuNzQzIDMuNzQzIDAgMCAxIDExLjAwNiAxaDQuMjQ1YS43NS43NSAwIDAgMSAuNzUuNzV2MTAuNWEuNzUuNzUgMCAwIDEtLjc1Ljc1aC00LjUwN2EyLjI1IDIuMjUgMCAwIDAtMS41OTEuNjU5bC0uNjIyLjYyMWEuNzUuNzUgMCAwIDEtMS4wNiAwbC0uNjIyLS42MjFBMi4yNSAyLjI1IDAgMCAwIDUuMjU4IDEzSC43NWEuNzUuNzUgMCAwIDEtLjc1LS43NVptNy4yNTEgMTAuMzI0LjAwNC01LjA3My0uMDAyLTIuMjUzQTIuMjUgMi4yNSAwIDAgMCA1LjAwMyAyLjVIMS41djloMy43NTdhMy43NSAzLjc1IDAgMCAxIDEuOTk0LjU3NFpNOC43NTUgNC43NWwtLjAwNCA3LjMyMmEzLjc1MiAzLjc1MiAwIDAgMSAxLjk5Mi0uNTcySDE0LjV2LTloLTMuNDk1YTIuMjUgMi4yNSAwIDAgMC0yLjI1IDIuMjVaIi8%2BPC9zdmc%2B)](https://xcat2.github.io/confluent-docs/) +![Python 3](https://img.shields.io/badge/python-3-3776ab?logo=python&logoColor=white) +[![License](https://img.shields.io/github/license/xcat2/confluent)](LICENSE) Confluent is a software package to handle essential bootstrap and operation of scale-out server configurations. It supports stateful and stateless deployments for various operating systems.