diff --git a/.github/scripts/build-debs.sh b/.github/scripts/build-debs.sh new file mode 100755 index 00000000..efe8d1cf --- /dev/null +++ b/.github/scripts/build-debs.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Build the confluent debs in an Ubuntu container, like build/arm/*/buildapt.sh: +# every component with a builddeb script, plus alien conversions of the rpms +# that have no deb packaging of their own (osdeploy, ipxe, browserfs, vinz). +# With --arch-only, only the architecture-specific ones: confluent_vtbufferd +# and the conversions of rpms that are not noarch. +# +# Usage: build-debs.sh [--arch-only] [] +set -e + +ARCHONLY= +if [ "$1" = "--arch-only" ]; then + ARCHONLY=1 + shift +fi +OUT=$(readlink -f "$1") +RPMS=${2:+$(readlink -f "$2")} +[ -n "$OUT" ] || { echo "usage: $0 [--arch-only] [rpmdir]" >&2; exit 1; } + +cd "$(dirname "$0")/../.." +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get -y --no-install-recommends install git build-essential debhelper devscripts dh-python fakeroot \ + python3-all python3-setuptools python3-stdeb pandoc patch alien +# stdeb on noble still imports the SafeConfigParser python 3.12 removed +if (cd / && patch -p1 --dry-run -s < "$OLDPWD/build/arm/noble/stdeb.patch") > /dev/null 2>&1; then + (cd / && patch -p1 < "$OLDPWD/build/arm/noble/stdeb.patch") +fi +git config --global --add safe.directory '*' +./mkversion + +mkdir -p "$OUT" +# vtbufferd first: it recreates /tmp/confluent, which client and server reuse +./confluent_vtbufferd/builddeb "$OUT" +if [ -z "$ARCHONLY" ]; then + ./confluent_client/builddeb "$OUT" + ./confluent_server/builddeb "$OUT" + ./imgutil/builddeb "$OUT" +fi + +if [ -n "$RPMS" ]; then + skip=(! -name '*.src.rpm') + [ -n "$ARCHONLY" ] && skip+=(! -name '*.noarch.rpm') + cd "$OUT" + find "$RPMS" -name '*.rpm' "${skip[@]}" -print0 | + xargs -0 -r -n1 fakeroot alien --to-deb --scripts --keep-version +fi +ls -la "$OUT" diff --git a/.github/scripts/build-go-rpm.sh b/.github/scripts/build-go-rpm.sh new file mode 100755 index 00000000..bdf88627 --- /dev/null +++ b/.github/scripts/build-go-rpm.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Build the rpm of one of the Go helpers confluent runs but keeps in separate +# repositories (browserfs, vinz), for the native architecture only. +# +# Mirrors the buildrpm script in those repositories, which needs git tags for +# its version. They carry none yet, so fall back to 0~dev+g, +# which sorts below any tagged release. +# +# Usage: build-go-rpm.sh [ref] +set -e + +NAME=$1 +URL=$2 +REF=${3:-HEAD} +WORK=$(mktemp -d) + +git clone -q "$URL" "$WORK/$NAME" +cd "$WORK/$NAME" +git checkout -q "$REF" +if desc=$(git describe 2>/dev/null); then + VERSION=${desc%%-*} + NUMCOMMITS=$(echo "$desc" | cut -d- -f2) + if [ "$NUMCOMMITS" != "$VERSION" ]; then + VERSION=$(echo "$VERSION" | awk -F. -v OFS=. '{$NF=$NF+1; print}') + VERSION=$VERSION~dev$NUMCOMMITS+$(echo "$desc" | cut -d- -f3) + fi +else + VERSION=0~dev$(git rev-list --count HEAD)+g$(git rev-parse --short HEAD) +fi +echo "$NAME $VERSION ($(git rev-parse HEAD))" + +mkdir -p ~/rpmbuild/SOURCES +(cd .. && tar cf ~/rpmbuild/SOURCES/confluent-$NAME.tar --exclude=.git "$NAME") +sed -e "s/#VERSION#/$VERSION/" $NAME.spec.tmpl > $NAME.spec +rpmbuild -ba $NAME.spec diff --git a/.github/scripts/build-ipxe.sh b/.github/scripts/build-ipxe.sh new file mode 100755 index 00000000..0d795669 --- /dev/null +++ b/.github/scripts/build-ipxe.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Build the confluent_ipxe rpm. The spec repackages the prebuilt boot roms +# that iPXE publishes with each release; rpmbuild does not fetch remote +# sources itself, so download the tarball of the release the spec names. +set -e + +cd "$(dirname "$0")/../.." +SPEC=ipxe/confluent_ipxe.spec +VERSION=$(rpmspec -q --srpm --qf '%{version}' $SPEC) +mkdir -p ~/rpmbuild/SOURCES +curl -fsSL -o ~/rpmbuild/SOURCES/ipxeboot.tar.gz \ + "https://github.com/ipxe/ipxe/releases/download/v$VERSION/ipxeboot.tar.gz" +rpmbuild -ba $SPEC diff --git a/.github/scripts/build-osdeploy.sh b/.github/scripts/build-osdeploy.sh new file mode 100755 index 00000000..c71efb1d --- /dev/null +++ b/.github/scripts/build-osdeploy.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Build the confluent_osdeploy rpm the way confluent_osdeploy/buildrpm and +# buildrpm-aarch64 do, but from utils binaries built by earlier CI jobs +# (build-utils.sh) instead of compiling them in local podman containers. +# +# Usage: build-osdeploy.sh +set -e + +ARCH=$1 +EL8UTILS=$(readlink -f "$2") +EL9UTILS=$(readlink -f "$3") + +cd "$(dirname "$0")/../.." +VERSION=$(./mkversion --tilde) + +case "$ARCH" in +x86_64) SPEC=confluent_osdeploy.spec ;; +aarch64) SPEC=confluent_osdeploy-aarch64.spec ;; +*) echo "unknown arch $ARCH" >&2; exit 1 ;; +esac +sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy/$SPEC.tmpl > confluent_osdeploy/$SPEC + +mkdir -p ~/rpmbuild/SOURCES +cp LICENSE confluent_osdeploy/ +tar Jcf ~/rpmbuild/SOURCES/confluent_osdeploy.tar.xz confluent_osdeploy + +for gen in el8 el9; do + utils=$EL8UTILS + [ "$gen" = el9 ] && utils=$EL9UTILS + rm -rf ${gen}bin + mkdir -p ${gen}bin/opt/confluent/bin ${gen}bin/stateless-bin + (cd "$utils" && cp confluent_imginfo copernicus clortho autocons "$OLDPWD"/${gen}bin/opt/confluent/bin/) + (cd "$utils" && cp start_root urlmount "$OLDPWD"/${gen}bin/stateless-bin/) + # artifact downloads lose the execute bit + chmod 755 ${gen}bin/opt/confluent/bin/* ${gen}bin/stateless-bin/* + tar Jcf ~/rpmbuild/SOURCES/confluent_${gen}bin.tar.xz ${gen}bin/ + rm -rf ${gen}bin +done + +rpmbuild -ba confluent_osdeploy/$SPEC diff --git a/.github/scripts/build-rpms.sh b/.github/scripts/build-rpms.sh new file mode 100755 index 00000000..92981385 --- /dev/null +++ b/.github/scripts/build-rpms.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Build the confluent rpms whose spec depends on the target distribution: +# confluent_vtbufferd (arch-specific) and, unless --arch-only is given, the +# noarch client, server and imgutil. Runs inside an EL or openSUSE container. +# +# Usage: build-rpms.sh [--arch-only] +set -e + +cd "$(dirname "$0")/../.." + +if command -v zypper > /dev/null; then + zypper -n install --no-recommends rpm-build gcc make git-core tar xz pandoc systemd python3 python3-setuptools + # the buildrpm scripts stage into ~/rpmbuild, SUSE defaults to /usr/src/packages + echo "%_topdir $HOME/rpmbuild" >> ~/.rpmmacros +else + dnf -y --setopt=install_weak_deps=False install epel-release + dnf -y --setopt=install_weak_deps=False install rpm-build gcc make git-core tar xz pandoc systemd python3 python3-setuptools +fi +git config --global --add safe.directory "$PWD" +./mkversion +mkdir -p ~/rpmbuild/SOURCES ~/rpmbuild/SPECS + +# the buildrpm scripts hide rpmbuild's output, so on failure rerun the spec +# they wrote to show why +build() { + "$1/buildrpm" && return 0 + spec=~/rpmbuild/SPECS/$(basename "$1").spec + [ -f "$spec" ] && rpmbuild -ba "$spec" + return 1 +} + +build confluent_vtbufferd +if [ "$1" != "--arch-only" ]; then + build confluent_client + build confluent_server + # imgutil/buildrpm calls rpmbuild directly, so its output is visible + ./imgutil/buildrpm +fi diff --git a/.github/scripts/build-utils.sh b/.github/scripts/build-utils.sh new file mode 100755 index 00000000..4bbac8f8 --- /dev/null +++ b/.github/scripts/build-utils.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Build the confluent_osdeploy utils inside a distro container, standing in for +# the el7build/el8builder/el9build podman images confluent_osdeploy/buildrpm +# uses. The el8bin set is linked against the oldest glibc any target runs +# (el7 on x86_64, el8 on aarch64), so it must be built on that distro. +# +# Usage: build-utils.sh (run from the repository root) +set -e + +OUT=$1 +[ -n "$OUT" ] || { echo "usage: $0 " >&2; exit 1; } + +. /etc/os-release +case "$VERSION_ID" in +7*) + # CentOS 7 is EOL: its mirrors are gone, only the vault remains + sed -i -e 's/^mirrorlist=/#mirrorlist=/' \ + -e 's|^#\?baseurl=http://mirror.centos.org|baseurl=https://vault.centos.org|' \ + /etc/yum.repos.d/CentOS-*.repo + yum -y install gcc make binutils libcurl-devel fuse-devel + ;; +8*) + dnf -y --setopt=install_weak_deps=False install dnf-plugins-core + dnf config-manager --set-enabled powertools + dnf -y --setopt=install_weak_deps=False install gcc make binutils libcurl-devel fuse-devel libxcrypt-devel + ;; +*) + dnf -y --setopt=install_weak_deps=False install dnf-plugins-core + dnf config-manager --set-enabled crb + dnf -y --setopt=install_weak_deps=False install gcc make binutils libcurl-devel fuse-devel libxcrypt-devel + ;; +esac + +make -C confluent_osdeploy/utils +mkdir -p "$OUT" +cd confluent_osdeploy/utils +cp confluent_imginfo copernicus clortho autocons start_root urlmount "$OUT"/ +make clean diff --git a/.github/scripts/mkbundle.sh b/.github/scripts/mkbundle.sh new file mode 100755 index 00000000..964afc44 --- /dev/null +++ b/.github/scripts/mkbundle.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Assemble the nightly bundle for one target: the packages, repository +# metadata so the directory also works as a dnf/zypper/apt repository, and +# an INSTALL.txt. Produces confluent-nightly-.tar.gz in . +# +# Usage: mkbundle.sh ... +# target is - +set -e + +TARGET=$1 +VERSION=$2 +OUTDIR=$(readlink -f "$3") +shift 3 + +NAME=confluent-nightly-$TARGET +DIR=$OUTDIR/$NAME +REPO=${GITHUB_REPOSITORY:-xcat2/confluent} +SHA=${GITHUB_SHA:-$(git rev-parse HEAD)} +rm -rf "$DIR" +mkdir -p "$DIR" +cp "$@" "$DIR"/ +echo "$VERSION" > "$DIR"/VERSION + +cat > "$DIR"/INSTALL.txt <> "$DIR"/INSTALL.txt <> "$DIR"/INSTALL.txt < /dev/null | gzip -9 > Packages.gz) + cat >> "$DIR"/INSTALL.txt < /etc/apt/sources.list.d/confluent-nightly.list + apt update && apt install confluent-server +EOF + ;; +*) + echo "unknown target $TARGET" >&2 + exit 1 + ;; +esac + +cat >> "$DIR"/INSTALL.txt < +set -x + +PKGDIR=$(readlink -f "$1") +FAIL=0 +. /etc/os-release + +case "$ID" in +almalinux|rocky|rhel|centos) + dnf -y install epel-release + dnf config-manager --set-enabled crb + dnf -y install procps-ng util-linux + echo "::group::install packages" + dnf -y install "$PKGDIR"/*.rpm || FAIL=1 + echo "::endgroup::" + ;; +opensuse-leap|sles) + zypper -n install procps util-linux + echo "::group::install packages" + # nightly rpms are unsigned + zypper -n --no-gpg-checks install --allow-unsigned-rpm "$PKGDIR"/*.rpm || FAIL=1 + echo "::endgroup::" + ;; +ubuntu|debian) + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get -y install procps util-linux + echo "::group::install packages" + apt-get -y install "$PKGDIR"/*.deb || FAIL=1 + echo "::endgroup::" + ;; +*) + echo "unsupported distribution $ID" >&2 + exit 1 + ;; +esac + +echo "::group::installed content" +test -d /opt/confluent/lib/osdeploy/el10/profiles || FAIL=1 +test -f /opt/confluent/lib/ipxe/ipxe.efi || FAIL=1 +test -x /opt/confluent/bin/browserfs || FAIL=1 +test -x /opt/confluent/bin/vinz || FAIL=1 +test -x /opt/confluent/bin/vtbufferd || FAIL=1 +test -x /opt/confluent/bin/imgutil || FAIL=1 +echo "::endgroup::" + +echo "::group::import and daemon smoke test" +export PYTHONPATH=/opt/confluent/lib/python +python3 -c "import confluent.main; print('IMPORT confluent.main: OK')" || FAIL=1 +python3 -c "import confluent.snmputil; print('IMPORT confluent.snmputil: OK')" || FAIL=1 +python3 -c "import aiohmi.redfish.command; print('IMPORT aiohmi.redfish: OK')" || FAIL=1 +# packaging sets up non-root operation (/etc/confluent owned by confluent), +# and there is no systemd in the container to start it, so run it directly +mkdir -p /var/run/confluent +chown confluent:confluent /var/run/confluent +timeout 60 runuser -u confluent -- /opt/confluent/bin/confluent -f > /tmp/confluentd.log 2>&1 & +for _ in $(seq 30); do + [ -S /var/run/confluent/api.sock ] && break + sleep 1 +done +if [ -S /var/run/confluent/api.sock ] && /opt/confluent/bin/confetty show /; then + echo "DAEMON_RUNNING=yes" +else + echo "DAEMON_RUNNING=no" + FAIL=1 +fi +cat /tmp/confluentd.log +pkill -f /opt/confluent/bin/confluent +echo "::endgroup::" + +exit $FAIL diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000..edf6fc21 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,458 @@ +name: Nightly + +# Builds the confluent rpms (EL9, EL10, SUSE 16) and debs (Ubuntu noble, +# resolute) for x86_64 and aarch64, installs each bundle on a clean container, +# and publishes the bundles as the rolling "nightly" pre-release. Pull +# requests that touch the packaging build and test, but publish nothing. + +on: + schedule: + - cron: '23 2 * * *' + workflow_dispatch: + inputs: + publish: + description: Publish the result as the nightly pre-release + type: boolean + default: true + force: + description: Build even if nothing changed since the last nightly + type: boolean + default: false + pull_request: + paths: + - .github/workflows/nightly.yml + - .github/scripts/** + - '**/buildrpm' + - '**/buildrpm-aarch64' + - '**/builddeb' + - '**/makesetup' + - '**/setup.py.tmpl' + - '**/*.spec' + - '**/*.spec.tmpl' + - confluent_osdeploy/utils/** + - build/arm/** + - mkversion + - VERSION + +permissions: + contents: read + +concurrency: + group: nightly-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + BROWSERFS_REPO: https://github.com/xcat2/browserfs + VINZ_REPO: https://github.com/xcat2/vinz + +defaults: + run: + shell: bash + +jobs: + prepare: + name: prepare + runs-on: ubuntu-26.04 + outputs: + build: ${{ steps.check.outputs.build }} + version: ${{ steps.check.outputs.version }} + key: ${{ steps.check.outputs.key }} + browserfs: ${{ steps.check.outputs.browserfs }} + vinz: ${{ steps.check.outputs.vinz }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Version and change check + id: check + env: + GH_TOKEN: ${{ github.token }} + FORCE: ${{ inputs.force }} + # The release notes carry the commits of the last published build, so + # a scheduled run with nothing new to build stops here. + run: | + version=$(./mkversion --tilde) + browserfs=$(git ls-remote "$BROWSERFS_REPO" HEAD | cut -f1) + vinz=$(git ls-remote "$VINZ_REPO" HEAD | cut -f1) + key="confluent@$GITHUB_SHA browserfs@$browserfs vinz@$vinz" + # no release yet on the first run, and bash runs with pipefail here + last=$({ gh release view nightly --repo "$GITHUB_REPOSITORY" --json body -q .body 2>/dev/null || true; } | + sed -n 's/^$/\1/p') + build=true + if [ "$GITHUB_EVENT_NAME" != pull_request ] && [ "$FORCE" != true ] && [ "$key" = "$last" ]; then + echo "::notice::Nothing changed since the last nightly ($last)" + build=false + fi + { + echo "build=$build" + echo "version=$version" + echo "key=$key" + echo "browserfs=$browserfs" + echo "vinz=$vinz" + } | tee -a "$GITHUB_OUTPUT" + + utils: + name: osdeploy utils (${{ matrix.set }}-${{ matrix.arch }}) + needs: prepare + if: needs.prepare.outputs.build == 'true' + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + # The el8bin set must run on the oldest target userspace, so it is + # built where confluent_osdeploy/buildrpm builds it: el7 for x86_64, + # el8 for aarch64. CentOS 7 is too old for the node-based actions, + # so these run the image by hand instead of as a job container. + include: + - { set: el8, arch: x86_64, image: "centos:7", runner: ubuntu-26.04 } + - { set: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 } + - { set: el8, arch: aarch64, image: "almalinux:8", runner: ubuntu-26.04-arm } + - { set: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm } + steps: + - uses: actions/checkout@v7 + - name: Build utils + run: docker run --rm -v "$PWD:/src" -w /src ${{ matrix.image }} bash .github/scripts/build-utils.sh /src/utilsbin + - uses: actions/upload-artifact@v7 + with: + name: utils-${{ matrix.set }}-${{ matrix.arch }} + path: utilsbin/ + retention-days: 3 + + rpms: + name: RPMs (${{ matrix.dist }}-${{ matrix.arch }}) + needs: prepare + if: needs.prepare.outputs.build == 'true' + runs-on: ${{ matrix.runner }} + container: ${{ matrix.image }} + strategy: + fail-fast: false + matrix: + # noarch rpms are built in the x86_64 leg only, and shared by both + include: + - { dist: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 } + - { dist: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm } + - { dist: el10, arch: x86_64, image: "almalinux:10", runner: ubuntu-26.04 } + - { dist: el10, arch: aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm } + - { dist: suse16, arch: x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 } + - { dist: suse16, arch: aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm } + steps: + - name: Install git + # without git in the container, checkout downloads a tarball and + # mkversion has no history to describe + run: | + if command -v zypper > /dev/null; then + zypper -n install --no-recommends git-core tar gzip + else + dnf -y --setopt=install_weak_deps=False install git-core + fi + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Build RPMs + run: bash .github/scripts/build-rpms.sh ${{ matrix.arch == 'aarch64' && '--arch-only' || '' }} + - uses: actions/upload-artifact@v7 + with: + name: rpms-${{ matrix.dist }}-${{ matrix.arch }} + path: ~/rpmbuild/RPMS/ + retention-days: 3 + + osdeploy: + name: osdeploy RPM (${{ matrix.arch }}) + needs: utils + runs-on: ubuntu-26.04 + container: almalinux:9 + strategy: + fail-fast: false + matrix: + arch: [x86_64, aarch64] + steps: + - name: Install build dependencies + run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core tar xz cpio + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/download-artifact@v8 + with: + name: utils-el8-${{ matrix.arch }} + path: el8utils + - uses: actions/download-artifact@v8 + with: + name: utils-el9-${{ matrix.arch }} + path: el9utils + - name: Build osdeploy RPM + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + bash .github/scripts/build-osdeploy.sh ${{ matrix.arch }} el8utils el9utils + - uses: actions/upload-artifact@v7 + with: + name: rpms-osdeploy-${{ matrix.arch }} + path: ~/rpmbuild/RPMS/ + retention-days: 3 + + ipxe: + name: iPXE RPM + needs: prepare + if: needs.prepare.outputs.build == 'true' + runs-on: ubuntu-26.04 + container: almalinux:9 + steps: + - name: Install build dependencies + run: dnf -y --setopt=install_weak_deps=False install rpm-build tar gzip + - uses: actions/checkout@v7 + - name: Build iPXE RPM + run: bash .github/scripts/build-ipxe.sh + - uses: actions/upload-artifact@v7 + with: + name: rpms-ipxe + path: ~/rpmbuild/RPMS/ + retention-days: 3 + + go: + name: browserfs and vinz RPMs (${{ matrix.arch }}) + needs: prepare + if: needs.prepare.outputs.build == 'true' + runs-on: ${{ matrix.runner }} + container: almalinux:10 + strategy: + fail-fast: false + matrix: + # static (CGO_ENABLED=0) binaries, so one build serves every distro + include: + - { arch: x86_64, runner: ubuntu-26.04 } + - { arch: aarch64, runner: ubuntu-26.04-arm } + steps: + - name: Install build dependencies + run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core golang tar + - uses: actions/checkout@v7 + - name: Build RPMs + run: | + bash .github/scripts/build-go-rpm.sh browserfs "$BROWSERFS_REPO" ${{ needs.prepare.outputs.browserfs }} + bash .github/scripts/build-go-rpm.sh vinz "$VINZ_REPO" ${{ needs.prepare.outputs.vinz }} + - uses: actions/upload-artifact@v7 + with: + name: rpms-go-${{ matrix.arch }} + path: ~/rpmbuild/RPMS/ + retention-days: 3 + + debs: + name: debs (${{ matrix.release }}-${{ matrix.arch }}) + needs: [osdeploy, ipxe, go] + runs-on: ${{ matrix.runner }} + container: ubuntu:${{ matrix.release }} + strategy: + fail-fast: false + matrix: + release: [noble, resolute] + arch: [amd64, arm64] + include: + # alien only converts rpms for the architecture it runs on + - { arch: amd64, rpmarch: x86_64, runner: ubuntu-26.04 } + - { arch: arm64, rpmarch: aarch64, runner: ubuntu-26.04-arm } + steps: + - name: Install git + # ca-certificates is only a Recommends of git, and checkout clones over https + run: apt-get update && apt-get -y --no-install-recommends install git ca-certificates + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + # noarch, so converted in the amd64 leg only, like the noarch rpms + - uses: actions/download-artifact@v8 + if: matrix.arch == 'amd64' + with: + pattern: rpms-osdeploy-* + path: alien + merge-multiple: true + - uses: actions/download-artifact@v8 + if: matrix.arch == 'amd64' + with: + name: rpms-ipxe + path: alien + - uses: actions/download-artifact@v8 + with: + name: rpms-go-${{ matrix.rpmarch }} + path: alien + - name: Build debs + run: bash .github/scripts/build-debs.sh ${{ matrix.arch == 'arm64' && '--arch-only' || '' }} debout alien + - uses: actions/upload-artifact@v7 + with: + name: debs-${{ matrix.release }}-${{ matrix.arch }} + path: debout/*.deb + retention-days: 3 + if-no-files-found: error + + bundle: + name: assemble bundles + needs: [prepare, rpms, osdeploy, ipxe, go, debs] + runs-on: ubuntu-26.04 + steps: + - uses: actions/checkout@v7 + - name: Install tools + run: sudo apt-get update && sudo apt-get -y --no-install-recommends install createrepo-c dpkg-dev + - uses: actions/download-artifact@v8 + with: + path: art + - name: Compose bundles + env: + VERSION: ${{ needs.prepare.outputs.version }} + run: | + set -e + pick() { find "$@" -name '*.rpm' ! -name '*.src.rpm'; } + for dist in el9 el10 suse16; do + for arch in x86_64 aarch64; do + mapfile -t files < <( + pick "art/rpms-$dist-x86_64" -name '*.noarch.rpm' + pick "art/rpms-$dist-$arch" -name "*.$arch.rpm" + pick art/rpms-osdeploy-* art/rpms-ipxe "art/rpms-go-$arch") + bash .github/scripts/mkbundle.sh "$dist-$arch" "$VERSION" bundles "${files[@]}" + done + done + # Architecture: all debs come from the amd64 leg only + pickdeb() { # + for f in "$1"/*.deb; do + [ "$(dpkg-deb -f "$f" Architecture)" = "$2" ] && echo "$f" + done + return 0 + } + for release in noble resolute; do + for arch in amd64 arm64; do + mapfile -t files < <( + pickdeb "art/debs-$release-amd64" all + pickdeb "art/debs-$release-$arch" "$arch") + bash .github/scripts/mkbundle.sh "$release-$arch" "$VERSION" bundles "${files[@]}" + done + done + - name: Write run summary + env: + VERSION: ${{ needs.prepare.outputs.version }} + run: | + { + echo "## confluent nightly $VERSION" + echo + for b in bundles/*.tar.gz; do + echo "
$(basename "$b")" + echo + tar tzf "$b" | sed -n 's|^[^/]*/\([^/]*\.\(rpm\|deb\)\)$|- \1|p' | sort + echo + echo "
" + done + } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v7 + with: + name: bundles + path: bundles/*.tar.gz + retention-days: 7 + if-no-files-found: error + + smoketest: + name: smoketest (${{ matrix.target }}) + needs: bundle + runs-on: ${{ matrix.runner }} + container: ${{ matrix.image }} + strategy: + fail-fast: false + matrix: + include: + - { target: el9-x86_64, image: "almalinux:9", runner: ubuntu-26.04 } + - { target: el9-aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm } + - { target: el10-x86_64, image: "almalinux:10", runner: ubuntu-26.04 } + - { target: el10-aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm } + - { target: suse16-x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 } + - { target: suse16-aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm } + - { target: noble-amd64, image: "ubuntu:noble", runner: ubuntu-26.04 } + - { target: noble-arm64, image: "ubuntu:noble", runner: ubuntu-26.04-arm } + - { target: resolute-amd64, image: "ubuntu:resolute", runner: ubuntu-26.04 } + - { target: resolute-arm64, image: "ubuntu:resolute", runner: ubuntu-26.04-arm } + steps: + - name: Install tar + run: | + if command -v zypper > /dev/null; then + zypper -n install --no-recommends tar gzip + elif command -v dnf > /dev/null; then + dnf -y --setopt=install_weak_deps=False install tar gzip + fi + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: bundles + path: bundles + - name: Install and smoke test + run: | + tar xzf bundles/confluent-nightly-${{ matrix.target }}.tar.gz + bash .github/scripts/smoketest.sh confluent-nightly-${{ matrix.target }} + + publish: + name: publish nightly release + needs: [prepare, smoketest] + if: >- + needs.prepare.outputs.build == 'true' && + (github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && inputs.publish)) + runs-on: ubuntu-26.04 + permissions: + contents: write + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + VERSION: ${{ needs.prepare.outputs.version }} + KEY: ${{ needs.prepare.outputs.key }} + BROWSERFS: ${{ needs.prepare.outputs.browserfs }} + VINZ: ${{ needs.prepare.outputs.vinz }} + steps: + - uses: actions/download-artifact@v8 + with: + name: bundles + path: bundles + - name: Write release notes + run: | + url=https://github.com/$GITHUB_REPOSITORY + cat > notes.md <> notes.md + done + cat >> notes.md < + EOF + cat notes.md + - name: Publish + # Move the tag and replace the assets in place, so the release and its + # download URLs never disappear between nights. + run: | + set -e + if gh release view nightly > /dev/null 2>&1; then + gh api -X PATCH "repos/$GH_REPO/git/refs/tags/nightly" -f sha="$GITHUB_SHA" -F force=true > /dev/null + gh release edit nightly --title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false + for old in $(gh release view nightly --json assets -q '.assets[].name'); do + [ -f "bundles/$old" ] || gh release delete-asset nightly "$old" -y + done + gh release upload nightly bundles/*.tar.gz --clobber + else + # a stale tag without a release would keep pointing at its old commit + gh api -X DELETE "repos/$GH_REPO/git/refs/tags/nightly" > /dev/null 2>&1 || true + gh release create nightly bundles/*.tar.gz --target "$GITHUB_SHA" \ + --title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4243ed10..a71c9a3c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,8 @@ on: push: tags: - '*' + # the rolling pre-release nightly.yml maintains + - '!nightly' permissions: contents: write diff --git a/README.md b/README.md index 27d110b0..162c85e8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ # Confluent -![Python 3](https://img.shields.io/badge/python-3-blue.svg) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/xcat2/confluent/blob/master/LICENSE) +[![CI](https://github.com/xcat2/confluent/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/xcat2/confluent/actions/workflows/ci.yml?query=branch%3Amaster) +[![Nightly](https://github.com/xcat2/confluent/actions/workflows/nightly.yml/badge.svg?branch=master)](https://github.com/xcat2/confluent/actions/workflows/nightly.yml?query=branch%3Amaster) +[![Release](https://img.shields.io/github/v/release/xcat2/confluent?sort=semver)](https://github.com/xcat2/confluent/releases/latest) +[![Docs](https://img.shields.io/badge/docs-confluent--docs-blueviolet?logo=data:image/svg%2Bxml;base64,PHN2ZyBmaWxsPSJ3aGl0ZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB3aWR0aD0iMTYiIGhlaWdodD0iMTYiIHZpZXdCb3g9IjAgMCAxNiAxNiI%2BPHBhdGggZD0iTTAgMS43NUEuNzUuNzUgMCAwIDEgLjc1IDFoNC4yNTNjMS4yMjcgMCAyLjMxNy41OSAzIDEuNTAxQTMuNzQzIDMuNzQzIDAgMCAxIDExLjAwNiAxaDQuMjQ1YS43NS43NSAwIDAgMSAuNzUuNzV2MTAuNWEuNzUuNzUgMCAwIDEtLjc1Ljc1aC00LjUwN2EyLjI1IDIuMjUgMCAwIDAtMS41OTEuNjU5bC0uNjIyLjYyMWEuNzUuNzUgMCAwIDEtMS4wNiAwbC0uNjIyLS42MjFBMi4yNSAyLjI1IDAgMCAwIDUuMjU4IDEzSC43NWEuNzUuNzUgMCAwIDEtLjc1LS43NVptNy4yNTEgMTAuMzI0LjAwNC01LjA3My0uMDAyLTIuMjUzQTIuMjUgMi4yNSAwIDAgMCA1LjAwMyAyLjVIMS41djloMy43NTdhMy43NSAzLjc1IDAgMCAxIDEuOTk0LjU3NFpNOC43NTUgNC43NWwtLjAwNCA3LjMyMmEzLjc1MiAzLjc1MiAwIDAgMSAxLjk5Mi0uNTcySDE0LjV2LTloLTMuNDk1YTIuMjUgMi4yNSAwIDAgMC0yLjI1IDIuMjVaIi8%2BPC9zdmc%2B)](https://xcat2.github.io/confluent-docs/) +![Python 3](https://img.shields.io/badge/python-3-3776ab?logo=python&logoColor=white) +[![License](https://img.shields.io/github/license/xcat2/confluent)](LICENSE) Confluent is a software package to handle essential bootstrap and operation of scale-out server configurations. It supports stateful and stateless deployments for various operating systems. @@ -48,3 +53,11 @@ Cutting a new series: wrong series. Patch releases land on branch `X.Y` and are tagged `X.Y.Z` there; no `VERSION` edit is needed. + +## Nightly builds + +`.github/workflows/nightly.yml` builds the rpms (EL9, EL10, SUSE 16) and debs (Ubuntu 24.04, 26.04) +for x86_64 and aarch64 every night that something changed, installs each set on a clean container, +and publishes them as the `nightly` pre-release. Every bundle there holds all packages for one +target and an `INSTALL.txt`. Pull requests that touch the packaging run the same build and tests +without publishing. Genesis is not built there yet.