mirror of
https://github.com/xcat2/confluent.git
synced 2026-10-20 14:45:45 +00:00
Build and publish nightly packages
Build the rpms for EL9, EL10 and SUSE 16 and the debs for Ubuntu noble and resolute, each for x86_64 and aarch64, install every set on a clean container and publish them as a rolling nightly pre-release. Each release asset is one target's bundle: all of its packages, repository metadata and an INSTALL.txt. Besides the in-tree components this packages confluent_ipxe from the iPXE release roms and the browserfs and vinz helpers from their own repositories. The osdeploy utils are built where buildrpm builds them, el7 for the x86_64 el8bin set, so they still run on the oldest targets. Genesis is left out for now. Noarch packages are built once, in the x86_64 legs, and the aarch64 legs build only what is architecture specific. Build tools are installed without weak dependencies, while the smoke tests keep the distribution defaults, since they stand in for a user's install. Scheduled runs skip the build when neither confluent nor the helper repositories moved since the last published nightly. Pull requests that touch the packaging build and test without publishing.
This commit is contained in:
Markus Hilger
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/bin/bash
|
||||
# Build the confluent debs in an Ubuntu container, like build/arm/*/buildapt.sh:
|
||||
# every component with a builddeb script, plus alien conversions of the rpms
|
||||
# that have no deb packaging of their own (osdeploy, ipxe, browserfs, vinz).
|
||||
# With --arch-only, only the architecture-specific ones: confluent_vtbufferd
|
||||
# and the conversions of rpms that are not noarch.
|
||||
#
|
||||
# Usage: build-debs.sh [--arch-only] <outdir> [<dir with rpms to convert>]
|
||||
set -e
|
||||
|
||||
ARCHONLY=
|
||||
if [ "$1" = "--arch-only" ]; then
|
||||
ARCHONLY=1
|
||||
shift
|
||||
fi
|
||||
OUT=$(readlink -f "$1")
|
||||
RPMS=${2:+$(readlink -f "$2")}
|
||||
[ -n "$OUT" ] || { echo "usage: $0 [--arch-only] <outdir> [rpmdir]" >&2; exit 1; }
|
||||
|
||||
cd "$(dirname "$0")/../.."
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get -y --no-install-recommends install git build-essential debhelper devscripts dh-python fakeroot \
|
||||
python3-all python3-setuptools python3-stdeb pandoc patch alien
|
||||
# stdeb on noble still imports the SafeConfigParser python 3.12 removed
|
||||
if (cd / && patch -p1 --dry-run -s < "$OLDPWD/build/arm/noble/stdeb.patch") > /dev/null 2>&1; then
|
||||
(cd / && patch -p1 < "$OLDPWD/build/arm/noble/stdeb.patch")
|
||||
fi
|
||||
git config --global --add safe.directory '*'
|
||||
./mkversion
|
||||
|
||||
mkdir -p "$OUT"
|
||||
# vtbufferd first: it recreates /tmp/confluent, which client and server reuse
|
||||
./confluent_vtbufferd/builddeb "$OUT"
|
||||
if [ -z "$ARCHONLY" ]; then
|
||||
./confluent_client/builddeb "$OUT"
|
||||
./confluent_server/builddeb "$OUT"
|
||||
./imgutil/builddeb "$OUT"
|
||||
fi
|
||||
|
||||
if [ -n "$RPMS" ]; then
|
||||
skip=(! -name '*.src.rpm')
|
||||
[ -n "$ARCHONLY" ] && skip+=(! -name '*.noarch.rpm')
|
||||
cd "$OUT"
|
||||
find "$RPMS" -name '*.rpm' "${skip[@]}" -print0 |
|
||||
xargs -0 -r -n1 fakeroot alien --to-deb --scripts --keep-version
|
||||
fi
|
||||
ls -la "$OUT"
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
# Build the rpm of one of the Go helpers confluent runs but keeps in separate
|
||||
# repositories (browserfs, vinz), for the native architecture only.
|
||||
#
|
||||
# Mirrors the buildrpm script in those repositories, which needs git tags for
|
||||
# its version. They carry none yet, so fall back to 0~dev<commits>+g<hash>,
|
||||
# which sorts below any tagged release.
|
||||
#
|
||||
# Usage: build-go-rpm.sh <name> <git-url> [ref]
|
||||
set -e
|
||||
|
||||
NAME=$1
|
||||
URL=$2
|
||||
REF=${3:-HEAD}
|
||||
WORK=$(mktemp -d)
|
||||
|
||||
git clone -q "$URL" "$WORK/$NAME"
|
||||
cd "$WORK/$NAME"
|
||||
git checkout -q "$REF"
|
||||
if desc=$(git describe 2>/dev/null); then
|
||||
VERSION=${desc%%-*}
|
||||
NUMCOMMITS=$(echo "$desc" | cut -d- -f2)
|
||||
if [ "$NUMCOMMITS" != "$VERSION" ]; then
|
||||
VERSION=$(echo "$VERSION" | awk -F. -v OFS=. '{$NF=$NF+1; print}')
|
||||
VERSION=$VERSION~dev$NUMCOMMITS+$(echo "$desc" | cut -d- -f3)
|
||||
fi
|
||||
else
|
||||
VERSION=0~dev$(git rev-list --count HEAD)+g$(git rev-parse --short HEAD)
|
||||
fi
|
||||
echo "$NAME $VERSION ($(git rev-parse HEAD))"
|
||||
|
||||
mkdir -p ~/rpmbuild/SOURCES
|
||||
(cd .. && tar cf ~/rpmbuild/SOURCES/confluent-$NAME.tar --exclude=.git "$NAME")
|
||||
sed -e "s/#VERSION#/$VERSION/" $NAME.spec.tmpl > $NAME.spec
|
||||
rpmbuild -ba $NAME.spec
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
# Build the confluent_ipxe rpm. The spec repackages the prebuilt boot roms
|
||||
# that iPXE publishes with each release; rpmbuild does not fetch remote
|
||||
# sources itself, so download the tarball of the release the spec names.
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")/../.."
|
||||
SPEC=ipxe/confluent_ipxe.spec
|
||||
VERSION=$(rpmspec -q --srpm --qf '%{version}' $SPEC)
|
||||
mkdir -p ~/rpmbuild/SOURCES
|
||||
curl -fsSL -o ~/rpmbuild/SOURCES/ipxeboot.tar.gz \
|
||||
"https://github.com/ipxe/ipxe/releases/download/v$VERSION/ipxeboot.tar.gz"
|
||||
rpmbuild -ba $SPEC
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/bash
|
||||
# Build the confluent_osdeploy rpm the way confluent_osdeploy/buildrpm and
|
||||
# buildrpm-aarch64 do, but from utils binaries built by earlier CI jobs
|
||||
# (build-utils.sh) instead of compiling them in local podman containers.
|
||||
#
|
||||
# Usage: build-osdeploy.sh <x86_64|aarch64> <el8utils-dir> <el9utils-dir>
|
||||
set -e
|
||||
|
||||
ARCH=$1
|
||||
EL8UTILS=$(readlink -f "$2")
|
||||
EL9UTILS=$(readlink -f "$3")
|
||||
|
||||
cd "$(dirname "$0")/../.."
|
||||
VERSION=$(./mkversion --tilde)
|
||||
|
||||
case "$ARCH" in
|
||||
x86_64) SPEC=confluent_osdeploy.spec ;;
|
||||
aarch64) SPEC=confluent_osdeploy-aarch64.spec ;;
|
||||
*) echo "unknown arch $ARCH" >&2; exit 1 ;;
|
||||
esac
|
||||
sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy/$SPEC.tmpl > confluent_osdeploy/$SPEC
|
||||
|
||||
mkdir -p ~/rpmbuild/SOURCES
|
||||
cp LICENSE confluent_osdeploy/
|
||||
tar Jcf ~/rpmbuild/SOURCES/confluent_osdeploy.tar.xz confluent_osdeploy
|
||||
|
||||
for gen in el8 el9; do
|
||||
utils=$EL8UTILS
|
||||
[ "$gen" = el9 ] && utils=$EL9UTILS
|
||||
rm -rf ${gen}bin
|
||||
mkdir -p ${gen}bin/opt/confluent/bin ${gen}bin/stateless-bin
|
||||
(cd "$utils" && cp confluent_imginfo copernicus clortho autocons "$OLDPWD"/${gen}bin/opt/confluent/bin/)
|
||||
(cd "$utils" && cp start_root urlmount "$OLDPWD"/${gen}bin/stateless-bin/)
|
||||
# artifact downloads lose the execute bit
|
||||
chmod 755 ${gen}bin/opt/confluent/bin/* ${gen}bin/stateless-bin/*
|
||||
tar Jcf ~/rpmbuild/SOURCES/confluent_${gen}bin.tar.xz ${gen}bin/
|
||||
rm -rf ${gen}bin
|
||||
done
|
||||
|
||||
rpmbuild -ba confluent_osdeploy/$SPEC
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
# Build the confluent rpms whose spec depends on the target distribution:
|
||||
# confluent_vtbufferd (arch-specific) and, unless --arch-only is given, the
|
||||
# noarch client, server and imgutil. Runs inside an EL or openSUSE container.
|
||||
#
|
||||
# Usage: build-rpms.sh [--arch-only]
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")/../.."
|
||||
|
||||
if command -v zypper > /dev/null; then
|
||||
zypper -n install --no-recommends rpm-build gcc make git-core tar xz pandoc systemd python3 python3-setuptools
|
||||
# the buildrpm scripts stage into ~/rpmbuild, SUSE defaults to /usr/src/packages
|
||||
echo "%_topdir $HOME/rpmbuild" >> ~/.rpmmacros
|
||||
else
|
||||
dnf -y --setopt=install_weak_deps=False install epel-release
|
||||
dnf -y --setopt=install_weak_deps=False install rpm-build gcc make git-core tar xz pandoc systemd python3 python3-setuptools
|
||||
fi
|
||||
git config --global --add safe.directory "$PWD"
|
||||
./mkversion
|
||||
mkdir -p ~/rpmbuild/SOURCES ~/rpmbuild/SPECS
|
||||
|
||||
# the buildrpm scripts hide rpmbuild's output, so on failure rerun the spec
|
||||
# they wrote to show why
|
||||
build() {
|
||||
"$1/buildrpm" && return 0
|
||||
spec=~/rpmbuild/SPECS/$(basename "$1").spec
|
||||
[ -f "$spec" ] && rpmbuild -ba "$spec"
|
||||
return 1
|
||||
}
|
||||
|
||||
build confluent_vtbufferd
|
||||
if [ "$1" != "--arch-only" ]; then
|
||||
build confluent_client
|
||||
build confluent_server
|
||||
# imgutil/buildrpm calls rpmbuild directly, so its output is visible
|
||||
./imgutil/buildrpm
|
||||
fi
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
# Build the confluent_osdeploy utils inside a distro container, standing in for
|
||||
# the el7build/el8builder/el9build podman images confluent_osdeploy/buildrpm
|
||||
# uses. The el8bin set is linked against the oldest glibc any target runs
|
||||
# (el7 on x86_64, el8 on aarch64), so it must be built on that distro.
|
||||
#
|
||||
# Usage: build-utils.sh <outdir> (run from the repository root)
|
||||
set -e
|
||||
|
||||
OUT=$1
|
||||
[ -n "$OUT" ] || { echo "usage: $0 <outdir>" >&2; exit 1; }
|
||||
|
||||
. /etc/os-release
|
||||
case "$VERSION_ID" in
|
||||
7*)
|
||||
# CentOS 7 is EOL: its mirrors are gone, only the vault remains
|
||||
sed -i -e 's/^mirrorlist=/#mirrorlist=/' \
|
||||
-e 's|^#\?baseurl=http://mirror.centos.org|baseurl=https://vault.centos.org|' \
|
||||
/etc/yum.repos.d/CentOS-*.repo
|
||||
yum -y install gcc make binutils libcurl-devel fuse-devel
|
||||
;;
|
||||
8*)
|
||||
dnf -y --setopt=install_weak_deps=False install dnf-plugins-core
|
||||
dnf config-manager --set-enabled powertools
|
||||
dnf -y --setopt=install_weak_deps=False install gcc make binutils libcurl-devel fuse-devel libxcrypt-devel
|
||||
;;
|
||||
*)
|
||||
dnf -y --setopt=install_weak_deps=False install dnf-plugins-core
|
||||
dnf config-manager --set-enabled crb
|
||||
dnf -y --setopt=install_weak_deps=False install gcc make binutils libcurl-devel fuse-devel libxcrypt-devel
|
||||
;;
|
||||
esac
|
||||
|
||||
make -C confluent_osdeploy/utils
|
||||
mkdir -p "$OUT"
|
||||
cd confluent_osdeploy/utils
|
||||
cp confluent_imginfo copernicus clortho autocons start_root urlmount "$OUT"/
|
||||
make clean
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/bin/bash
|
||||
# Assemble the nightly bundle for one target: the packages, repository
|
||||
# metadata so the directory also works as a dnf/zypper/apt repository, and
|
||||
# an INSTALL.txt. Produces confluent-nightly-<target>.tar.gz in <outdir>.
|
||||
#
|
||||
# Usage: mkbundle.sh <target> <version> <outdir> <package file>...
|
||||
# target is <el9|el10|suse16|noble|resolute>-<arch>
|
||||
set -e
|
||||
|
||||
TARGET=$1
|
||||
VERSION=$2
|
||||
OUTDIR=$(readlink -f "$3")
|
||||
shift 3
|
||||
|
||||
NAME=confluent-nightly-$TARGET
|
||||
DIR=$OUTDIR/$NAME
|
||||
REPO=${GITHUB_REPOSITORY:-xcat2/confluent}
|
||||
SHA=${GITHUB_SHA:-$(git rev-parse HEAD)}
|
||||
rm -rf "$DIR"
|
||||
mkdir -p "$DIR"
|
||||
cp "$@" "$DIR"/
|
||||
echo "$VERSION" > "$DIR"/VERSION
|
||||
|
||||
cat > "$DIR"/INSTALL.txt <<EOF
|
||||
confluent nightly $VERSION ($TARGET)
|
||||
|
||||
Unofficial development build of https://github.com/$REPO
|
||||
commit $SHA. Packages are unsigned. Not meant for production.
|
||||
|
||||
EOF
|
||||
|
||||
case "$TARGET" in
|
||||
el*)
|
||||
createrepo_c -q "$DIR"
|
||||
cat >> "$DIR"/INSTALL.txt <<EOF
|
||||
Dependencies come from the distribution plus EPEL and CRB:
|
||||
|
||||
dnf install epel-release
|
||||
crb enable
|
||||
|
||||
Install everything:
|
||||
|
||||
dnf install ./$NAME/*.rpm
|
||||
|
||||
or only some of it, using the directory as a repository:
|
||||
|
||||
dnf install --repofrompath=confluent-nightly,\$PWD/$NAME \\
|
||||
--setopt=confluent-nightly.gpgcheck=0 confluent_server
|
||||
EOF
|
||||
;;
|
||||
suse*)
|
||||
createrepo_c -q "$DIR"
|
||||
cat >> "$DIR"/INSTALL.txt <<EOF
|
||||
Install everything:
|
||||
|
||||
zypper --no-gpg-checks install --allow-unsigned-rpm ./$NAME/*.rpm
|
||||
|
||||
or only some of it, using the directory as a repository:
|
||||
|
||||
zypper addrepo --no-gpgcheck \$PWD/$NAME confluent-nightly
|
||||
zypper install confluent_server
|
||||
EOF
|
||||
;;
|
||||
noble*|resolute*)
|
||||
(cd "$DIR" && dpkg-scanpackages --multiversion . /dev/null 2> /dev/null | gzip -9 > Packages.gz)
|
||||
cat >> "$DIR"/INSTALL.txt <<EOF
|
||||
Install everything:
|
||||
|
||||
apt install ./$NAME/*.deb
|
||||
|
||||
or only some of it, using the directory as a repository:
|
||||
|
||||
echo "deb [trusted=yes] file:\$PWD/$NAME ./" > /etc/apt/sources.list.d/confluent-nightly.list
|
||||
apt update && apt install confluent-server
|
||||
EOF
|
||||
;;
|
||||
*)
|
||||
echo "unknown target $TARGET" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
cat >> "$DIR"/INSTALL.txt <<EOF
|
||||
|
||||
The bundle carries the osdeploy support for both x86_64 and aarch64
|
||||
deployment targets. Genesis (confluent-genesis-*) is not built yet.
|
||||
EOF
|
||||
|
||||
tar -C "$OUTDIR" -czf "$OUTDIR/$NAME.tar.gz" "$NAME"
|
||||
rm -rf "$DIR"
|
||||
echo "$OUTDIR/$NAME.tar.gz"
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/bin/bash
|
||||
# Install a nightly bundle on a clean EL, openSUSE or Ubuntu container and
|
||||
# check that the server imports and starts.
|
||||
#
|
||||
# Every dependency has to resolve from the distribution's own repositories
|
||||
# (plus EPEL on EL), so a missing one fails the run.
|
||||
#
|
||||
# Usage: smoketest.sh <dir with the packages to install>
|
||||
set -x
|
||||
|
||||
PKGDIR=$(readlink -f "$1")
|
||||
FAIL=0
|
||||
. /etc/os-release
|
||||
|
||||
case "$ID" in
|
||||
almalinux|rocky|rhel|centos)
|
||||
dnf -y install epel-release
|
||||
dnf config-manager --set-enabled crb
|
||||
dnf -y install procps-ng util-linux
|
||||
echo "::group::install packages"
|
||||
dnf -y install "$PKGDIR"/*.rpm || FAIL=1
|
||||
echo "::endgroup::"
|
||||
;;
|
||||
opensuse-leap|sles)
|
||||
zypper -n install procps util-linux
|
||||
echo "::group::install packages"
|
||||
# nightly rpms are unsigned
|
||||
zypper -n --no-gpg-checks install --allow-unsigned-rpm "$PKGDIR"/*.rpm || FAIL=1
|
||||
echo "::endgroup::"
|
||||
;;
|
||||
ubuntu|debian)
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get -y install procps util-linux
|
||||
echo "::group::install packages"
|
||||
apt-get -y install "$PKGDIR"/*.deb || FAIL=1
|
||||
echo "::endgroup::"
|
||||
;;
|
||||
*)
|
||||
echo "unsupported distribution $ID" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "::group::installed content"
|
||||
test -d /opt/confluent/lib/osdeploy/el10/profiles || FAIL=1
|
||||
test -f /opt/confluent/lib/ipxe/ipxe.efi || FAIL=1
|
||||
test -x /opt/confluent/bin/browserfs || FAIL=1
|
||||
test -x /opt/confluent/bin/vinz || FAIL=1
|
||||
test -x /opt/confluent/bin/vtbufferd || FAIL=1
|
||||
test -x /opt/confluent/bin/imgutil || FAIL=1
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::import and daemon smoke test"
|
||||
export PYTHONPATH=/opt/confluent/lib/python
|
||||
python3 -c "import confluent.main; print('IMPORT confluent.main: OK')" || FAIL=1
|
||||
python3 -c "import confluent.snmputil; print('IMPORT confluent.snmputil: OK')" || FAIL=1
|
||||
python3 -c "import aiohmi.redfish.command; print('IMPORT aiohmi.redfish: OK')" || FAIL=1
|
||||
# packaging sets up non-root operation (/etc/confluent owned by confluent),
|
||||
# and there is no systemd in the container to start it, so run it directly
|
||||
mkdir -p /var/run/confluent
|
||||
chown confluent:confluent /var/run/confluent
|
||||
timeout 60 runuser -u confluent -- /opt/confluent/bin/confluent -f > /tmp/confluentd.log 2>&1 &
|
||||
for _ in $(seq 30); do
|
||||
[ -S /var/run/confluent/api.sock ] && break
|
||||
sleep 1
|
||||
done
|
||||
if [ -S /var/run/confluent/api.sock ] && /opt/confluent/bin/confetty show /; then
|
||||
echo "DAEMON_RUNNING=yes"
|
||||
else
|
||||
echo "DAEMON_RUNNING=no"
|
||||
FAIL=1
|
||||
fi
|
||||
cat /tmp/confluentd.log
|
||||
pkill -f /opt/confluent/bin/confluent
|
||||
echo "::endgroup::"
|
||||
|
||||
exit $FAIL
|
||||
@@ -0,0 +1,458 @@
|
||||
name: Nightly
|
||||
|
||||
# Builds the confluent rpms (EL9, EL10, SUSE 16) and debs (Ubuntu noble,
|
||||
# resolute) for x86_64 and aarch64, installs each bundle on a clean container,
|
||||
# and publishes the bundles as the rolling "nightly" pre-release. Pull
|
||||
# requests that touch the packaging build and test, but publish nothing.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '23 2 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
publish:
|
||||
description: Publish the result as the nightly pre-release
|
||||
type: boolean
|
||||
default: true
|
||||
force:
|
||||
description: Build even if nothing changed since the last nightly
|
||||
type: boolean
|
||||
default: false
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/nightly.yml
|
||||
- .github/scripts/**
|
||||
- '**/buildrpm'
|
||||
- '**/buildrpm-aarch64'
|
||||
- '**/builddeb'
|
||||
- '**/makesetup'
|
||||
- '**/setup.py.tmpl'
|
||||
- '**/*.spec'
|
||||
- '**/*.spec.tmpl'
|
||||
- confluent_osdeploy/utils/**
|
||||
- build/arm/**
|
||||
- mkversion
|
||||
- VERSION
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: nightly-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
env:
|
||||
BROWSERFS_REPO: https://github.com/xcat2/browserfs
|
||||
VINZ_REPO: https://github.com/xcat2/vinz
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
name: prepare
|
||||
runs-on: ubuntu-26.04
|
||||
outputs:
|
||||
build: ${{ steps.check.outputs.build }}
|
||||
version: ${{ steps.check.outputs.version }}
|
||||
key: ${{ steps.check.outputs.key }}
|
||||
browserfs: ${{ steps.check.outputs.browserfs }}
|
||||
vinz: ${{ steps.check.outputs.vinz }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Version and change check
|
||||
id: check
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
FORCE: ${{ inputs.force }}
|
||||
# The release notes carry the commits of the last published build, so
|
||||
# a scheduled run with nothing new to build stops here.
|
||||
run: |
|
||||
version=$(./mkversion --tilde)
|
||||
browserfs=$(git ls-remote "$BROWSERFS_REPO" HEAD | cut -f1)
|
||||
vinz=$(git ls-remote "$VINZ_REPO" HEAD | cut -f1)
|
||||
key="confluent@$GITHUB_SHA browserfs@$browserfs vinz@$vinz"
|
||||
# no release yet on the first run, and bash runs with pipefail here
|
||||
last=$({ gh release view nightly --repo "$GITHUB_REPOSITORY" --json body -q .body 2>/dev/null || true; } |
|
||||
sed -n 's/^<!-- build: \(.*\) -->$/\1/p')
|
||||
build=true
|
||||
if [ "$GITHUB_EVENT_NAME" != pull_request ] && [ "$FORCE" != true ] && [ "$key" = "$last" ]; then
|
||||
echo "::notice::Nothing changed since the last nightly ($last)"
|
||||
build=false
|
||||
fi
|
||||
{
|
||||
echo "build=$build"
|
||||
echo "version=$version"
|
||||
echo "key=$key"
|
||||
echo "browserfs=$browserfs"
|
||||
echo "vinz=$vinz"
|
||||
} | tee -a "$GITHUB_OUTPUT"
|
||||
|
||||
utils:
|
||||
name: osdeploy utils (${{ matrix.set }}-${{ matrix.arch }})
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build == 'true'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# The el8bin set must run on the oldest target userspace, so it is
|
||||
# built where confluent_osdeploy/buildrpm builds it: el7 for x86_64,
|
||||
# el8 for aarch64. CentOS 7 is too old for the node-based actions,
|
||||
# so these run the image by hand instead of as a job container.
|
||||
include:
|
||||
- { set: el8, arch: x86_64, image: "centos:7", runner: ubuntu-26.04 }
|
||||
- { set: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
|
||||
- { set: el8, arch: aarch64, image: "almalinux:8", runner: ubuntu-26.04-arm }
|
||||
- { set: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- name: Build utils
|
||||
run: docker run --rm -v "$PWD:/src" -w /src ${{ matrix.image }} bash .github/scripts/build-utils.sh /src/utilsbin
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: utils-${{ matrix.set }}-${{ matrix.arch }}
|
||||
path: utilsbin/
|
||||
retention-days: 3
|
||||
|
||||
rpms:
|
||||
name: RPMs (${{ matrix.dist }}-${{ matrix.arch }})
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build == 'true'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
container: ${{ matrix.image }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# noarch rpms are built in the x86_64 leg only, and shared by both
|
||||
include:
|
||||
- { dist: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
|
||||
- { dist: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
|
||||
- { dist: el10, arch: x86_64, image: "almalinux:10", runner: ubuntu-26.04 }
|
||||
- { dist: el10, arch: aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm }
|
||||
- { dist: suse16, arch: x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 }
|
||||
- { dist: suse16, arch: aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm }
|
||||
steps:
|
||||
- name: Install git
|
||||
# without git in the container, checkout downloads a tarball and
|
||||
# mkversion has no history to describe
|
||||
run: |
|
||||
if command -v zypper > /dev/null; then
|
||||
zypper -n install --no-recommends git-core tar gzip
|
||||
else
|
||||
dnf -y --setopt=install_weak_deps=False install git-core
|
||||
fi
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build RPMs
|
||||
run: bash .github/scripts/build-rpms.sh ${{ matrix.arch == 'aarch64' && '--arch-only' || '' }}
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rpms-${{ matrix.dist }}-${{ matrix.arch }}
|
||||
path: ~/rpmbuild/RPMS/
|
||||
retention-days: 3
|
||||
|
||||
osdeploy:
|
||||
name: osdeploy RPM (${{ matrix.arch }})
|
||||
needs: utils
|
||||
runs-on: ubuntu-26.04
|
||||
container: almalinux:9
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
arch: [x86_64, aarch64]
|
||||
steps:
|
||||
- name: Install build dependencies
|
||||
run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core tar xz cpio
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: utils-el8-${{ matrix.arch }}
|
||||
path: el8utils
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: utils-el9-${{ matrix.arch }}
|
||||
path: el9utils
|
||||
- name: Build osdeploy RPM
|
||||
run: |
|
||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
bash .github/scripts/build-osdeploy.sh ${{ matrix.arch }} el8utils el9utils
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rpms-osdeploy-${{ matrix.arch }}
|
||||
path: ~/rpmbuild/RPMS/
|
||||
retention-days: 3
|
||||
|
||||
ipxe:
|
||||
name: iPXE RPM
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build == 'true'
|
||||
runs-on: ubuntu-26.04
|
||||
container: almalinux:9
|
||||
steps:
|
||||
- name: Install build dependencies
|
||||
run: dnf -y --setopt=install_weak_deps=False install rpm-build tar gzip
|
||||
- uses: actions/checkout@v7
|
||||
- name: Build iPXE RPM
|
||||
run: bash .github/scripts/build-ipxe.sh
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rpms-ipxe
|
||||
path: ~/rpmbuild/RPMS/
|
||||
retention-days: 3
|
||||
|
||||
go:
|
||||
name: browserfs and vinz RPMs (${{ matrix.arch }})
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build == 'true'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
container: almalinux:10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# static (CGO_ENABLED=0) binaries, so one build serves every distro
|
||||
include:
|
||||
- { arch: x86_64, runner: ubuntu-26.04 }
|
||||
- { arch: aarch64, runner: ubuntu-26.04-arm }
|
||||
steps:
|
||||
- name: Install build dependencies
|
||||
run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core golang tar
|
||||
- uses: actions/checkout@v7
|
||||
- name: Build RPMs
|
||||
run: |
|
||||
bash .github/scripts/build-go-rpm.sh browserfs "$BROWSERFS_REPO" ${{ needs.prepare.outputs.browserfs }}
|
||||
bash .github/scripts/build-go-rpm.sh vinz "$VINZ_REPO" ${{ needs.prepare.outputs.vinz }}
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rpms-go-${{ matrix.arch }}
|
||||
path: ~/rpmbuild/RPMS/
|
||||
retention-days: 3
|
||||
|
||||
debs:
|
||||
name: debs (${{ matrix.release }}-${{ matrix.arch }})
|
||||
needs: [osdeploy, ipxe, go]
|
||||
runs-on: ${{ matrix.runner }}
|
||||
container: ubuntu:${{ matrix.release }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
release: [noble, resolute]
|
||||
arch: [amd64, arm64]
|
||||
include:
|
||||
# alien only converts rpms for the architecture it runs on
|
||||
- { arch: amd64, rpmarch: x86_64, runner: ubuntu-26.04 }
|
||||
- { arch: arm64, rpmarch: aarch64, runner: ubuntu-26.04-arm }
|
||||
steps:
|
||||
- name: Install git
|
||||
# ca-certificates is only a Recommends of git, and checkout clones over https
|
||||
run: apt-get update && apt-get -y --no-install-recommends install git ca-certificates
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
# noarch, so converted in the amd64 leg only, like the noarch rpms
|
||||
- uses: actions/download-artifact@v8
|
||||
if: matrix.arch == 'amd64'
|
||||
with:
|
||||
pattern: rpms-osdeploy-*
|
||||
path: alien
|
||||
merge-multiple: true
|
||||
- uses: actions/download-artifact@v8
|
||||
if: matrix.arch == 'amd64'
|
||||
with:
|
||||
name: rpms-ipxe
|
||||
path: alien
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: rpms-go-${{ matrix.rpmarch }}
|
||||
path: alien
|
||||
- name: Build debs
|
||||
run: bash .github/scripts/build-debs.sh ${{ matrix.arch == 'arm64' && '--arch-only' || '' }} debout alien
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: debs-${{ matrix.release }}-${{ matrix.arch }}
|
||||
path: debout/*.deb
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
bundle:
|
||||
name: assemble bundles
|
||||
needs: [prepare, rpms, osdeploy, ipxe, go, debs]
|
||||
runs-on: ubuntu-26.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- name: Install tools
|
||||
run: sudo apt-get update && sudo apt-get -y --no-install-recommends install createrepo-c dpkg-dev
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
path: art
|
||||
- name: Compose bundles
|
||||
env:
|
||||
VERSION: ${{ needs.prepare.outputs.version }}
|
||||
run: |
|
||||
set -e
|
||||
pick() { find "$@" -name '*.rpm' ! -name '*.src.rpm'; }
|
||||
for dist in el9 el10 suse16; do
|
||||
for arch in x86_64 aarch64; do
|
||||
mapfile -t files < <(
|
||||
pick "art/rpms-$dist-x86_64" -name '*.noarch.rpm'
|
||||
pick "art/rpms-$dist-$arch" -name "*.$arch.rpm"
|
||||
pick art/rpms-osdeploy-* art/rpms-ipxe "art/rpms-go-$arch")
|
||||
bash .github/scripts/mkbundle.sh "$dist-$arch" "$VERSION" bundles "${files[@]}"
|
||||
done
|
||||
done
|
||||
# Architecture: all debs come from the amd64 leg only
|
||||
pickdeb() { # <dir> <all|arch>
|
||||
for f in "$1"/*.deb; do
|
||||
[ "$(dpkg-deb -f "$f" Architecture)" = "$2" ] && echo "$f"
|
||||
done
|
||||
return 0
|
||||
}
|
||||
for release in noble resolute; do
|
||||
for arch in amd64 arm64; do
|
||||
mapfile -t files < <(
|
||||
pickdeb "art/debs-$release-amd64" all
|
||||
pickdeb "art/debs-$release-$arch" "$arch")
|
||||
bash .github/scripts/mkbundle.sh "$release-$arch" "$VERSION" bundles "${files[@]}"
|
||||
done
|
||||
done
|
||||
- name: Write run summary
|
||||
env:
|
||||
VERSION: ${{ needs.prepare.outputs.version }}
|
||||
run: |
|
||||
{
|
||||
echo "## confluent nightly $VERSION"
|
||||
echo
|
||||
for b in bundles/*.tar.gz; do
|
||||
echo "<details><summary>$(basename "$b")</summary>"
|
||||
echo
|
||||
tar tzf "$b" | sed -n 's|^[^/]*/\([^/]*\.\(rpm\|deb\)\)$|- \1|p' | sort
|
||||
echo
|
||||
echo "</details>"
|
||||
done
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: bundles
|
||||
path: bundles/*.tar.gz
|
||||
retention-days: 7
|
||||
if-no-files-found: error
|
||||
|
||||
smoketest:
|
||||
name: smoketest (${{ matrix.target }})
|
||||
needs: bundle
|
||||
runs-on: ${{ matrix.runner }}
|
||||
container: ${{ matrix.image }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { target: el9-x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
|
||||
- { target: el9-aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
|
||||
- { target: el10-x86_64, image: "almalinux:10", runner: ubuntu-26.04 }
|
||||
- { target: el10-aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm }
|
||||
- { target: suse16-x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 }
|
||||
- { target: suse16-aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm }
|
||||
- { target: noble-amd64, image: "ubuntu:noble", runner: ubuntu-26.04 }
|
||||
- { target: noble-arm64, image: "ubuntu:noble", runner: ubuntu-26.04-arm }
|
||||
- { target: resolute-amd64, image: "ubuntu:resolute", runner: ubuntu-26.04 }
|
||||
- { target: resolute-arm64, image: "ubuntu:resolute", runner: ubuntu-26.04-arm }
|
||||
steps:
|
||||
- name: Install tar
|
||||
run: |
|
||||
if command -v zypper > /dev/null; then
|
||||
zypper -n install --no-recommends tar gzip
|
||||
elif command -v dnf > /dev/null; then
|
||||
dnf -y --setopt=install_weak_deps=False install tar gzip
|
||||
fi
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: bundles
|
||||
path: bundles
|
||||
- name: Install and smoke test
|
||||
run: |
|
||||
tar xzf bundles/confluent-nightly-${{ matrix.target }}.tar.gz
|
||||
bash .github/scripts/smoketest.sh confluent-nightly-${{ matrix.target }}
|
||||
|
||||
publish:
|
||||
name: publish nightly release
|
||||
needs: [prepare, smoketest]
|
||||
if: >-
|
||||
needs.prepare.outputs.build == 'true' &&
|
||||
(github.event_name == 'schedule' ||
|
||||
(github.event_name == 'workflow_dispatch' && inputs.publish))
|
||||
runs-on: ubuntu-26.04
|
||||
permissions:
|
||||
contents: write
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
VERSION: ${{ needs.prepare.outputs.version }}
|
||||
KEY: ${{ needs.prepare.outputs.key }}
|
||||
BROWSERFS: ${{ needs.prepare.outputs.browserfs }}
|
||||
VINZ: ${{ needs.prepare.outputs.vinz }}
|
||||
steps:
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: bundles
|
||||
path: bundles
|
||||
- name: Write release notes
|
||||
run: |
|
||||
url=https://github.com/$GITHUB_REPOSITORY
|
||||
cat > notes.md <<EOF
|
||||
Unofficial development build of $url/commit/$GITHUB_SHA, rebuilt every night when something changed. Packages are unsigned and not meant for production.
|
||||
|
||||
| Bundle | Target |
|
||||
|---|---|
|
||||
EOF
|
||||
for b in bundles/*.tar.gz; do
|
||||
n=$(basename "$b")
|
||||
t=${n#confluent-nightly-}
|
||||
echo "| [$n]($url/releases/download/nightly/$n) | ${t%.tar.gz} |" >> notes.md
|
||||
done
|
||||
cat >> notes.md <<EOF
|
||||
|
||||
Each bundle holds every package for its target plus repository metadata, so the directory also works as a dnf, zypper or apt repository. EL example:
|
||||
|
||||
\`\`\`
|
||||
curl -LO $url/releases/download/nightly/confluent-nightly-el9-x86_64.tar.gz
|
||||
tar xf confluent-nightly-el9-x86_64.tar.gz
|
||||
dnf install epel-release && crb enable
|
||||
dnf install ./confluent-nightly-el9-x86_64/*.rpm
|
||||
\`\`\`
|
||||
|
||||
See INSTALL.txt in each bundle for the other distributions.
|
||||
|
||||
Built from:
|
||||
- confluent $GITHUB_SHA
|
||||
- browserfs $BROWSERFS_REPO/commit/$BROWSERFS
|
||||
- vinz $VINZ_REPO/commit/$VINZ
|
||||
- iPXE prebuilt release roms (see ipxe/confluent_ipxe.spec)
|
||||
|
||||
<!-- build: $KEY -->
|
||||
EOF
|
||||
cat notes.md
|
||||
- name: Publish
|
||||
# Move the tag and replace the assets in place, so the release and its
|
||||
# download URLs never disappear between nights.
|
||||
run: |
|
||||
set -e
|
||||
if gh release view nightly > /dev/null 2>&1; then
|
||||
gh api -X PATCH "repos/$GH_REPO/git/refs/tags/nightly" -f sha="$GITHUB_SHA" -F force=true > /dev/null
|
||||
gh release edit nightly --title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false
|
||||
for old in $(gh release view nightly --json assets -q '.assets[].name'); do
|
||||
[ -f "bundles/$old" ] || gh release delete-asset nightly "$old" -y
|
||||
done
|
||||
gh release upload nightly bundles/*.tar.gz --clobber
|
||||
else
|
||||
# a stale tag without a release would keep pointing at its old commit
|
||||
gh api -X DELETE "repos/$GH_REPO/git/refs/tags/nightly" > /dev/null 2>&1 || true
|
||||
gh release create nightly bundles/*.tar.gz --target "$GITHUB_SHA" \
|
||||
--title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false
|
||||
fi
|
||||
@@ -4,6 +4,8 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
# the rolling pre-release nightly.yml maintains
|
||||
- '!nightly'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
@@ -48,3 +48,11 @@ Cutting a new series:
|
||||
wrong series.
|
||||
|
||||
Patch releases land on branch `X.Y` and are tagged `X.Y.Z` there; no `VERSION` edit is needed.
|
||||
|
||||
## Nightly builds
|
||||
|
||||
`.github/workflows/nightly.yml` builds the rpms (EL9, EL10, SUSE 16) and debs (Ubuntu 24.04, 26.04)
|
||||
for x86_64 and aarch64 every night that something changed, installs each set on a clean container,
|
||||
and publishes them as the `nightly` pre-release. Every bundle there holds all packages for one
|
||||
target and an `INSTALL.txt`. Pull requests that touch the packaging run the same build and tests
|
||||
without publishing. Genesis is not built there yet.
|
||||
|
||||
Reference in New Issue
Block a user