mirror of
https://github.com/xcat2/confluent.git
synced 2026-10-20 06:35:33 +00:00
Merge pull request #312 from Obihoernchen/ci/nightly-packages
Build and publish nightly packages
This commit is contained in:
Jarrod Johnson
GitHub
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/bin/bash
|
||||
# Build the confluent debs in an Ubuntu container, like build/arm/*/buildapt.sh:
|
||||
# every component with a builddeb script, plus alien conversions of the rpms
|
||||
# that have no deb packaging of their own (osdeploy, ipxe, browserfs, vinz).
|
||||
# With --arch-only, only the architecture-specific ones: confluent_vtbufferd
|
||||
# and the conversions of rpms that are not noarch.
|
||||
#
|
||||
# Usage: build-debs.sh [--arch-only] <outdir> [<dir with rpms to convert>]
|
||||
set -e
|
||||
|
||||
ARCHONLY=
|
||||
if [ "$1" = "--arch-only" ]; then
|
||||
ARCHONLY=1
|
||||
shift
|
||||
fi
|
||||
OUT=$(readlink -f "$1")
|
||||
RPMS=${2:+$(readlink -f "$2")}
|
||||
[ -n "$OUT" ] || { echo "usage: $0 [--arch-only] <outdir> [rpmdir]" >&2; exit 1; }
|
||||
|
||||
cd "$(dirname "$0")/../.."
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get -y --no-install-recommends install git build-essential debhelper devscripts dh-python fakeroot \
|
||||
python3-all python3-setuptools python3-stdeb pandoc patch alien
|
||||
# stdeb on noble still imports the SafeConfigParser python 3.12 removed
|
||||
if (cd / && patch -p1 --dry-run -s < "$OLDPWD/build/arm/noble/stdeb.patch") > /dev/null 2>&1; then
|
||||
(cd / && patch -p1 < "$OLDPWD/build/arm/noble/stdeb.patch")
|
||||
fi
|
||||
git config --global --add safe.directory '*'
|
||||
./mkversion
|
||||
|
||||
mkdir -p "$OUT"
|
||||
# vtbufferd first: it recreates /tmp/confluent, which client and server reuse
|
||||
./confluent_vtbufferd/builddeb "$OUT"
|
||||
if [ -z "$ARCHONLY" ]; then
|
||||
./confluent_client/builddeb "$OUT"
|
||||
./confluent_server/builddeb "$OUT"
|
||||
./imgutil/builddeb "$OUT"
|
||||
fi
|
||||
|
||||
if [ -n "$RPMS" ]; then
|
||||
skip=(! -name '*.src.rpm')
|
||||
[ -n "$ARCHONLY" ] && skip+=(! -name '*.noarch.rpm')
|
||||
cd "$OUT"
|
||||
find "$RPMS" -name '*.rpm' "${skip[@]}" -print0 |
|
||||
xargs -0 -r -n1 fakeroot alien --to-deb --scripts --keep-version
|
||||
fi
|
||||
ls -la "$OUT"
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
# Build the rpm of one of the Go helpers confluent runs but keeps in separate
|
||||
# repositories (browserfs, vinz), for the native architecture only.
|
||||
#
|
||||
# Mirrors the buildrpm script in those repositories, which needs git tags for
|
||||
# its version. They carry none yet, so fall back to 0~dev<commits>+g<hash>,
|
||||
# which sorts below any tagged release.
|
||||
#
|
||||
# Usage: build-go-rpm.sh <name> <git-url> [ref]
|
||||
set -e
|
||||
|
||||
NAME=$1
|
||||
URL=$2
|
||||
REF=${3:-HEAD}
|
||||
WORK=$(mktemp -d)
|
||||
|
||||
git clone -q "$URL" "$WORK/$NAME"
|
||||
cd "$WORK/$NAME"
|
||||
git checkout -q "$REF"
|
||||
if desc=$(git describe 2>/dev/null); then
|
||||
VERSION=${desc%%-*}
|
||||
NUMCOMMITS=$(echo "$desc" | cut -d- -f2)
|
||||
if [ "$NUMCOMMITS" != "$VERSION" ]; then
|
||||
VERSION=$(echo "$VERSION" | awk -F. -v OFS=. '{$NF=$NF+1; print}')
|
||||
VERSION=$VERSION~dev$NUMCOMMITS+$(echo "$desc" | cut -d- -f3)
|
||||
fi
|
||||
else
|
||||
VERSION=0~dev$(git rev-list --count HEAD)+g$(git rev-parse --short HEAD)
|
||||
fi
|
||||
echo "$NAME $VERSION ($(git rev-parse HEAD))"
|
||||
|
||||
mkdir -p ~/rpmbuild/SOURCES
|
||||
(cd .. && tar cf ~/rpmbuild/SOURCES/confluent-$NAME.tar --exclude=.git "$NAME")
|
||||
sed -e "s/#VERSION#/$VERSION/" $NAME.spec.tmpl > $NAME.spec
|
||||
rpmbuild -ba $NAME.spec
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
# Build the confluent_ipxe rpm. The spec repackages the prebuilt boot roms
|
||||
# that iPXE publishes with each release; rpmbuild does not fetch remote
|
||||
# sources itself, so download the tarball of the release the spec names.
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")/../.."
|
||||
SPEC=ipxe/confluent_ipxe.spec
|
||||
VERSION=$(rpmspec -q --srpm --qf '%{version}' $SPEC)
|
||||
mkdir -p ~/rpmbuild/SOURCES
|
||||
curl -fsSL -o ~/rpmbuild/SOURCES/ipxeboot.tar.gz \
|
||||
"https://github.com/ipxe/ipxe/releases/download/v$VERSION/ipxeboot.tar.gz"
|
||||
rpmbuild -ba $SPEC
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/bash
|
||||
# Build the confluent_osdeploy rpm the way confluent_osdeploy/buildrpm and
|
||||
# buildrpm-aarch64 do, but from utils binaries built by earlier CI jobs
|
||||
# (build-utils.sh) instead of compiling them in local podman containers.
|
||||
#
|
||||
# Usage: build-osdeploy.sh <x86_64|aarch64> <el8utils-dir> <el9utils-dir>
|
||||
set -e
|
||||
|
||||
ARCH=$1
|
||||
EL8UTILS=$(readlink -f "$2")
|
||||
EL9UTILS=$(readlink -f "$3")
|
||||
|
||||
cd "$(dirname "$0")/../.."
|
||||
VERSION=$(./mkversion --tilde)
|
||||
|
||||
case "$ARCH" in
|
||||
x86_64) SPEC=confluent_osdeploy.spec ;;
|
||||
aarch64) SPEC=confluent_osdeploy-aarch64.spec ;;
|
||||
*) echo "unknown arch $ARCH" >&2; exit 1 ;;
|
||||
esac
|
||||
sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy/$SPEC.tmpl > confluent_osdeploy/$SPEC
|
||||
|
||||
mkdir -p ~/rpmbuild/SOURCES
|
||||
cp LICENSE confluent_osdeploy/
|
||||
tar Jcf ~/rpmbuild/SOURCES/confluent_osdeploy.tar.xz confluent_osdeploy
|
||||
|
||||
for gen in el8 el9; do
|
||||
utils=$EL8UTILS
|
||||
[ "$gen" = el9 ] && utils=$EL9UTILS
|
||||
rm -rf ${gen}bin
|
||||
mkdir -p ${gen}bin/opt/confluent/bin ${gen}bin/stateless-bin
|
||||
(cd "$utils" && cp confluent_imginfo copernicus clortho autocons "$OLDPWD"/${gen}bin/opt/confluent/bin/)
|
||||
(cd "$utils" && cp start_root urlmount "$OLDPWD"/${gen}bin/stateless-bin/)
|
||||
# artifact downloads lose the execute bit
|
||||
chmod 755 ${gen}bin/opt/confluent/bin/* ${gen}bin/stateless-bin/*
|
||||
tar Jcf ~/rpmbuild/SOURCES/confluent_${gen}bin.tar.xz ${gen}bin/
|
||||
rm -rf ${gen}bin
|
||||
done
|
||||
|
||||
rpmbuild -ba confluent_osdeploy/$SPEC
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
# Build the confluent rpms whose spec depends on the target distribution:
|
||||
# confluent_vtbufferd (arch-specific) and, unless --arch-only is given, the
|
||||
# noarch client, server and imgutil. Runs inside an EL or openSUSE container.
|
||||
#
|
||||
# Usage: build-rpms.sh [--arch-only]
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")/../.."
|
||||
|
||||
if command -v zypper > /dev/null; then
|
||||
zypper -n install --no-recommends rpm-build gcc make git-core tar xz pandoc systemd python3 python3-setuptools
|
||||
# the buildrpm scripts stage into ~/rpmbuild, SUSE defaults to /usr/src/packages
|
||||
echo "%_topdir $HOME/rpmbuild" >> ~/.rpmmacros
|
||||
else
|
||||
dnf -y --setopt=install_weak_deps=False install epel-release
|
||||
dnf -y --setopt=install_weak_deps=False install rpm-build gcc make git-core tar xz pandoc systemd python3 python3-setuptools
|
||||
fi
|
||||
git config --global --add safe.directory "$PWD"
|
||||
./mkversion
|
||||
mkdir -p ~/rpmbuild/SOURCES ~/rpmbuild/SPECS
|
||||
|
||||
# the buildrpm scripts hide rpmbuild's output, so on failure rerun the spec
|
||||
# they wrote to show why
|
||||
build() {
|
||||
"$1/buildrpm" && return 0
|
||||
spec=~/rpmbuild/SPECS/$(basename "$1").spec
|
||||
[ -f "$spec" ] && rpmbuild -ba "$spec"
|
||||
return 1
|
||||
}
|
||||
|
||||
build confluent_vtbufferd
|
||||
if [ "$1" != "--arch-only" ]; then
|
||||
build confluent_client
|
||||
build confluent_server
|
||||
# imgutil/buildrpm calls rpmbuild directly, so its output is visible
|
||||
./imgutil/buildrpm
|
||||
fi
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
# Build the confluent_osdeploy utils inside a distro container, standing in for
|
||||
# the el7build/el8builder/el9build podman images confluent_osdeploy/buildrpm
|
||||
# uses. The el8bin set is linked against the oldest glibc any target runs
|
||||
# (el7 on x86_64, el8 on aarch64), so it must be built on that distro.
|
||||
#
|
||||
# Usage: build-utils.sh <outdir> (run from the repository root)
|
||||
set -e
|
||||
|
||||
OUT=$1
|
||||
[ -n "$OUT" ] || { echo "usage: $0 <outdir>" >&2; exit 1; }
|
||||
|
||||
. /etc/os-release
|
||||
case "$VERSION_ID" in
|
||||
7*)
|
||||
# CentOS 7 is EOL: its mirrors are gone, only the vault remains
|
||||
sed -i -e 's/^mirrorlist=/#mirrorlist=/' \
|
||||
-e 's|^#\?baseurl=http://mirror.centos.org|baseurl=https://vault.centos.org|' \
|
||||
/etc/yum.repos.d/CentOS-*.repo
|
||||
yum -y install gcc make binutils libcurl-devel fuse-devel
|
||||
;;
|
||||
8*)
|
||||
dnf -y --setopt=install_weak_deps=False install dnf-plugins-core
|
||||
dnf config-manager --set-enabled powertools
|
||||
dnf -y --setopt=install_weak_deps=False install gcc make binutils libcurl-devel fuse-devel libxcrypt-devel
|
||||
;;
|
||||
*)
|
||||
dnf -y --setopt=install_weak_deps=False install dnf-plugins-core
|
||||
dnf config-manager --set-enabled crb
|
||||
dnf -y --setopt=install_weak_deps=False install gcc make binutils libcurl-devel fuse-devel libxcrypt-devel
|
||||
;;
|
||||
esac
|
||||
|
||||
make -C confluent_osdeploy/utils
|
||||
mkdir -p "$OUT"
|
||||
cd confluent_osdeploy/utils
|
||||
cp confluent_imginfo copernicus clortho autocons start_root urlmount "$OUT"/
|
||||
make clean
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/bin/bash
|
||||
# Assemble the nightly bundle for one target: the packages, repository
|
||||
# metadata so the directory also works as a dnf/zypper/apt repository, and
|
||||
# an INSTALL.txt. Produces confluent-nightly-<target>.tar.gz in <outdir>.
|
||||
#
|
||||
# Usage: mkbundle.sh <target> <version> <outdir> <package file>...
|
||||
# target is <el9|el10|suse16|noble|resolute>-<arch>
|
||||
set -e
|
||||
|
||||
TARGET=$1
|
||||
VERSION=$2
|
||||
OUTDIR=$(readlink -f "$3")
|
||||
shift 3
|
||||
|
||||
NAME=confluent-nightly-$TARGET
|
||||
DIR=$OUTDIR/$NAME
|
||||
REPO=${GITHUB_REPOSITORY:-xcat2/confluent}
|
||||
SHA=${GITHUB_SHA:-$(git rev-parse HEAD)}
|
||||
rm -rf "$DIR"
|
||||
mkdir -p "$DIR"
|
||||
cp "$@" "$DIR"/
|
||||
echo "$VERSION" > "$DIR"/VERSION
|
||||
|
||||
cat > "$DIR"/INSTALL.txt <<EOF
|
||||
confluent nightly $VERSION ($TARGET)
|
||||
|
||||
Unofficial development build of https://github.com/$REPO
|
||||
commit $SHA. Packages are unsigned. Not meant for production.
|
||||
|
||||
EOF
|
||||
|
||||
case "$TARGET" in
|
||||
el*)
|
||||
createrepo_c -q "$DIR"
|
||||
cat >> "$DIR"/INSTALL.txt <<EOF
|
||||
Dependencies come from the distribution plus EPEL and CRB:
|
||||
|
||||
dnf install epel-release
|
||||
crb enable
|
||||
|
||||
Install everything:
|
||||
|
||||
dnf install ./$NAME/*.rpm
|
||||
|
||||
or only some of it, using the directory as a repository:
|
||||
|
||||
dnf install --repofrompath=confluent-nightly,\$PWD/$NAME \\
|
||||
--setopt=confluent-nightly.gpgcheck=0 confluent_server
|
||||
EOF
|
||||
;;
|
||||
suse*)
|
||||
createrepo_c -q "$DIR"
|
||||
cat >> "$DIR"/INSTALL.txt <<EOF
|
||||
Install everything:
|
||||
|
||||
zypper --no-gpg-checks install --allow-unsigned-rpm ./$NAME/*.rpm
|
||||
|
||||
or only some of it, using the directory as a repository:
|
||||
|
||||
zypper addrepo --no-gpgcheck \$PWD/$NAME confluent-nightly
|
||||
zypper install confluent_server
|
||||
EOF
|
||||
;;
|
||||
noble*|resolute*)
|
||||
(cd "$DIR" && dpkg-scanpackages --multiversion . /dev/null 2> /dev/null | gzip -9 > Packages.gz)
|
||||
cat >> "$DIR"/INSTALL.txt <<EOF
|
||||
Install everything:
|
||||
|
||||
apt install ./$NAME/*.deb
|
||||
|
||||
or only some of it, using the directory as a repository:
|
||||
|
||||
echo "deb [trusted=yes] file:\$PWD/$NAME ./" > /etc/apt/sources.list.d/confluent-nightly.list
|
||||
apt update && apt install confluent-server
|
||||
EOF
|
||||
;;
|
||||
*)
|
||||
echo "unknown target $TARGET" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
cat >> "$DIR"/INSTALL.txt <<EOF
|
||||
|
||||
The bundle carries the osdeploy support for both x86_64 and aarch64
|
||||
deployment targets. Genesis (confluent-genesis-*) is not built yet.
|
||||
EOF
|
||||
|
||||
tar -C "$OUTDIR" -czf "$OUTDIR/$NAME.tar.gz" "$NAME"
|
||||
rm -rf "$DIR"
|
||||
echo "$OUTDIR/$NAME.tar.gz"
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/bin/bash
|
||||
# Install a nightly bundle on a clean EL, openSUSE or Ubuntu container and
|
||||
# check that the server imports and starts.
|
||||
#
|
||||
# Every dependency has to resolve from the distribution's own repositories
|
||||
# (plus EPEL on EL), so a missing one fails the run.
|
||||
#
|
||||
# Usage: smoketest.sh <dir with the packages to install>
|
||||
set -x
|
||||
|
||||
PKGDIR=$(readlink -f "$1")
|
||||
FAIL=0
|
||||
. /etc/os-release
|
||||
|
||||
case "$ID" in
|
||||
almalinux|rocky|rhel|centos)
|
||||
dnf -y install epel-release
|
||||
dnf config-manager --set-enabled crb
|
||||
dnf -y install procps-ng util-linux
|
||||
echo "::group::install packages"
|
||||
dnf -y install "$PKGDIR"/*.rpm || FAIL=1
|
||||
echo "::endgroup::"
|
||||
;;
|
||||
opensuse-leap|sles)
|
||||
zypper -n install procps util-linux
|
||||
echo "::group::install packages"
|
||||
# nightly rpms are unsigned
|
||||
zypper -n --no-gpg-checks install --allow-unsigned-rpm "$PKGDIR"/*.rpm || FAIL=1
|
||||
echo "::endgroup::"
|
||||
;;
|
||||
ubuntu|debian)
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get -y install procps util-linux
|
||||
echo "::group::install packages"
|
||||
apt-get -y install "$PKGDIR"/*.deb || FAIL=1
|
||||
echo "::endgroup::"
|
||||
;;
|
||||
*)
|
||||
echo "unsupported distribution $ID" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "::group::installed content"
|
||||
test -d /opt/confluent/lib/osdeploy/el10/profiles || FAIL=1
|
||||
test -f /opt/confluent/lib/ipxe/ipxe.efi || FAIL=1
|
||||
test -x /opt/confluent/bin/browserfs || FAIL=1
|
||||
test -x /opt/confluent/bin/vinz || FAIL=1
|
||||
test -x /opt/confluent/bin/vtbufferd || FAIL=1
|
||||
test -x /opt/confluent/bin/imgutil || FAIL=1
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::import and daemon smoke test"
|
||||
export PYTHONPATH=/opt/confluent/lib/python
|
||||
python3 -c "import confluent.main; print('IMPORT confluent.main: OK')" || FAIL=1
|
||||
python3 -c "import confluent.snmputil; print('IMPORT confluent.snmputil: OK')" || FAIL=1
|
||||
python3 -c "import aiohmi.redfish.command; print('IMPORT aiohmi.redfish: OK')" || FAIL=1
|
||||
# packaging sets up non-root operation (/etc/confluent owned by confluent),
|
||||
# and there is no systemd in the container to start it, so run it directly
|
||||
mkdir -p /var/run/confluent
|
||||
chown confluent:confluent /var/run/confluent
|
||||
timeout 60 runuser -u confluent -- /opt/confluent/bin/confluent -f > /tmp/confluentd.log 2>&1 &
|
||||
for _ in $(seq 30); do
|
||||
[ -S /var/run/confluent/api.sock ] && break
|
||||
sleep 1
|
||||
done
|
||||
if [ -S /var/run/confluent/api.sock ] && /opt/confluent/bin/confetty show /; then
|
||||
echo "DAEMON_RUNNING=yes"
|
||||
else
|
||||
echo "DAEMON_RUNNING=no"
|
||||
FAIL=1
|
||||
fi
|
||||
cat /tmp/confluentd.log
|
||||
pkill -f /opt/confluent/bin/confluent
|
||||
echo "::endgroup::"
|
||||
|
||||
exit $FAIL
|
||||
@@ -0,0 +1,458 @@
|
||||
name: Nightly
|
||||
|
||||
# Builds the confluent rpms (EL9, EL10, SUSE 16) and debs (Ubuntu noble,
|
||||
# resolute) for x86_64 and aarch64, installs each bundle on a clean container,
|
||||
# and publishes the bundles as the rolling "nightly" pre-release. Pull
|
||||
# requests that touch the packaging build and test, but publish nothing.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '23 2 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
publish:
|
||||
description: Publish the result as the nightly pre-release
|
||||
type: boolean
|
||||
default: true
|
||||
force:
|
||||
description: Build even if nothing changed since the last nightly
|
||||
type: boolean
|
||||
default: false
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/nightly.yml
|
||||
- .github/scripts/**
|
||||
- '**/buildrpm'
|
||||
- '**/buildrpm-aarch64'
|
||||
- '**/builddeb'
|
||||
- '**/makesetup'
|
||||
- '**/setup.py.tmpl'
|
||||
- '**/*.spec'
|
||||
- '**/*.spec.tmpl'
|
||||
- confluent_osdeploy/utils/**
|
||||
- build/arm/**
|
||||
- mkversion
|
||||
- VERSION
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: nightly-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
env:
|
||||
BROWSERFS_REPO: https://github.com/xcat2/browserfs
|
||||
VINZ_REPO: https://github.com/xcat2/vinz
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
name: prepare
|
||||
runs-on: ubuntu-26.04
|
||||
outputs:
|
||||
build: ${{ steps.check.outputs.build }}
|
||||
version: ${{ steps.check.outputs.version }}
|
||||
key: ${{ steps.check.outputs.key }}
|
||||
browserfs: ${{ steps.check.outputs.browserfs }}
|
||||
vinz: ${{ steps.check.outputs.vinz }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Version and change check
|
||||
id: check
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
FORCE: ${{ inputs.force }}
|
||||
# The release notes carry the commits of the last published build, so
|
||||
# a scheduled run with nothing new to build stops here.
|
||||
run: |
|
||||
version=$(./mkversion --tilde)
|
||||
browserfs=$(git ls-remote "$BROWSERFS_REPO" HEAD | cut -f1)
|
||||
vinz=$(git ls-remote "$VINZ_REPO" HEAD | cut -f1)
|
||||
key="confluent@$GITHUB_SHA browserfs@$browserfs vinz@$vinz"
|
||||
# no release yet on the first run, and bash runs with pipefail here
|
||||
last=$({ gh release view nightly --repo "$GITHUB_REPOSITORY" --json body -q .body 2>/dev/null || true; } |
|
||||
sed -n 's/^<!-- build: \(.*\) -->$/\1/p')
|
||||
build=true
|
||||
if [ "$GITHUB_EVENT_NAME" != pull_request ] && [ "$FORCE" != true ] && [ "$key" = "$last" ]; then
|
||||
echo "::notice::Nothing changed since the last nightly ($last)"
|
||||
build=false
|
||||
fi
|
||||
{
|
||||
echo "build=$build"
|
||||
echo "version=$version"
|
||||
echo "key=$key"
|
||||
echo "browserfs=$browserfs"
|
||||
echo "vinz=$vinz"
|
||||
} | tee -a "$GITHUB_OUTPUT"
|
||||
|
||||
utils:
|
||||
name: osdeploy utils (${{ matrix.set }}-${{ matrix.arch }})
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build == 'true'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# The el8bin set must run on the oldest target userspace, so it is
|
||||
# built where confluent_osdeploy/buildrpm builds it: el7 for x86_64,
|
||||
# el8 for aarch64. CentOS 7 is too old for the node-based actions,
|
||||
# so these run the image by hand instead of as a job container.
|
||||
include:
|
||||
- { set: el8, arch: x86_64, image: "centos:7", runner: ubuntu-26.04 }
|
||||
- { set: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
|
||||
- { set: el8, arch: aarch64, image: "almalinux:8", runner: ubuntu-26.04-arm }
|
||||
- { set: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- name: Build utils
|
||||
run: docker run --rm -v "$PWD:/src" -w /src ${{ matrix.image }} bash .github/scripts/build-utils.sh /src/utilsbin
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: utils-${{ matrix.set }}-${{ matrix.arch }}
|
||||
path: utilsbin/
|
||||
retention-days: 3
|
||||
|
||||
rpms:
|
||||
name: RPMs (${{ matrix.dist }}-${{ matrix.arch }})
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build == 'true'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
container: ${{ matrix.image }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# noarch rpms are built in the x86_64 leg only, and shared by both
|
||||
include:
|
||||
- { dist: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
|
||||
- { dist: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
|
||||
- { dist: el10, arch: x86_64, image: "almalinux:10", runner: ubuntu-26.04 }
|
||||
- { dist: el10, arch: aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm }
|
||||
- { dist: suse16, arch: x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 }
|
||||
- { dist: suse16, arch: aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm }
|
||||
steps:
|
||||
- name: Install git
|
||||
# without git in the container, checkout downloads a tarball and
|
||||
# mkversion has no history to describe
|
||||
run: |
|
||||
if command -v zypper > /dev/null; then
|
||||
zypper -n install --no-recommends git-core tar gzip
|
||||
else
|
||||
dnf -y --setopt=install_weak_deps=False install git-core
|
||||
fi
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build RPMs
|
||||
run: bash .github/scripts/build-rpms.sh ${{ matrix.arch == 'aarch64' && '--arch-only' || '' }}
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rpms-${{ matrix.dist }}-${{ matrix.arch }}
|
||||
path: ~/rpmbuild/RPMS/
|
||||
retention-days: 3
|
||||
|
||||
osdeploy:
|
||||
name: osdeploy RPM (${{ matrix.arch }})
|
||||
needs: utils
|
||||
runs-on: ubuntu-26.04
|
||||
container: almalinux:9
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
arch: [x86_64, aarch64]
|
||||
steps:
|
||||
- name: Install build dependencies
|
||||
run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core tar xz cpio
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: utils-el8-${{ matrix.arch }}
|
||||
path: el8utils
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: utils-el9-${{ matrix.arch }}
|
||||
path: el9utils
|
||||
- name: Build osdeploy RPM
|
||||
run: |
|
||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
bash .github/scripts/build-osdeploy.sh ${{ matrix.arch }} el8utils el9utils
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rpms-osdeploy-${{ matrix.arch }}
|
||||
path: ~/rpmbuild/RPMS/
|
||||
retention-days: 3
|
||||
|
||||
ipxe:
|
||||
name: iPXE RPM
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build == 'true'
|
||||
runs-on: ubuntu-26.04
|
||||
container: almalinux:9
|
||||
steps:
|
||||
- name: Install build dependencies
|
||||
run: dnf -y --setopt=install_weak_deps=False install rpm-build tar gzip
|
||||
- uses: actions/checkout@v7
|
||||
- name: Build iPXE RPM
|
||||
run: bash .github/scripts/build-ipxe.sh
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rpms-ipxe
|
||||
path: ~/rpmbuild/RPMS/
|
||||
retention-days: 3
|
||||
|
||||
go:
|
||||
name: browserfs and vinz RPMs (${{ matrix.arch }})
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.build == 'true'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
container: almalinux:10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# static (CGO_ENABLED=0) binaries, so one build serves every distro
|
||||
include:
|
||||
- { arch: x86_64, runner: ubuntu-26.04 }
|
||||
- { arch: aarch64, runner: ubuntu-26.04-arm }
|
||||
steps:
|
||||
- name: Install build dependencies
|
||||
run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core golang tar
|
||||
- uses: actions/checkout@v7
|
||||
- name: Build RPMs
|
||||
run: |
|
||||
bash .github/scripts/build-go-rpm.sh browserfs "$BROWSERFS_REPO" ${{ needs.prepare.outputs.browserfs }}
|
||||
bash .github/scripts/build-go-rpm.sh vinz "$VINZ_REPO" ${{ needs.prepare.outputs.vinz }}
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rpms-go-${{ matrix.arch }}
|
||||
path: ~/rpmbuild/RPMS/
|
||||
retention-days: 3
|
||||
|
||||
debs:
|
||||
name: debs (${{ matrix.release }}-${{ matrix.arch }})
|
||||
needs: [osdeploy, ipxe, go]
|
||||
runs-on: ${{ matrix.runner }}
|
||||
container: ubuntu:${{ matrix.release }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
release: [noble, resolute]
|
||||
arch: [amd64, arm64]
|
||||
include:
|
||||
# alien only converts rpms for the architecture it runs on
|
||||
- { arch: amd64, rpmarch: x86_64, runner: ubuntu-26.04 }
|
||||
- { arch: arm64, rpmarch: aarch64, runner: ubuntu-26.04-arm }
|
||||
steps:
|
||||
- name: Install git
|
||||
# ca-certificates is only a Recommends of git, and checkout clones over https
|
||||
run: apt-get update && apt-get -y --no-install-recommends install git ca-certificates
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
# noarch, so converted in the amd64 leg only, like the noarch rpms
|
||||
- uses: actions/download-artifact@v8
|
||||
if: matrix.arch == 'amd64'
|
||||
with:
|
||||
pattern: rpms-osdeploy-*
|
||||
path: alien
|
||||
merge-multiple: true
|
||||
- uses: actions/download-artifact@v8
|
||||
if: matrix.arch == 'amd64'
|
||||
with:
|
||||
name: rpms-ipxe
|
||||
path: alien
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: rpms-go-${{ matrix.rpmarch }}
|
||||
path: alien
|
||||
- name: Build debs
|
||||
run: bash .github/scripts/build-debs.sh ${{ matrix.arch == 'arm64' && '--arch-only' || '' }} debout alien
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: debs-${{ matrix.release }}-${{ matrix.arch }}
|
||||
path: debout/*.deb
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
bundle:
|
||||
name: assemble bundles
|
||||
needs: [prepare, rpms, osdeploy, ipxe, go, debs]
|
||||
runs-on: ubuntu-26.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- name: Install tools
|
||||
run: sudo apt-get update && sudo apt-get -y --no-install-recommends install createrepo-c dpkg-dev
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
path: art
|
||||
- name: Compose bundles
|
||||
env:
|
||||
VERSION: ${{ needs.prepare.outputs.version }}
|
||||
run: |
|
||||
set -e
|
||||
pick() { find "$@" -name '*.rpm' ! -name '*.src.rpm'; }
|
||||
for dist in el9 el10 suse16; do
|
||||
for arch in x86_64 aarch64; do
|
||||
mapfile -t files < <(
|
||||
pick "art/rpms-$dist-x86_64" -name '*.noarch.rpm'
|
||||
pick "art/rpms-$dist-$arch" -name "*.$arch.rpm"
|
||||
pick art/rpms-osdeploy-* art/rpms-ipxe "art/rpms-go-$arch")
|
||||
bash .github/scripts/mkbundle.sh "$dist-$arch" "$VERSION" bundles "${files[@]}"
|
||||
done
|
||||
done
|
||||
# Architecture: all debs come from the amd64 leg only
|
||||
pickdeb() { # <dir> <all|arch>
|
||||
for f in "$1"/*.deb; do
|
||||
[ "$(dpkg-deb -f "$f" Architecture)" = "$2" ] && echo "$f"
|
||||
done
|
||||
return 0
|
||||
}
|
||||
for release in noble resolute; do
|
||||
for arch in amd64 arm64; do
|
||||
mapfile -t files < <(
|
||||
pickdeb "art/debs-$release-amd64" all
|
||||
pickdeb "art/debs-$release-$arch" "$arch")
|
||||
bash .github/scripts/mkbundle.sh "$release-$arch" "$VERSION" bundles "${files[@]}"
|
||||
done
|
||||
done
|
||||
- name: Write run summary
|
||||
env:
|
||||
VERSION: ${{ needs.prepare.outputs.version }}
|
||||
run: |
|
||||
{
|
||||
echo "## confluent nightly $VERSION"
|
||||
echo
|
||||
for b in bundles/*.tar.gz; do
|
||||
echo "<details><summary>$(basename "$b")</summary>"
|
||||
echo
|
||||
tar tzf "$b" | sed -n 's|^[^/]*/\([^/]*\.\(rpm\|deb\)\)$|- \1|p' | sort
|
||||
echo
|
||||
echo "</details>"
|
||||
done
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: bundles
|
||||
path: bundles/*.tar.gz
|
||||
retention-days: 7
|
||||
if-no-files-found: error
|
||||
|
||||
smoketest:
|
||||
name: smoketest (${{ matrix.target }})
|
||||
needs: bundle
|
||||
runs-on: ${{ matrix.runner }}
|
||||
container: ${{ matrix.image }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { target: el9-x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
|
||||
- { target: el9-aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
|
||||
- { target: el10-x86_64, image: "almalinux:10", runner: ubuntu-26.04 }
|
||||
- { target: el10-aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm }
|
||||
- { target: suse16-x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 }
|
||||
- { target: suse16-aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm }
|
||||
- { target: noble-amd64, image: "ubuntu:noble", runner: ubuntu-26.04 }
|
||||
- { target: noble-arm64, image: "ubuntu:noble", runner: ubuntu-26.04-arm }
|
||||
- { target: resolute-amd64, image: "ubuntu:resolute", runner: ubuntu-26.04 }
|
||||
- { target: resolute-arm64, image: "ubuntu:resolute", runner: ubuntu-26.04-arm }
|
||||
steps:
|
||||
- name: Install tar
|
||||
run: |
|
||||
if command -v zypper > /dev/null; then
|
||||
zypper -n install --no-recommends tar gzip
|
||||
elif command -v dnf > /dev/null; then
|
||||
dnf -y --setopt=install_weak_deps=False install tar gzip
|
||||
fi
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: bundles
|
||||
path: bundles
|
||||
- name: Install and smoke test
|
||||
run: |
|
||||
tar xzf bundles/confluent-nightly-${{ matrix.target }}.tar.gz
|
||||
bash .github/scripts/smoketest.sh confluent-nightly-${{ matrix.target }}
|
||||
|
||||
publish:
|
||||
name: publish nightly release
|
||||
needs: [prepare, smoketest]
|
||||
if: >-
|
||||
needs.prepare.outputs.build == 'true' &&
|
||||
(github.event_name == 'schedule' ||
|
||||
(github.event_name == 'workflow_dispatch' && inputs.publish))
|
||||
runs-on: ubuntu-26.04
|
||||
permissions:
|
||||
contents: write
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
VERSION: ${{ needs.prepare.outputs.version }}
|
||||
KEY: ${{ needs.prepare.outputs.key }}
|
||||
BROWSERFS: ${{ needs.prepare.outputs.browserfs }}
|
||||
VINZ: ${{ needs.prepare.outputs.vinz }}
|
||||
steps:
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: bundles
|
||||
path: bundles
|
||||
- name: Write release notes
|
||||
run: |
|
||||
url=https://github.com/$GITHUB_REPOSITORY
|
||||
cat > notes.md <<EOF
|
||||
Unofficial development build of $url/commit/$GITHUB_SHA, rebuilt every night when something changed. Packages are unsigned and not meant for production.
|
||||
|
||||
| Bundle | Target |
|
||||
|---|---|
|
||||
EOF
|
||||
for b in bundles/*.tar.gz; do
|
||||
n=$(basename "$b")
|
||||
t=${n#confluent-nightly-}
|
||||
echo "| [$n]($url/releases/download/nightly/$n) | ${t%.tar.gz} |" >> notes.md
|
||||
done
|
||||
cat >> notes.md <<EOF
|
||||
|
||||
Each bundle holds every package for its target plus repository metadata, so the directory also works as a dnf, zypper or apt repository. EL example:
|
||||
|
||||
\`\`\`
|
||||
curl -LO $url/releases/download/nightly/confluent-nightly-el9-x86_64.tar.gz
|
||||
tar xf confluent-nightly-el9-x86_64.tar.gz
|
||||
dnf install epel-release && crb enable
|
||||
dnf install ./confluent-nightly-el9-x86_64/*.rpm
|
||||
\`\`\`
|
||||
|
||||
See INSTALL.txt in each bundle for the other distributions.
|
||||
|
||||
Built from:
|
||||
- confluent $GITHUB_SHA
|
||||
- browserfs $BROWSERFS_REPO/commit/$BROWSERFS
|
||||
- vinz $VINZ_REPO/commit/$VINZ
|
||||
- iPXE prebuilt release roms (see ipxe/confluent_ipxe.spec)
|
||||
|
||||
<!-- build: $KEY -->
|
||||
EOF
|
||||
cat notes.md
|
||||
- name: Publish
|
||||
# Move the tag and replace the assets in place, so the release and its
|
||||
# download URLs never disappear between nights.
|
||||
run: |
|
||||
set -e
|
||||
if gh release view nightly > /dev/null 2>&1; then
|
||||
gh api -X PATCH "repos/$GH_REPO/git/refs/tags/nightly" -f sha="$GITHUB_SHA" -F force=true > /dev/null
|
||||
gh release edit nightly --title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false
|
||||
for old in $(gh release view nightly --json assets -q '.assets[].name'); do
|
||||
[ -f "bundles/$old" ] || gh release delete-asset nightly "$old" -y
|
||||
done
|
||||
gh release upload nightly bundles/*.tar.gz --clobber
|
||||
else
|
||||
# a stale tag without a release would keep pointing at its old commit
|
||||
gh api -X DELETE "repos/$GH_REPO/git/refs/tags/nightly" > /dev/null 2>&1 || true
|
||||
gh release create nightly bundles/*.tar.gz --target "$GITHUB_SHA" \
|
||||
--title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false
|
||||
fi
|
||||
@@ -4,6 +4,8 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
# the rolling pre-release nightly.yml maintains
|
||||
- '!nightly'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
# Confluent
|
||||
|
||||
 [](https://github.com/xcat2/confluent/blob/master/LICENSE)
|
||||
[](https://github.com/xcat2/confluent/actions/workflows/ci.yml?query=branch%3Amaster)
|
||||
[](https://github.com/xcat2/confluent/actions/workflows/nightly.yml?query=branch%3Amaster)
|
||||
[](https://github.com/xcat2/confluent/releases/latest)
|
||||
[](https://xcat2.github.io/confluent-docs/)
|
||||
|
||||
[](LICENSE)
|
||||
|
||||
Confluent is a software package to handle essential bootstrap and operation of scale-out server configurations.
|
||||
It supports stateful and stateless deployments for various operating systems.
|
||||
@@ -48,3 +53,11 @@ Cutting a new series:
|
||||
wrong series.
|
||||
|
||||
Patch releases land on branch `X.Y` and are tagged `X.Y.Z` there; no `VERSION` edit is needed.
|
||||
|
||||
## Nightly builds
|
||||
|
||||
`.github/workflows/nightly.yml` builds the rpms (EL9, EL10, SUSE 16) and debs (Ubuntu 24.04, 26.04)
|
||||
for x86_64 and aarch64 every night that something changed, installs each set on a clean container,
|
||||
and publishes them as the `nightly` pre-release. Every bundle there holds all packages for one
|
||||
target and an `INSTALL.txt`. Pull requests that touch the packaging run the same build and tests
|
||||
without publishing. Genesis is not built there yet.
|
||||
|
||||
Reference in New Issue
Block a user