mirror of
https://github.com/xcat2/confluent.git
synced 2026-10-20 06:35:33 +00:00
Markus Hilger
6c48cb4fad
Build the rpms for EL9, EL10 and SUSE 16 and the debs for Ubuntu noble and resolute, each for x86_64 and aarch64, install every set on a clean container and publish them as a rolling nightly pre-release. Each release asset is one target's bundle: all of its packages, repository metadata and an INSTALL.txt. Besides the in-tree components this packages confluent_ipxe from the iPXE release roms and the browserfs and vinz helpers from their own repositories. The osdeploy utils are built where buildrpm builds them, el7 for the x86_64 el8bin set, so they still run on the oldest targets. Genesis is left out for now. Noarch packages are built once, in the x86_64 legs, and the aarch64 legs build only what is architecture specific. Build tools are installed without weak dependencies, while the smoke tests keep the distribution defaults, since they stand in for a user's install. Scheduled runs skip the build when neither confluent nor the helper repositories moved since the last published nightly. Pull requests that touch the packaging build and test without publishing.
459 lines
18 KiB
YAML
459 lines
18 KiB
YAML
name: Nightly
|
|
|
|
# Builds the confluent rpms (EL9, EL10, SUSE 16) and debs (Ubuntu noble,
|
|
# resolute) for x86_64 and aarch64, installs each bundle on a clean container,
|
|
# and publishes the bundles as the rolling "nightly" pre-release. Pull
|
|
# requests that touch the packaging build and test, but publish nothing.
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '23 2 * * *'
|
|
workflow_dispatch:
|
|
inputs:
|
|
publish:
|
|
description: Publish the result as the nightly pre-release
|
|
type: boolean
|
|
default: true
|
|
force:
|
|
description: Build even if nothing changed since the last nightly
|
|
type: boolean
|
|
default: false
|
|
pull_request:
|
|
paths:
|
|
- .github/workflows/nightly.yml
|
|
- .github/scripts/**
|
|
- '**/buildrpm'
|
|
- '**/buildrpm-aarch64'
|
|
- '**/builddeb'
|
|
- '**/makesetup'
|
|
- '**/setup.py.tmpl'
|
|
- '**/*.spec'
|
|
- '**/*.spec.tmpl'
|
|
- confluent_osdeploy/utils/**
|
|
- build/arm/**
|
|
- mkversion
|
|
- VERSION
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: nightly-${{ github.ref }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
env:
|
|
BROWSERFS_REPO: https://github.com/xcat2/browserfs
|
|
VINZ_REPO: https://github.com/xcat2/vinz
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
jobs:
|
|
prepare:
|
|
name: prepare
|
|
runs-on: ubuntu-26.04
|
|
outputs:
|
|
build: ${{ steps.check.outputs.build }}
|
|
version: ${{ steps.check.outputs.version }}
|
|
key: ${{ steps.check.outputs.key }}
|
|
browserfs: ${{ steps.check.outputs.browserfs }}
|
|
vinz: ${{ steps.check.outputs.vinz }}
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Version and change check
|
|
id: check
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
FORCE: ${{ inputs.force }}
|
|
# The release notes carry the commits of the last published build, so
|
|
# a scheduled run with nothing new to build stops here.
|
|
run: |
|
|
version=$(./mkversion --tilde)
|
|
browserfs=$(git ls-remote "$BROWSERFS_REPO" HEAD | cut -f1)
|
|
vinz=$(git ls-remote "$VINZ_REPO" HEAD | cut -f1)
|
|
key="confluent@$GITHUB_SHA browserfs@$browserfs vinz@$vinz"
|
|
# no release yet on the first run, and bash runs with pipefail here
|
|
last=$({ gh release view nightly --repo "$GITHUB_REPOSITORY" --json body -q .body 2>/dev/null || true; } |
|
|
sed -n 's/^<!-- build: \(.*\) -->$/\1/p')
|
|
build=true
|
|
if [ "$GITHUB_EVENT_NAME" != pull_request ] && [ "$FORCE" != true ] && [ "$key" = "$last" ]; then
|
|
echo "::notice::Nothing changed since the last nightly ($last)"
|
|
build=false
|
|
fi
|
|
{
|
|
echo "build=$build"
|
|
echo "version=$version"
|
|
echo "key=$key"
|
|
echo "browserfs=$browserfs"
|
|
echo "vinz=$vinz"
|
|
} | tee -a "$GITHUB_OUTPUT"
|
|
|
|
utils:
|
|
name: osdeploy utils (${{ matrix.set }}-${{ matrix.arch }})
|
|
needs: prepare
|
|
if: needs.prepare.outputs.build == 'true'
|
|
runs-on: ${{ matrix.runner }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# The el8bin set must run on the oldest target userspace, so it is
|
|
# built where confluent_osdeploy/buildrpm builds it: el7 for x86_64,
|
|
# el8 for aarch64. CentOS 7 is too old for the node-based actions,
|
|
# so these run the image by hand instead of as a job container.
|
|
include:
|
|
- { set: el8, arch: x86_64, image: "centos:7", runner: ubuntu-26.04 }
|
|
- { set: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
|
|
- { set: el8, arch: aarch64, image: "almalinux:8", runner: ubuntu-26.04-arm }
|
|
- { set: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Build utils
|
|
run: docker run --rm -v "$PWD:/src" -w /src ${{ matrix.image }} bash .github/scripts/build-utils.sh /src/utilsbin
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: utils-${{ matrix.set }}-${{ matrix.arch }}
|
|
path: utilsbin/
|
|
retention-days: 3
|
|
|
|
rpms:
|
|
name: RPMs (${{ matrix.dist }}-${{ matrix.arch }})
|
|
needs: prepare
|
|
if: needs.prepare.outputs.build == 'true'
|
|
runs-on: ${{ matrix.runner }}
|
|
container: ${{ matrix.image }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# noarch rpms are built in the x86_64 leg only, and shared by both
|
|
include:
|
|
- { dist: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
|
|
- { dist: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
|
|
- { dist: el10, arch: x86_64, image: "almalinux:10", runner: ubuntu-26.04 }
|
|
- { dist: el10, arch: aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm }
|
|
- { dist: suse16, arch: x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 }
|
|
- { dist: suse16, arch: aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm }
|
|
steps:
|
|
- name: Install git
|
|
# without git in the container, checkout downloads a tarball and
|
|
# mkversion has no history to describe
|
|
run: |
|
|
if command -v zypper > /dev/null; then
|
|
zypper -n install --no-recommends git-core tar gzip
|
|
else
|
|
dnf -y --setopt=install_weak_deps=False install git-core
|
|
fi
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Build RPMs
|
|
run: bash .github/scripts/build-rpms.sh ${{ matrix.arch == 'aarch64' && '--arch-only' || '' }}
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: rpms-${{ matrix.dist }}-${{ matrix.arch }}
|
|
path: ~/rpmbuild/RPMS/
|
|
retention-days: 3
|
|
|
|
osdeploy:
|
|
name: osdeploy RPM (${{ matrix.arch }})
|
|
needs: utils
|
|
runs-on: ubuntu-26.04
|
|
container: almalinux:9
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
arch: [x86_64, aarch64]
|
|
steps:
|
|
- name: Install build dependencies
|
|
run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core tar xz cpio
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
fetch-depth: 0
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
name: utils-el8-${{ matrix.arch }}
|
|
path: el8utils
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
name: utils-el9-${{ matrix.arch }}
|
|
path: el9utils
|
|
- name: Build osdeploy RPM
|
|
run: |
|
|
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
|
bash .github/scripts/build-osdeploy.sh ${{ matrix.arch }} el8utils el9utils
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: rpms-osdeploy-${{ matrix.arch }}
|
|
path: ~/rpmbuild/RPMS/
|
|
retention-days: 3
|
|
|
|
ipxe:
|
|
name: iPXE RPM
|
|
needs: prepare
|
|
if: needs.prepare.outputs.build == 'true'
|
|
runs-on: ubuntu-26.04
|
|
container: almalinux:9
|
|
steps:
|
|
- name: Install build dependencies
|
|
run: dnf -y --setopt=install_weak_deps=False install rpm-build tar gzip
|
|
- uses: actions/checkout@v7
|
|
- name: Build iPXE RPM
|
|
run: bash .github/scripts/build-ipxe.sh
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: rpms-ipxe
|
|
path: ~/rpmbuild/RPMS/
|
|
retention-days: 3
|
|
|
|
go:
|
|
name: browserfs and vinz RPMs (${{ matrix.arch }})
|
|
needs: prepare
|
|
if: needs.prepare.outputs.build == 'true'
|
|
runs-on: ${{ matrix.runner }}
|
|
container: almalinux:10
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# static (CGO_ENABLED=0) binaries, so one build serves every distro
|
|
include:
|
|
- { arch: x86_64, runner: ubuntu-26.04 }
|
|
- { arch: aarch64, runner: ubuntu-26.04-arm }
|
|
steps:
|
|
- name: Install build dependencies
|
|
run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core golang tar
|
|
- uses: actions/checkout@v7
|
|
- name: Build RPMs
|
|
run: |
|
|
bash .github/scripts/build-go-rpm.sh browserfs "$BROWSERFS_REPO" ${{ needs.prepare.outputs.browserfs }}
|
|
bash .github/scripts/build-go-rpm.sh vinz "$VINZ_REPO" ${{ needs.prepare.outputs.vinz }}
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: rpms-go-${{ matrix.arch }}
|
|
path: ~/rpmbuild/RPMS/
|
|
retention-days: 3
|
|
|
|
debs:
|
|
name: debs (${{ matrix.release }}-${{ matrix.arch }})
|
|
needs: [osdeploy, ipxe, go]
|
|
runs-on: ${{ matrix.runner }}
|
|
container: ubuntu:${{ matrix.release }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
release: [noble, resolute]
|
|
arch: [amd64, arm64]
|
|
include:
|
|
# alien only converts rpms for the architecture it runs on
|
|
- { arch: amd64, rpmarch: x86_64, runner: ubuntu-26.04 }
|
|
- { arch: arm64, rpmarch: aarch64, runner: ubuntu-26.04-arm }
|
|
steps:
|
|
- name: Install git
|
|
# ca-certificates is only a Recommends of git, and checkout clones over https
|
|
run: apt-get update && apt-get -y --no-install-recommends install git ca-certificates
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
fetch-depth: 0
|
|
# noarch, so converted in the amd64 leg only, like the noarch rpms
|
|
- uses: actions/download-artifact@v8
|
|
if: matrix.arch == 'amd64'
|
|
with:
|
|
pattern: rpms-osdeploy-*
|
|
path: alien
|
|
merge-multiple: true
|
|
- uses: actions/download-artifact@v8
|
|
if: matrix.arch == 'amd64'
|
|
with:
|
|
name: rpms-ipxe
|
|
path: alien
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
name: rpms-go-${{ matrix.rpmarch }}
|
|
path: alien
|
|
- name: Build debs
|
|
run: bash .github/scripts/build-debs.sh ${{ matrix.arch == 'arm64' && '--arch-only' || '' }} debout alien
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: debs-${{ matrix.release }}-${{ matrix.arch }}
|
|
path: debout/*.deb
|
|
retention-days: 3
|
|
if-no-files-found: error
|
|
|
|
bundle:
|
|
name: assemble bundles
|
|
needs: [prepare, rpms, osdeploy, ipxe, go, debs]
|
|
runs-on: ubuntu-26.04
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Install tools
|
|
run: sudo apt-get update && sudo apt-get -y --no-install-recommends install createrepo-c dpkg-dev
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
path: art
|
|
- name: Compose bundles
|
|
env:
|
|
VERSION: ${{ needs.prepare.outputs.version }}
|
|
run: |
|
|
set -e
|
|
pick() { find "$@" -name '*.rpm' ! -name '*.src.rpm'; }
|
|
for dist in el9 el10 suse16; do
|
|
for arch in x86_64 aarch64; do
|
|
mapfile -t files < <(
|
|
pick "art/rpms-$dist-x86_64" -name '*.noarch.rpm'
|
|
pick "art/rpms-$dist-$arch" -name "*.$arch.rpm"
|
|
pick art/rpms-osdeploy-* art/rpms-ipxe "art/rpms-go-$arch")
|
|
bash .github/scripts/mkbundle.sh "$dist-$arch" "$VERSION" bundles "${files[@]}"
|
|
done
|
|
done
|
|
# Architecture: all debs come from the amd64 leg only
|
|
pickdeb() { # <dir> <all|arch>
|
|
for f in "$1"/*.deb; do
|
|
[ "$(dpkg-deb -f "$f" Architecture)" = "$2" ] && echo "$f"
|
|
done
|
|
return 0
|
|
}
|
|
for release in noble resolute; do
|
|
for arch in amd64 arm64; do
|
|
mapfile -t files < <(
|
|
pickdeb "art/debs-$release-amd64" all
|
|
pickdeb "art/debs-$release-$arch" "$arch")
|
|
bash .github/scripts/mkbundle.sh "$release-$arch" "$VERSION" bundles "${files[@]}"
|
|
done
|
|
done
|
|
- name: Write run summary
|
|
env:
|
|
VERSION: ${{ needs.prepare.outputs.version }}
|
|
run: |
|
|
{
|
|
echo "## confluent nightly $VERSION"
|
|
echo
|
|
for b in bundles/*.tar.gz; do
|
|
echo "<details><summary>$(basename "$b")</summary>"
|
|
echo
|
|
tar tzf "$b" | sed -n 's|^[^/]*/\([^/]*\.\(rpm\|deb\)\)$|- \1|p' | sort
|
|
echo
|
|
echo "</details>"
|
|
done
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: bundles
|
|
path: bundles/*.tar.gz
|
|
retention-days: 7
|
|
if-no-files-found: error
|
|
|
|
smoketest:
|
|
name: smoketest (${{ matrix.target }})
|
|
needs: bundle
|
|
runs-on: ${{ matrix.runner }}
|
|
container: ${{ matrix.image }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- { target: el9-x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
|
|
- { target: el9-aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
|
|
- { target: el10-x86_64, image: "almalinux:10", runner: ubuntu-26.04 }
|
|
- { target: el10-aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm }
|
|
- { target: suse16-x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 }
|
|
- { target: suse16-aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm }
|
|
- { target: noble-amd64, image: "ubuntu:noble", runner: ubuntu-26.04 }
|
|
- { target: noble-arm64, image: "ubuntu:noble", runner: ubuntu-26.04-arm }
|
|
- { target: resolute-amd64, image: "ubuntu:resolute", runner: ubuntu-26.04 }
|
|
- { target: resolute-arm64, image: "ubuntu:resolute", runner: ubuntu-26.04-arm }
|
|
steps:
|
|
- name: Install tar
|
|
run: |
|
|
if command -v zypper > /dev/null; then
|
|
zypper -n install --no-recommends tar gzip
|
|
elif command -v dnf > /dev/null; then
|
|
dnf -y --setopt=install_weak_deps=False install tar gzip
|
|
fi
|
|
- uses: actions/checkout@v7
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
name: bundles
|
|
path: bundles
|
|
- name: Install and smoke test
|
|
run: |
|
|
tar xzf bundles/confluent-nightly-${{ matrix.target }}.tar.gz
|
|
bash .github/scripts/smoketest.sh confluent-nightly-${{ matrix.target }}
|
|
|
|
publish:
|
|
name: publish nightly release
|
|
needs: [prepare, smoketest]
|
|
if: >-
|
|
needs.prepare.outputs.build == 'true' &&
|
|
(github.event_name == 'schedule' ||
|
|
(github.event_name == 'workflow_dispatch' && inputs.publish))
|
|
runs-on: ubuntu-26.04
|
|
permissions:
|
|
contents: write
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
GH_REPO: ${{ github.repository }}
|
|
VERSION: ${{ needs.prepare.outputs.version }}
|
|
KEY: ${{ needs.prepare.outputs.key }}
|
|
BROWSERFS: ${{ needs.prepare.outputs.browserfs }}
|
|
VINZ: ${{ needs.prepare.outputs.vinz }}
|
|
steps:
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
name: bundles
|
|
path: bundles
|
|
- name: Write release notes
|
|
run: |
|
|
url=https://github.com/$GITHUB_REPOSITORY
|
|
cat > notes.md <<EOF
|
|
Unofficial development build of $url/commit/$GITHUB_SHA, rebuilt every night when something changed. Packages are unsigned and not meant for production.
|
|
|
|
| Bundle | Target |
|
|
|---|---|
|
|
EOF
|
|
for b in bundles/*.tar.gz; do
|
|
n=$(basename "$b")
|
|
t=${n#confluent-nightly-}
|
|
echo "| [$n]($url/releases/download/nightly/$n) | ${t%.tar.gz} |" >> notes.md
|
|
done
|
|
cat >> notes.md <<EOF
|
|
|
|
Each bundle holds every package for its target plus repository metadata, so the directory also works as a dnf, zypper or apt repository. EL example:
|
|
|
|
\`\`\`
|
|
curl -LO $url/releases/download/nightly/confluent-nightly-el9-x86_64.tar.gz
|
|
tar xf confluent-nightly-el9-x86_64.tar.gz
|
|
dnf install epel-release && crb enable
|
|
dnf install ./confluent-nightly-el9-x86_64/*.rpm
|
|
\`\`\`
|
|
|
|
See INSTALL.txt in each bundle for the other distributions.
|
|
|
|
Built from:
|
|
- confluent $GITHUB_SHA
|
|
- browserfs $BROWSERFS_REPO/commit/$BROWSERFS
|
|
- vinz $VINZ_REPO/commit/$VINZ
|
|
- iPXE prebuilt release roms (see ipxe/confluent_ipxe.spec)
|
|
|
|
<!-- build: $KEY -->
|
|
EOF
|
|
cat notes.md
|
|
- name: Publish
|
|
# Move the tag and replace the assets in place, so the release and its
|
|
# download URLs never disappear between nights.
|
|
run: |
|
|
set -e
|
|
if gh release view nightly > /dev/null 2>&1; then
|
|
gh api -X PATCH "repos/$GH_REPO/git/refs/tags/nightly" -f sha="$GITHUB_SHA" -F force=true > /dev/null
|
|
gh release edit nightly --title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false
|
|
for old in $(gh release view nightly --json assets -q '.assets[].name'); do
|
|
[ -f "bundles/$old" ] || gh release delete-asset nightly "$old" -y
|
|
done
|
|
gh release upload nightly bundles/*.tar.gz --clobber
|
|
else
|
|
# a stale tag without a release would keep pointing at its old commit
|
|
gh api -X DELETE "repos/$GH_REPO/git/refs/tags/nightly" > /dev/null 2>&1 || true
|
|
gh release create nightly bundles/*.tar.gz --target "$GITHUB_SHA" \
|
|
--title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false
|
|
fi
|