2
0
mirror of https://github.com/xcat2/confluent.git synced 2026-11-19 00:42:20 +00:00
Files
confluent/.github/workflows/nightly.yml
T
2026-10-24 03:22:18 +02:00

461 lines
18 KiB
YAML

name: Nightly
# Builds the confluent rpms (EL9, EL10, SUSE 16) and debs (Ubuntu noble,
# resolute) for x86_64 and aarch64, installs each bundle on a clean container,
# and publishes the bundles as the rolling "nightly" pre-release. Pull
# requests that touch the packaging build and test, but publish nothing.
on:
schedule:
- cron: '23 2 * * *'
workflow_dispatch:
inputs:
publish:
description: Publish the result as the nightly pre-release
type: boolean
default: true
force:
description: Build even if nothing changed since the last nightly
type: boolean
default: false
pull_request:
paths:
- .github/workflows/nightly.yml
- .github/scripts/**
- '**/buildrpm'
- '**/buildrpm-aarch64'
- '**/builddeb'
- '**/makesetup'
- '**/setup.py.tmpl'
- '**/*.spec'
- '**/*.spec.tmpl'
- confluent_osdeploy/utils/**
- build/arm/**
- mkversion
- VERSION
permissions:
contents: read
concurrency:
group: nightly-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
BROWSERFS_REPO: https://github.com/xcat2/browserfs
VINZ_REPO: https://github.com/xcat2/vinz
defaults:
run:
shell: bash
jobs:
prepare:
name: prepare
runs-on: ubuntu-26.04
outputs:
build: ${{ steps.check.outputs.build }}
version: ${{ steps.check.outputs.version }}
key: ${{ steps.check.outputs.key }}
browserfs: ${{ steps.check.outputs.browserfs }}
vinz: ${{ steps.check.outputs.vinz }}
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
- name: Version and change check
id: check
env:
GH_TOKEN: ${{ github.token }}
FORCE: ${{ inputs.force }}
# The release notes carry the commits of the last published build, so
# a scheduled run with nothing new to build stops here.
run: |
version=$(./mkversion --tilde)
browserfs=$(git ls-remote "$BROWSERFS_REPO" HEAD | cut -f1)
vinz=$(git ls-remote "$VINZ_REPO" HEAD | cut -f1)
key="confluent@$GITHUB_SHA browserfs@$browserfs vinz@$vinz"
# no release yet on the first run, and bash runs with pipefail here
last=$({ gh release view nightly --repo "$GITHUB_REPOSITORY" --json body -q .body 2>/dev/null || true; } |
sed -n 's/^<!-- build: \(.*\) -->$/\1/p')
build=true
if [ "$GITHUB_EVENT_NAME" != pull_request ] && [ "$FORCE" != true ] && [ "$key" = "$last" ]; then
echo "::notice::Nothing changed since the last nightly ($last)"
build=false
fi
{
echo "build=$build"
echo "version=$version"
echo "key=$key"
echo "browserfs=$browserfs"
echo "vinz=$vinz"
} | tee -a "$GITHUB_OUTPUT"
utils:
name: osdeploy utils (${{ matrix.set }}-${{ matrix.arch }})
needs: prepare
if: needs.prepare.outputs.build == 'true'
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
# The el8bin set must run on the oldest target userspace, so it is
# built where confluent_osdeploy/buildrpm builds it: el7 for x86_64,
# el8 for aarch64. CentOS 7 is too old for the node-based actions,
# so these run the image by hand instead of as a job container.
include:
- { set: el8, arch: x86_64, image: "centos:7", runner: ubuntu-26.04 }
- { set: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
- { set: el8, arch: aarch64, image: "almalinux:8", runner: ubuntu-26.04-arm }
- { set: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
steps:
- uses: actions/checkout@v7
- name: Build utils
run: docker run --rm -v "$PWD:/src" -w /src ${{ matrix.image }} bash .github/scripts/build-utils.sh /src/utilsbin
- uses: actions/upload-artifact@v7
with:
name: utils-${{ matrix.set }}-${{ matrix.arch }}
path: utilsbin/
retention-days: 3
rpms:
name: RPMs (${{ matrix.dist }}-${{ matrix.arch }})
needs: prepare
if: needs.prepare.outputs.build == 'true'
runs-on: ${{ matrix.runner }}
container: ${{ matrix.image }}
strategy:
fail-fast: false
matrix:
# noarch rpms are built in the x86_64 leg only, and shared by both
include:
- { dist: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
- { dist: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
- { dist: el10, arch: x86_64, image: "almalinux:10", runner: ubuntu-26.04 }
- { dist: el10, arch: aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm }
- { dist: suse16, arch: x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 }
- { dist: suse16, arch: aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm }
steps:
- name: Install git
# without git in the container, checkout downloads a tarball and
# mkversion has no history to describe
run: |
if command -v zypper > /dev/null; then
zypper -n install --no-recommends git-core tar gzip
else
dnf -y --setopt=install_weak_deps=False install git-core
fi
- uses: actions/checkout@v7
with:
fetch-depth: 0
- name: Build RPMs
run: bash .github/scripts/build-rpms.sh ${{ matrix.arch == 'aarch64' && '--arch-only' || '' }}
- uses: actions/upload-artifact@v7
with:
name: rpms-${{ matrix.dist }}-${{ matrix.arch }}
path: ~/rpmbuild/RPMS/
retention-days: 3
osdeploy:
name: osdeploy RPM (${{ matrix.arch }})
needs: utils
runs-on: ubuntu-26.04
container: almalinux:9
strategy:
fail-fast: false
matrix:
arch: [x86_64, aarch64]
steps:
- name: Install build dependencies
run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core tar xz cpio
- uses: actions/checkout@v7
with:
fetch-depth: 0
- uses: actions/download-artifact@v8
with:
name: utils-el8-${{ matrix.arch }}
path: el8utils
- uses: actions/download-artifact@v8
with:
name: utils-el9-${{ matrix.arch }}
path: el9utils
- name: Build osdeploy RPM
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
bash .github/scripts/build-osdeploy.sh ${{ matrix.arch }} el8utils el9utils
- uses: actions/upload-artifact@v7
with:
name: rpms-osdeploy-${{ matrix.arch }}
path: ~/rpmbuild/RPMS/
retention-days: 3
ipxe:
name: iPXE RPM
needs: prepare
if: needs.prepare.outputs.build == 'true'
runs-on: ubuntu-26.04
container: almalinux:9
steps:
- name: Install build dependencies
run: dnf -y --setopt=install_weak_deps=False install rpm-build tar gzip
- uses: actions/checkout@v7
- name: Build iPXE RPM
run: bash .github/scripts/build-ipxe.sh
- uses: actions/upload-artifact@v7
with:
name: rpms-ipxe
path: ~/rpmbuild/RPMS/
retention-days: 3
go:
name: browserfs and vinz RPMs (${{ matrix.arch }})
needs: prepare
if: needs.prepare.outputs.build == 'true'
runs-on: ${{ matrix.runner }}
container: almalinux:10
strategy:
fail-fast: false
matrix:
# static (CGO_ENABLED=0) binaries, so one build serves every distro
include:
- { arch: x86_64, runner: ubuntu-26.04 }
- { arch: aarch64, runner: ubuntu-26.04-arm }
steps:
- name: Install build dependencies
run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core golang tar
- uses: actions/checkout@v7
- name: Build RPMs
run: |
bash .github/scripts/build-go-rpm.sh browserfs "$BROWSERFS_REPO" ${{ needs.prepare.outputs.browserfs }}
bash .github/scripts/build-go-rpm.sh vinz "$VINZ_REPO" ${{ needs.prepare.outputs.vinz }}
- uses: actions/upload-artifact@v7
with:
name: rpms-go-${{ matrix.arch }}
path: ~/rpmbuild/RPMS/
retention-days: 3
debs:
name: debs (${{ matrix.release }}-${{ matrix.arch }})
needs: [osdeploy, ipxe, go]
runs-on: ${{ matrix.runner }}
container: ubuntu:${{ matrix.release }}
strategy:
fail-fast: false
matrix:
release: [noble, resolute]
arch: [amd64, arm64]
include:
# alien only converts rpms for the architecture it runs on
- { arch: amd64, rpmarch: x86_64, runner: ubuntu-26.04 }
- { arch: arm64, rpmarch: aarch64, runner: ubuntu-26.04-arm }
steps:
- name: Install git
# ca-certificates is only a Recommends of git, and checkout clones over https
run: apt-get update && apt-get -y --no-install-recommends install git ca-certificates
- uses: actions/checkout@v7
with:
fetch-depth: 0
# noarch, so converted in the amd64 leg only, like the noarch rpms
- uses: actions/download-artifact@v8
if: matrix.arch == 'amd64'
with:
pattern: rpms-osdeploy-*
path: alien
merge-multiple: true
- uses: actions/download-artifact@v8
if: matrix.arch == 'amd64'
with:
name: rpms-ipxe
path: alien
- uses: actions/download-artifact@v8
with:
name: rpms-go-${{ matrix.rpmarch }}
path: alien
- name: Build debs
run: bash .github/scripts/build-debs.sh ${{ matrix.arch == 'arm64' && '--arch-only' || '' }} debout alien
- uses: actions/upload-artifact@v7
with:
name: debs-${{ matrix.release }}-${{ matrix.arch }}
path: debout/*.deb
retention-days: 3
if-no-files-found: error
bundle:
name: assemble bundles
needs: [prepare, rpms, osdeploy, ipxe, go, debs]
runs-on: ubuntu-26.04
steps:
- uses: actions/checkout@v7
- name: Install tools
run: sudo apt-get update && sudo apt-get -y --no-install-recommends install createrepo-c dpkg-dev
- uses: actions/download-artifact@v8
with:
path: art
- name: Compose bundles
env:
VERSION: ${{ needs.prepare.outputs.version }}
run: |
set -e
pick() { find "$@" -name '*.rpm' ! -name '*.src.rpm'; }
for dist in el9 el10 suse16; do
for arch in x86_64 aarch64; do
mapfile -t files < <(
pick "art/rpms-$dist-x86_64" -name '*.noarch.rpm'
pick "art/rpms-$dist-$arch" -name "*.$arch.rpm"
pick art/rpms-osdeploy-* art/rpms-ipxe "art/rpms-go-$arch")
bash .github/scripts/mkbundle.sh "$dist-$arch" "$VERSION" bundles "${files[@]}"
done
done
# Architecture: all debs come from the amd64 leg only
pickdeb() { # <dir> <all|arch>
for f in "$1"/*.deb; do
[ "$(dpkg-deb -f "$f" Architecture)" = "$2" ] && echo "$f"
done
return 0
}
for release in noble resolute; do
for arch in amd64 arm64; do
mapfile -t files < <(
pickdeb "art/debs-$release-amd64" all
pickdeb "art/debs-$release-$arch" "$arch")
bash .github/scripts/mkbundle.sh "$release-$arch" "$VERSION" bundles "${files[@]}"
done
done
- name: Write run summary
env:
VERSION: ${{ needs.prepare.outputs.version }}
run: |
{
echo "## confluent nightly $VERSION"
echo
for b in bundles/*.tar.gz; do
echo "<details><summary>$(basename "$b")</summary>"
echo
tar tzf "$b" | sed -n 's|^[^/]*/\([^/]*\.\(rpm\|deb\)\)$|- \1|p' | sort
echo
echo "</details>"
done
} >> "$GITHUB_STEP_SUMMARY"
- uses: actions/upload-artifact@v7
with:
name: bundles
path: bundles/*.tar.gz
retention-days: 7
if-no-files-found: error
smoketest:
name: smoketest (${{ matrix.target }})
needs: bundle
runs-on: ${{ matrix.runner }}
container: ${{ matrix.image }}
strategy:
fail-fast: false
matrix:
include:
- { target: el9-x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
- { target: el9-aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
- { target: el10-x86_64, image: "almalinux:10", runner: ubuntu-26.04 }
- { target: el10-aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm }
- { target: suse16-x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 }
- { target: suse16-aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm }
- { target: noble-amd64, image: "ubuntu:noble", runner: ubuntu-26.04 }
- { target: noble-arm64, image: "ubuntu:noble", runner: ubuntu-26.04-arm }
- { target: resolute-amd64, image: "ubuntu:resolute", runner: ubuntu-26.04 }
- { target: resolute-arm64, image: "ubuntu:resolute", runner: ubuntu-26.04-arm }
steps:
- name: Install tar
run: |
if command -v zypper > /dev/null; then
zypper -n install --no-recommends tar gzip
elif command -v dnf > /dev/null; then
dnf -y --setopt=install_weak_deps=False install tar gzip
fi
- uses: actions/checkout@v7
- uses: actions/download-artifact@v8
with:
name: bundles
path: bundles
- name: Install and smoke test
run: |
tar xzf bundles/confluent-nightly-${{ matrix.target }}.tar.gz
bash .github/scripts/smoketest.sh confluent-nightly-${{ matrix.target }}
publish:
name: publish nightly release
needs: [prepare, smoketest]
if: >-
needs.prepare.outputs.build == 'true' &&
(github.event_name == 'schedule' ||
(github.event_name == 'workflow_dispatch' && inputs.publish))
runs-on: ubuntu-26.04
permissions:
contents: write
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
VERSION: ${{ needs.prepare.outputs.version }}
KEY: ${{ needs.prepare.outputs.key }}
BROWSERFS: ${{ needs.prepare.outputs.browserfs }}
VINZ: ${{ needs.prepare.outputs.vinz }}
steps:
- uses: actions/download-artifact@v8
with:
name: bundles
path: bundles
- name: Write release notes
run: |
url=https://github.com/$GITHUB_REPOSITORY
cat > notes.md <<EOF
Unofficial development build of $url/commit/$GITHUB_SHA, rebuilt every night when something changed. Packages are unsigned and not meant for production.
Built on $(date -u '+%Y-%m-%d at %H:%M UTC').
| Bundle | Target |
|---|---|
EOF
for b in bundles/*.tar.gz; do
n=$(basename "$b")
t=${n#confluent-nightly-}
echo "| [$n]($url/releases/download/nightly/$n) | ${t%.tar.gz} |" >> notes.md
done
cat >> notes.md <<EOF
Each bundle holds every package for its target plus repository metadata, so the directory also works as a dnf, zypper or apt repository. EL example:
\`\`\`
curl -LO $url/releases/download/nightly/confluent-nightly-el9-x86_64.tar.gz
tar xf confluent-nightly-el9-x86_64.tar.gz
dnf install epel-release && crb enable
dnf install ./confluent-nightly-el9-x86_64/*.rpm
\`\`\`
See INSTALL.txt in each bundle for the other distributions.
Built from:
- confluent $GITHUB_SHA
- browserfs $BROWSERFS_REPO/commit/$BROWSERFS
- vinz $VINZ_REPO/commit/$VINZ
- iPXE prebuilt release roms (see ipxe/confluent_ipxe.spec)
<!-- build: $KEY -->
EOF
cat notes.md
- name: Publish
# Move the tag and replace the assets in place, so the release and its
# download URLs never disappear between nights.
run: |
set -e
if gh release view nightly > /dev/null 2>&1; then
gh api -X PATCH "repos/$GH_REPO/git/refs/tags/nightly" -f sha="$GITHUB_SHA" -F force=true > /dev/null
gh release edit nightly --title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false
for old in $(gh release view nightly --json assets -q '.assets[].name'); do
[ -f "bundles/$old" ] || gh release delete-asset nightly "$old" -y
done
gh release upload nightly bundles/*.tar.gz --clobber
else
# a stale tag without a release would keep pointing at its old commit
gh api -X DELETE "repos/$GH_REPO/git/refs/tags/nightly" > /dev/null 2>&1 || true
gh release create nightly bundles/*.tar.gz --target "$GITHUB_SHA" \
--title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false
fi