mirror of
https://github.com/xcat2/confluent.git
synced 2026-11-19 00:42:20 +00:00
461 lines
18 KiB
YAML
461 lines
18 KiB
YAML
name: Nightly
|
|
|
|
# Builds the confluent rpms (EL9, EL10, SUSE 16) and debs (Ubuntu noble,
|
|
# resolute) for x86_64 and aarch64, installs each bundle on a clean container,
|
|
# and publishes the bundles as the rolling "nightly" pre-release. Pull
|
|
# requests that touch the packaging build and test, but publish nothing.
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '23 2 * * *'
|
|
workflow_dispatch:
|
|
inputs:
|
|
publish:
|
|
description: Publish the result as the nightly pre-release
|
|
type: boolean
|
|
default: true
|
|
force:
|
|
description: Build even if nothing changed since the last nightly
|
|
type: boolean
|
|
default: false
|
|
pull_request:
|
|
paths:
|
|
- .github/workflows/nightly.yml
|
|
- .github/scripts/**
|
|
- '**/buildrpm'
|
|
- '**/buildrpm-aarch64'
|
|
- '**/builddeb'
|
|
- '**/makesetup'
|
|
- '**/setup.py.tmpl'
|
|
- '**/*.spec'
|
|
- '**/*.spec.tmpl'
|
|
- confluent_osdeploy/utils/**
|
|
- build/arm/**
|
|
- mkversion
|
|
- VERSION
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: nightly-${{ github.ref }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
env:
|
|
BROWSERFS_REPO: https://github.com/xcat2/browserfs
|
|
VINZ_REPO: https://github.com/xcat2/vinz
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
jobs:
|
|
prepare:
|
|
name: prepare
|
|
runs-on: ubuntu-26.04
|
|
outputs:
|
|
build: ${{ steps.check.outputs.build }}
|
|
version: ${{ steps.check.outputs.version }}
|
|
key: ${{ steps.check.outputs.key }}
|
|
browserfs: ${{ steps.check.outputs.browserfs }}
|
|
vinz: ${{ steps.check.outputs.vinz }}
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Version and change check
|
|
id: check
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
FORCE: ${{ inputs.force }}
|
|
# The release notes carry the commits of the last published build, so
|
|
# a scheduled run with nothing new to build stops here.
|
|
run: |
|
|
version=$(./mkversion --tilde)
|
|
browserfs=$(git ls-remote "$BROWSERFS_REPO" HEAD | cut -f1)
|
|
vinz=$(git ls-remote "$VINZ_REPO" HEAD | cut -f1)
|
|
key="confluent@$GITHUB_SHA browserfs@$browserfs vinz@$vinz"
|
|
# no release yet on the first run, and bash runs with pipefail here
|
|
last=$({ gh release view nightly --repo "$GITHUB_REPOSITORY" --json body -q .body 2>/dev/null || true; } |
|
|
sed -n 's/^<!-- build: \(.*\) -->$/\1/p')
|
|
build=true
|
|
if [ "$GITHUB_EVENT_NAME" != pull_request ] && [ "$FORCE" != true ] && [ "$key" = "$last" ]; then
|
|
echo "::notice::Nothing changed since the last nightly ($last)"
|
|
build=false
|
|
fi
|
|
{
|
|
echo "build=$build"
|
|
echo "version=$version"
|
|
echo "key=$key"
|
|
echo "browserfs=$browserfs"
|
|
echo "vinz=$vinz"
|
|
} | tee -a "$GITHUB_OUTPUT"
|
|
|
|
utils:
|
|
name: osdeploy utils (${{ matrix.set }}-${{ matrix.arch }})
|
|
needs: prepare
|
|
if: needs.prepare.outputs.build == 'true'
|
|
runs-on: ${{ matrix.runner }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# The el8bin set must run on the oldest target userspace, so it is
|
|
# built where confluent_osdeploy/buildrpm builds it: el7 for x86_64,
|
|
# el8 for aarch64. CentOS 7 is too old for the node-based actions,
|
|
# so these run the image by hand instead of as a job container.
|
|
include:
|
|
- { set: el8, arch: x86_64, image: "centos:7", runner: ubuntu-26.04 }
|
|
- { set: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
|
|
- { set: el8, arch: aarch64, image: "almalinux:8", runner: ubuntu-26.04-arm }
|
|
- { set: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Build utils
|
|
run: docker run --rm -v "$PWD:/src" -w /src ${{ matrix.image }} bash .github/scripts/build-utils.sh /src/utilsbin
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: utils-${{ matrix.set }}-${{ matrix.arch }}
|
|
path: utilsbin/
|
|
retention-days: 3
|
|
|
|
rpms:
|
|
name: RPMs (${{ matrix.dist }}-${{ matrix.arch }})
|
|
needs: prepare
|
|
if: needs.prepare.outputs.build == 'true'
|
|
runs-on: ${{ matrix.runner }}
|
|
container: ${{ matrix.image }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# noarch rpms are built in the x86_64 leg only, and shared by both
|
|
include:
|
|
- { dist: el9, arch: x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
|
|
- { dist: el9, arch: aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
|
|
- { dist: el10, arch: x86_64, image: "almalinux:10", runner: ubuntu-26.04 }
|
|
- { dist: el10, arch: aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm }
|
|
- { dist: suse16, arch: x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 }
|
|
- { dist: suse16, arch: aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm }
|
|
steps:
|
|
- name: Install git
|
|
# without git in the container, checkout downloads a tarball and
|
|
# mkversion has no history to describe
|
|
run: |
|
|
if command -v zypper > /dev/null; then
|
|
zypper -n install --no-recommends git-core tar gzip
|
|
else
|
|
dnf -y --setopt=install_weak_deps=False install git-core
|
|
fi
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Build RPMs
|
|
run: bash .github/scripts/build-rpms.sh ${{ matrix.arch == 'aarch64' && '--arch-only' || '' }}
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: rpms-${{ matrix.dist }}-${{ matrix.arch }}
|
|
path: ~/rpmbuild/RPMS/
|
|
retention-days: 3
|
|
|
|
osdeploy:
|
|
name: osdeploy RPM (${{ matrix.arch }})
|
|
needs: utils
|
|
runs-on: ubuntu-26.04
|
|
container: almalinux:9
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
arch: [x86_64, aarch64]
|
|
steps:
|
|
- name: Install build dependencies
|
|
run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core tar xz cpio
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
fetch-depth: 0
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
name: utils-el8-${{ matrix.arch }}
|
|
path: el8utils
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
name: utils-el9-${{ matrix.arch }}
|
|
path: el9utils
|
|
- name: Build osdeploy RPM
|
|
run: |
|
|
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
|
bash .github/scripts/build-osdeploy.sh ${{ matrix.arch }} el8utils el9utils
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: rpms-osdeploy-${{ matrix.arch }}
|
|
path: ~/rpmbuild/RPMS/
|
|
retention-days: 3
|
|
|
|
ipxe:
|
|
name: iPXE RPM
|
|
needs: prepare
|
|
if: needs.prepare.outputs.build == 'true'
|
|
runs-on: ubuntu-26.04
|
|
container: almalinux:9
|
|
steps:
|
|
- name: Install build dependencies
|
|
run: dnf -y --setopt=install_weak_deps=False install rpm-build tar gzip
|
|
- uses: actions/checkout@v7
|
|
- name: Build iPXE RPM
|
|
run: bash .github/scripts/build-ipxe.sh
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: rpms-ipxe
|
|
path: ~/rpmbuild/RPMS/
|
|
retention-days: 3
|
|
|
|
go:
|
|
name: browserfs and vinz RPMs (${{ matrix.arch }})
|
|
needs: prepare
|
|
if: needs.prepare.outputs.build == 'true'
|
|
runs-on: ${{ matrix.runner }}
|
|
container: almalinux:10
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# static (CGO_ENABLED=0) binaries, so one build serves every distro
|
|
include:
|
|
- { arch: x86_64, runner: ubuntu-26.04 }
|
|
- { arch: aarch64, runner: ubuntu-26.04-arm }
|
|
steps:
|
|
- name: Install build dependencies
|
|
run: dnf -y --setopt=install_weak_deps=False install rpm-build git-core golang tar
|
|
- uses: actions/checkout@v7
|
|
- name: Build RPMs
|
|
run: |
|
|
bash .github/scripts/build-go-rpm.sh browserfs "$BROWSERFS_REPO" ${{ needs.prepare.outputs.browserfs }}
|
|
bash .github/scripts/build-go-rpm.sh vinz "$VINZ_REPO" ${{ needs.prepare.outputs.vinz }}
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: rpms-go-${{ matrix.arch }}
|
|
path: ~/rpmbuild/RPMS/
|
|
retention-days: 3
|
|
|
|
debs:
|
|
name: debs (${{ matrix.release }}-${{ matrix.arch }})
|
|
needs: [osdeploy, ipxe, go]
|
|
runs-on: ${{ matrix.runner }}
|
|
container: ubuntu:${{ matrix.release }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
release: [noble, resolute]
|
|
arch: [amd64, arm64]
|
|
include:
|
|
# alien only converts rpms for the architecture it runs on
|
|
- { arch: amd64, rpmarch: x86_64, runner: ubuntu-26.04 }
|
|
- { arch: arm64, rpmarch: aarch64, runner: ubuntu-26.04-arm }
|
|
steps:
|
|
- name: Install git
|
|
# ca-certificates is only a Recommends of git, and checkout clones over https
|
|
run: apt-get update && apt-get -y --no-install-recommends install git ca-certificates
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
fetch-depth: 0
|
|
# noarch, so converted in the amd64 leg only, like the noarch rpms
|
|
- uses: actions/download-artifact@v8
|
|
if: matrix.arch == 'amd64'
|
|
with:
|
|
pattern: rpms-osdeploy-*
|
|
path: alien
|
|
merge-multiple: true
|
|
- uses: actions/download-artifact@v8
|
|
if: matrix.arch == 'amd64'
|
|
with:
|
|
name: rpms-ipxe
|
|
path: alien
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
name: rpms-go-${{ matrix.rpmarch }}
|
|
path: alien
|
|
- name: Build debs
|
|
run: bash .github/scripts/build-debs.sh ${{ matrix.arch == 'arm64' && '--arch-only' || '' }} debout alien
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: debs-${{ matrix.release }}-${{ matrix.arch }}
|
|
path: debout/*.deb
|
|
retention-days: 3
|
|
if-no-files-found: error
|
|
|
|
bundle:
|
|
name: assemble bundles
|
|
needs: [prepare, rpms, osdeploy, ipxe, go, debs]
|
|
runs-on: ubuntu-26.04
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
- name: Install tools
|
|
run: sudo apt-get update && sudo apt-get -y --no-install-recommends install createrepo-c dpkg-dev
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
path: art
|
|
- name: Compose bundles
|
|
env:
|
|
VERSION: ${{ needs.prepare.outputs.version }}
|
|
run: |
|
|
set -e
|
|
pick() { find "$@" -name '*.rpm' ! -name '*.src.rpm'; }
|
|
for dist in el9 el10 suse16; do
|
|
for arch in x86_64 aarch64; do
|
|
mapfile -t files < <(
|
|
pick "art/rpms-$dist-x86_64" -name '*.noarch.rpm'
|
|
pick "art/rpms-$dist-$arch" -name "*.$arch.rpm"
|
|
pick art/rpms-osdeploy-* art/rpms-ipxe "art/rpms-go-$arch")
|
|
bash .github/scripts/mkbundle.sh "$dist-$arch" "$VERSION" bundles "${files[@]}"
|
|
done
|
|
done
|
|
# Architecture: all debs come from the amd64 leg only
|
|
pickdeb() { # <dir> <all|arch>
|
|
for f in "$1"/*.deb; do
|
|
[ "$(dpkg-deb -f "$f" Architecture)" = "$2" ] && echo "$f"
|
|
done
|
|
return 0
|
|
}
|
|
for release in noble resolute; do
|
|
for arch in amd64 arm64; do
|
|
mapfile -t files < <(
|
|
pickdeb "art/debs-$release-amd64" all
|
|
pickdeb "art/debs-$release-$arch" "$arch")
|
|
bash .github/scripts/mkbundle.sh "$release-$arch" "$VERSION" bundles "${files[@]}"
|
|
done
|
|
done
|
|
- name: Write run summary
|
|
env:
|
|
VERSION: ${{ needs.prepare.outputs.version }}
|
|
run: |
|
|
{
|
|
echo "## confluent nightly $VERSION"
|
|
echo
|
|
for b in bundles/*.tar.gz; do
|
|
echo "<details><summary>$(basename "$b")</summary>"
|
|
echo
|
|
tar tzf "$b" | sed -n 's|^[^/]*/\([^/]*\.\(rpm\|deb\)\)$|- \1|p' | sort
|
|
echo
|
|
echo "</details>"
|
|
done
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
- uses: actions/upload-artifact@v7
|
|
with:
|
|
name: bundles
|
|
path: bundles/*.tar.gz
|
|
retention-days: 7
|
|
if-no-files-found: error
|
|
|
|
smoketest:
|
|
name: smoketest (${{ matrix.target }})
|
|
needs: bundle
|
|
runs-on: ${{ matrix.runner }}
|
|
container: ${{ matrix.image }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- { target: el9-x86_64, image: "almalinux:9", runner: ubuntu-26.04 }
|
|
- { target: el9-aarch64, image: "almalinux:9", runner: ubuntu-26.04-arm }
|
|
- { target: el10-x86_64, image: "almalinux:10", runner: ubuntu-26.04 }
|
|
- { target: el10-aarch64, image: "almalinux:10", runner: ubuntu-26.04-arm }
|
|
- { target: suse16-x86_64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04 }
|
|
- { target: suse16-aarch64, image: "registry.opensuse.org/opensuse/leap:16.0", runner: ubuntu-26.04-arm }
|
|
- { target: noble-amd64, image: "ubuntu:noble", runner: ubuntu-26.04 }
|
|
- { target: noble-arm64, image: "ubuntu:noble", runner: ubuntu-26.04-arm }
|
|
- { target: resolute-amd64, image: "ubuntu:resolute", runner: ubuntu-26.04 }
|
|
- { target: resolute-arm64, image: "ubuntu:resolute", runner: ubuntu-26.04-arm }
|
|
steps:
|
|
- name: Install tar
|
|
run: |
|
|
if command -v zypper > /dev/null; then
|
|
zypper -n install --no-recommends tar gzip
|
|
elif command -v dnf > /dev/null; then
|
|
dnf -y --setopt=install_weak_deps=False install tar gzip
|
|
fi
|
|
- uses: actions/checkout@v7
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
name: bundles
|
|
path: bundles
|
|
- name: Install and smoke test
|
|
run: |
|
|
tar xzf bundles/confluent-nightly-${{ matrix.target }}.tar.gz
|
|
bash .github/scripts/smoketest.sh confluent-nightly-${{ matrix.target }}
|
|
|
|
publish:
|
|
name: publish nightly release
|
|
needs: [prepare, smoketest]
|
|
if: >-
|
|
needs.prepare.outputs.build == 'true' &&
|
|
(github.event_name == 'schedule' ||
|
|
(github.event_name == 'workflow_dispatch' && inputs.publish))
|
|
runs-on: ubuntu-26.04
|
|
permissions:
|
|
contents: write
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
GH_REPO: ${{ github.repository }}
|
|
VERSION: ${{ needs.prepare.outputs.version }}
|
|
KEY: ${{ needs.prepare.outputs.key }}
|
|
BROWSERFS: ${{ needs.prepare.outputs.browserfs }}
|
|
VINZ: ${{ needs.prepare.outputs.vinz }}
|
|
steps:
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
name: bundles
|
|
path: bundles
|
|
- name: Write release notes
|
|
run: |
|
|
url=https://github.com/$GITHUB_REPOSITORY
|
|
cat > notes.md <<EOF
|
|
Unofficial development build of $url/commit/$GITHUB_SHA, rebuilt every night when something changed. Packages are unsigned and not meant for production.
|
|
|
|
Built on $(date -u '+%Y-%m-%d at %H:%M UTC').
|
|
|
|
| Bundle | Target |
|
|
|---|---|
|
|
EOF
|
|
for b in bundles/*.tar.gz; do
|
|
n=$(basename "$b")
|
|
t=${n#confluent-nightly-}
|
|
echo "| [$n]($url/releases/download/nightly/$n) | ${t%.tar.gz} |" >> notes.md
|
|
done
|
|
cat >> notes.md <<EOF
|
|
|
|
Each bundle holds every package for its target plus repository metadata, so the directory also works as a dnf, zypper or apt repository. EL example:
|
|
|
|
\`\`\`
|
|
curl -LO $url/releases/download/nightly/confluent-nightly-el9-x86_64.tar.gz
|
|
tar xf confluent-nightly-el9-x86_64.tar.gz
|
|
dnf install epel-release && crb enable
|
|
dnf install ./confluent-nightly-el9-x86_64/*.rpm
|
|
\`\`\`
|
|
|
|
See INSTALL.txt in each bundle for the other distributions.
|
|
|
|
Built from:
|
|
- confluent $GITHUB_SHA
|
|
- browserfs $BROWSERFS_REPO/commit/$BROWSERFS
|
|
- vinz $VINZ_REPO/commit/$VINZ
|
|
- iPXE prebuilt release roms (see ipxe/confluent_ipxe.spec)
|
|
|
|
<!-- build: $KEY -->
|
|
EOF
|
|
cat notes.md
|
|
- name: Publish
|
|
# Move the tag and replace the assets in place, so the release and its
|
|
# download URLs never disappear between nights.
|
|
run: |
|
|
set -e
|
|
if gh release view nightly > /dev/null 2>&1; then
|
|
gh api -X PATCH "repos/$GH_REPO/git/refs/tags/nightly" -f sha="$GITHUB_SHA" -F force=true > /dev/null
|
|
gh release edit nightly --title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false
|
|
for old in $(gh release view nightly --json assets -q '.assets[].name'); do
|
|
[ -f "bundles/$old" ] || gh release delete-asset nightly "$old" -y
|
|
done
|
|
gh release upload nightly bundles/*.tar.gz --clobber
|
|
else
|
|
# a stale tag without a release would keep pointing at its old commit
|
|
gh api -X DELETE "repos/$GH_REPO/git/refs/tags/nightly" > /dev/null 2>&1 || true
|
|
gh release create nightly bundles/*.tar.gz --target "$GITHUB_SHA" \
|
|
--title "Nightly $VERSION" --notes-file notes.md --prerelease --latest=false
|
|
fi
|