From 5515dfbe3e5ac3a305a60422def89f12e6318ce8 Mon Sep 17 00:00:00 2001 From: Daniel Hilst <392820+dhilst@users.noreply.github.com> Date: Tue, 1 Sep 2026 07:07:20 -0300 Subject: [PATCH] fix(build): keep build-ubunturepo as an oracle, and fix what running it found builddebs.pl was written against build-ubunturepo but never run beside it. Keeping the old script in the tree for one more cycle makes the comparison repeatable, and running the two on xcat-master-ub found four defects that no unit test would have. * builddebs.pl was not executable. github_action_xcat_test.pl runs `sudo ./builddebs.pl`, so CI would have failed with Permission denied. * .deb files were collected after every ARCHITECTURE. dpkg-genbuildinfo reads the sibling artifacts of the source package it is building, so moving the amd64 .deb away before ppc64el ran killed the second build with dpkg-genbuildinfo: error: cannot fstat file ../xcat_..._amd64.deb Collection now happens once a package's last architecture is done. * A checkout dirtied by an aborted run poisoned the next one: a leftover .changes made dpkg-genbuildinfo name an architecture the run had not reached. The dpkg output is now cleared at start. * The maintainer was xcat@xcat.org where build-ubunturepo uses xcat-build@xcat.org, and a Release file in the tree was ignored. buildrpms.pl WRITES Release, so a pipeline building both would have stamped its debs with a different release from its rpms. Both corrected. Equivalence, measured rather than assumed. Both builders were run on xcat-master-ub from the same tree: package set identical, 14/14, same names control metadata identical, 14/14 payload contents identical, 14/14 changelog.Debian differs, 2 packages -- deliberately, see below Byte-identity is NOT the criterion, because neither implementation has it: two consecutive runs of build-ubunturepo on the same tree agree on 0 of 14 packages. A .deb records the build wall-clock time in its ar member mtimes and SOURCE_DATE_EPOCH does not reach dpkg-deb here, so every run differs from every other. That is a pre-existing property of the Debian build, not a regression, and worth fixing separately. The changelog difference is build-ubunturepo being wrong. Its sed -i "s/^ -- .*/ -- $DEBFULLNAME <$DEBEMAIL> $deterministic_date/" carries no line address, so it rewrites EVERY trailer in debian/changelog: "OCF xCAT Mon, 25 Oct 2010" ships as "xCAT Build Tue, 01 Sep 2026". It falsifies the authorship and dates of the 2008 and 2010 releases. builddebs.pl rewrites only the top stanza. Matching byte-for-byte would mean reproducing the defect, so this difference stays. (xcat-vlan appeared to differ under `diff -r`; that was diff reporting dangling symlinks in both trees. Its member listing is identical.) build-ubunturepo is documented as retained-for-comparison and not to be extended. It goes once the Ubuntu CD pipelines call builddebs.pl. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com> --- build-ubunturepo | 710 ++++++++++++++++++ builddebs.pl | 40 +- docs/source/developers/guides/code/builds.rst | 12 +- 3 files changed, 753 insertions(+), 9 deletions(-) create mode 100755 build-ubunturepo mode change 100644 => 100755 builddebs.pl diff --git a/build-ubunturepo b/build-ubunturepo new file mode 100755 index 000000000..bc92165ee --- /dev/null +++ b/build-ubunturepo @@ -0,0 +1,710 @@ +#!/bin/bash +# Update GSA Ubuntu Repositories or create a local repository +# +# Author: Leonardo Tonetto (tonetto@linux.vnet.ibm.com) +# Revisor: Arif Ali (aali@ocf.co.uk) +# +# +# Getting Started: +# - Clone the xcat-core git repository under a directory named "xcat-core/src" +# - make sure reprepro is installed on the build machine +# - Run this script from the local git repository you just created. +# ./build-ubunturepo -c BUILDALL=1 + +# Usage: attr=value attr=value ... ./build-ubunturepo { -c | -d } +# PROMOTE=1 - if the attribute "PROMOTE" is specified, means an official dot release. This does not +# actually build xcat, just uploads the most recent snap build to http://xcat.org/files/xcat/ . +# If not specified, a snap build is assumed, which uploads to https://xcat.org/files/xcat/ +# PREGA=1 - use this option with PROMOTE=1 on a branch that already has a released dot release, but this +# build is a GA candidate build, not to be released yet. This will result in the tarball +# being uploaded to http://xcat.org/files/xcat/repos/apt +# (but the tarball file name will be like a released tarball, not a snap build). +# When you are ready to release this build, use PROMOTE=1 without PREGA +# BUILDALL=1 - build all rpms, whether they changed or not. Should be used for snap builds that are in +# prep for a release. +# GPGSIGN=0 - Do not sign the repo in the end of the build. The repo will be signed by default +# +# LOCAL_KEY=1 Use local keys to sign repo instead of WGET from GSA. By default use GSA. +# +# GPG_HOME= - Use the specified directory as GNUPGHOME for signing (no passphrase assumed). +# Bypasses GSA download and LOCAL_KEY. +# +# SETUP=1 Setup environment for build. By default do not setup environment. +# +# LOG= - provide an LOG file option to redirect some output into log file +# +# DEST= - provide a directory to contains the build result +# +# Running builds in parallel on one host: the build lock is scoped to the source checkout (see +# the "build-lock" block below), so two builds from DIFFERENT +# checkouts -- e.g. the devel and stable Ubuntu CD lanes -- run +# concurrently, while two builds of the SAME checkout still fail-fast +# (they build in-place and would corrupt each other). For parallel +# builds give each a separate checkout and a separate output tree +# (a distinct DEST). GPG_HOME may be SHARED between parallel builds +# -- it is used read-only for signing. +# +# For the dependency packages 1. All the xcat dependency deb packages should be uploaded to +# "pokgsa/projects/x/xcat/build/ubuntu/xcat-dep/debs/" on GSA +# 2. run ./build-ubunturepo -d +# +# 3. the built xcat-dep deb packages tarball can be found in "../../xcat-dep" +# related to the path of this script +############################ +printusage() +{ + printf "Usage: %s {-c | -d} \n" $(basename $0) >&2 + echo " -c : Build the xcat-core packages and create the repo" + echo " -d : Create the xcat-dep repo." +} +# For the purpose of getting the distribution name +if [[ ! -f /etc/lsb-release ]]; then + echo "ERROR: Could not find /etc/lsb-release, is this script executed on a Ubuntu machine?" + exit 1 +fi +. /etc/lsb-release + +export HOME=/root + + +# Process cmd line variable assignments, assigning each attr=val pair to a variable of same name +for i in $*; do + echo $i | grep '=' -q + if [ $? != 0 ];then + continue + fi + # upper case the variable name + varstring=`echo "$i"|cut -d '=' -f 1|tr '[a-z]' '[A-Z]'`=`echo "$i"|cut -d '=' -f 2` + export $varstring +done + +#Setup environment so the xcat-deps can be built on a FVT test machine +if [ "$SETUP" = "1" ];then + #Mount GSA + POKGSA="/gsa/pokgsa" + POKGSA2="/gsa/pokgsa-p2" + POKGSAIBM="pokgsa.ibm.com" + if [ ! -d $POKGSA ];then + mkdir -p $POKGSA + mount ${POKGSAIBM}:${POKGSA} ${POKGSA} + fi + if [ ! -d $POKGSA2 ];then + mkdir -p $POKGSA2 + mount ${POKGSAIBM}:${POKGSA2} ${POKGSA2} + fi + + # Verify needed packages installed + REPREPO="reprepro" + DEVSCRIPTS="devscripts" + DEBHELPER="debhelper" + QUILT="quilt" + + apt-get -y install $REPREPO $DEVSCRIPTS $DEBHELPER $QUILT + + echo "Finished setup for xcat-dep build. Rerun this script with SETUP=0 LOCAL_KEY=1 flags" + exit 1 +fi + +# Check the necessary packages before starting the build +declare -a packages=( "reprepro" "devscripts" "debhelper" "libsoap-lite-perl" "libdbi-perl" "quilt" "git") + +for package in ${packages[@]}; do + RC=`dpkg -l | grep $package >> /dev/null 2>&1; echo $?` + if [[ ${RC} != 0 ]]; then + echo "ERROR: Could not find $package, install using 'apt-get install $package' to continue" + exit 1 + fi +done + +# Supported distributions. Set DISTS="jammy noble resolute" to limit local validation builds. +dists="${DISTS:-saucy trusty utopic xenial bionic focal jammy noble resolute}" + +# GPG key used to sign the apt repo (reprepro SignWith). Defaults to the historic +# name. Override with GPG_KEY_ID= (space-free, since it is passed via +# the attr=value parser above), e.g. GPG_KEY_ID=xcat-build@xcat.org +GPG_KEY_ID="${GPG_KEY_ID:-xCAT Automatic Signing Key}" + +c_flag= # xcat-core (trunk-delvel) path +d_flag= # xcat-dep (trunk) path +r_flag= #genesis base rpm package path + +while getopts 'cdr:' OPTION +do + case $OPTION in + c) c_flag=1 + ;; + d) d_flag=1 + ;; + r) r_flag=1 + genesis_rpm_path="$OPTARG" + ;; + ?) printusage + exit 2 + ;; + esac +done +shift $(($OPTIND - 1)) + +if [ -z "$c_flag" -a -z "$d_flag" ];then + printusage + exit 2 +fi + +if [ "$c_flag" -a "$d_flag" ];then + printusage + exit 2 +fi + +if [ -z "$BUILDALL" ]; then + BUILDALL=1 +fi + +# Find where this script is located to set some build variables +old_pwd=`pwd` +cd `dirname $0` +curdir=`pwd` + +# Scope the build lock to THIS checkout. build-ubunturepo builds the packages in-place +# in its own source tree (it rewrites debian/changelog and debian/control, drops +# *.orig.tar.gz at the checkout root and runs dpkg-buildpackage inside the package +# dirs), so the resource two builds actually contend for is the checkout -- not the +# host. The historic single /var/lock/xcatbld.lock was host-global and fail-fast, so +# two builds from *different* checkouts (e.g. the devel and stable Ubuntu CD lanes on +# one build host) collided and the loser failed the pipeline even though they share +# nothing. Key the lock on the checkout path instead: builds of the SAME checkout +# still fail-fast (they would corrupt each other in-place), while builds of DISTINCT +# checkouts get distinct locks and run in parallel. The lock file stays on the local +# /var/lock (reliable flock; the checkout may live on NFS/virtiofs where flock is not) +# and the source tree is left byte-pristine. +# +# NOTE: the two marked regions below are extracted verbatim and exercised by the unit +# test xCAT-test/unit/build_ubunturepo_lock.t (which runs them with a chosen $curdir) +# -- keep the markers, and keep each region self-contained. +# BEGIN build-lock-id +lock_id_for() { printf '%s' "$1" | md5sum | cut -c1-12; } +LOCKFILE="/var/lock/xcatbld-$(lock_id_for "$curdir").lock" +# END build-lock-id +# BEGIN build-lock-acquire +exec 8>"$LOCKFILE" +if ! flock -n 8; then + echo "ERROR: Can't get lock $LOCKFILE for checkout $curdir. Another build is already using this checkout. Exiting...." + exit 1 +fi +# END build-lock-acquire + +# for the git case, query the current branch and set REL (changing master to devel if necessary) +function setbranch { + # Get the current branch name. safe.directory='*' so this still works when the + # build runs as root against a repo owned by another user (otherwise git errors + # with "dubious ownership", returns empty, and REL collapses to an unstable value). + branch=`git -c safe.directory='*' rev-parse --abbrev-ref HEAD 2>/dev/null` + if [ "$branch" = "master" ]; then + REL="devel" + elif [ "$branch" = "HEAD" ] || [ -z "$branch" ]; then + # Special handling when in a 'detached HEAD' state + branch=`git -c safe.directory='*' describe --abbrev=0 HEAD 2>/dev/null` + [[ -n "$branch" ]] && REL=`echo $branch|cut -d. -f 1,2` + else + REL=$branch + fi +} + +WGET_CMD="wget" +if [ ! -z ${LOG} ]; then + WGET_CMD="wget -o ${LOG}" +fi + +if [ "$GPGSIGN" = "0" ];then + echo "GPGSIGN=$GPGSIGN specified, skip gnupg key downloading" +elif [ -n "$GPG_HOME" ];then + echo "GPG_HOME=$GPG_HOME specified, using provided GNUPGHOME" + export GNUPGHOME="$GPG_HOME" +else + #sync the gpg key to the build machine local + gsa_url=http://pokgsa.ibm.com/projects/x/xcat/build/linux + mkdir -p $HOME/.gnupg + for key_name in pubring.gpg secring.gpg trustdb.gpg; do + if [ "$LOCAL_KEY" = "1" ];then + # Keys are already in the local $HOME/.gnupg directory + chmod 600 $HOME/.gnupg/$key_name + else + # Need to download keys from GSA + if [ ! -f $HOME/.gnupg/$key_name ] || [ `wc -c $HOME/.gnupg/$key_name|cut -f 1 -d' '` == 0 ]; then + rm -f $HOME/.gnupg/$key_name + ${WGET_CMD} -P $HOME/.gnupg $gsa_url/keys/$key_name + chmod 600 $HOME/.gnupg/$key_name + fi + fi + done +fi + +REL=xcat-core +if [ "$c_flag" ] +then + setbranch + # Sanitize REL into a stable, filesystem-safe token: replace any character that + # isn't [A-Za-z0-9._-] (e.g. the '/' in a branch like feat/ubuntu-e2e, which would + # otherwise create nested dirs) with '-', and never let it be empty. + REL=${REL//[^A-Za-z0-9._-]/-} + [ -z "$REL" ] && REL="local" + package_dir_name=debs$REL + + #define the dep source code path, core build target path and dep build target path + if [ -z "$DEST" ]; then + local_core_repo_path="$curdir/../../xcat-core" + PKGDIR="../../$package_dir_name" + else + local_core_repo_path="$DEST/$package_dir_name/xcat-core" + PKGDIR="$DEST/$package_dir_name/$package_dir_name" + fi + if [ ! -d "$PKGDIR" ];then + mkdir -p "$PKGDIR" + fi + + echo "#############################################################" + echo "Building xcat-core on branch ($REL) to $local_core_repo_path" + echo "#############################################################" + if [ "$PROMOTE" != 1 ]; then + code_change=0 + update_log='' + + if [ -z "$GITUP" ];then + update_log=../coregitup + echo "git pull > $update_log" + git pull > $update_log + else + update_log=$GITUP + fi + + if ! grep -q 'Already up-to-date' $update_log; then + code_change=1 + fi + ver=`cat Version` + short_ver=`cat Version|cut -d. -f 1,2` + short_short_ver=`cat Version|cut -d. -f 1` + commit_id_long=`git rev-parse HEAD` + commit_id="${commit_id_long:0:7}" + if [ -f Gitepoch ]; then + source_date_epoch=$(cat Gitepoch) + else + source_date_epoch=$(git log -1 --format=%ct HEAD 2>/dev/null || date +%s) + fi + export SOURCE_DATE_EPOCH="$source_date_epoch" + export DEBEMAIL="xcat-build@xcat.org" + export DEBFULLNAME="xCAT Build" + build_time=$(date -d "@$source_date_epoch" --utc '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || date -u) + build_machine=`hostname` + + if [ $code_change == 0 -a "$UP" != 1 -a "$BUILDALL" != 1 ]; then + echo "Nothing new detected. Exiting...." + exit 0 + fi + + echo "###############################" + echo "# Building xcat-core packages #" + echo "###############################" + + #the package type: local | snap | alpha + #the build introduce string + build_string="Snap_Build" + if [ -f Release ]; then + xcat_release=$(cat Release) + else + xcat_release="snap$(date -d "@$source_date_epoch" --utc '+%Y%m%d%H%M')" + fi + pkg_version="${ver}-${xcat_release}" + + packages="xCAT-client xCAT-genesis-scripts perl-xCAT xCAT-server xCAT xCATsn xCAT-test xCAT-buildkit xCAT-vlan xCAT-confluent xCAT-probe" + if [ -n "$PACKAGE" ]; then + match="" + for p in $packages; do + p_low=$(echo "$p" | tr '[A-Z]' '[a-z]') + pkg_low=$(echo "$PACKAGE" | tr '[A-Z]' '[a-z]') + if [ "$p_low" = "$pkg_low" ]; then + match="$p" + break + fi + done + if [ -z "$match" ]; then + echo "ERROR: Package '$PACKAGE' not found. Valid packages: $packages" + exit 1 + fi + packages="$match" + fi + target_archs=(amd64 ppc64el) + for file in $packages + do + file_low=`echo $file | tr '[A-Z]' '[a-z]'` + if [ "$file" = "xCAT" -o "$file" = "xCAT-genesis-scripts" -o "$file" = "xCATsn" ]; then + target_archs="amd64 ppc64el" + else + target_archs="all" + fi + for target_arch in $target_archs + do + tar_orig="${file_low}_${ver}.orig.tar.gz" + if grep -q "3.0 (quilt)" "${file}/debian/source/format" && [ ! -f "$tar_orig" ]; then + tar czf "$tar_orig" --exclude debian -C "$file" . + fi + + if grep -q $file $update_log || [ "$BUILDALL" == 1 -o "$file" = "perl-xCAT" ]; then + rm -f $PKGDIR/${file_low}_*.$target_arch.deb + cd $file + CURDIR=$(pwd) + + find . -name '*.dch' -delete + deterministic_date=$(date -R -d "@$SOURCE_DATE_EPOCH" --utc 2>/dev/null || date -R --utc) + sed -i "1s/(.*)/(${pkg_version})/" debian/changelog + sed -i "s/^ -- .*/ -- $DEBFULLNAME <$DEBEMAIL> $deterministic_date/" debian/changelog + if [ "$target_arch" = "all" ]; then + #xcat probe use some functions shipped by xCAT, for below reasons we need to copy files to xCAT-probe directory + #1 make xcat probe code to be self-contained + #2 don't maintain two files for each script + #3 symbolic link can't work during package + if [ $file_low = "xcat-probe" ]; then + mkdir -p ${CURDIR}/lib/perl/xCAT/ + cp -f ${CURDIR}/../perl-xCAT/xCAT/NetworkUtils.pm ${CURDIR}/lib/perl/xCAT/ + cp -f ${CURDIR}/../perl-xCAT/xCAT/GlobalDef.pm ${CURDIR}/lib/perl/xCAT/ + cp -f ${CURDIR}/../perl-xCAT/xCAT/ServiceNodeUtils.pm ${CURDIR}/lib/perl/xCAT/ + fi + CURDIR=$(pwd) + cp ${CURDIR}/debian/control ${CURDIR}/debian/control.save.998 + # Magic string used here + sed -i -e "s#>= 2.13-snap000000000000#= ${pkg_version}#g" ${CURDIR}/debian/control + dpkg-buildpackage -rfakeroot -uc -us + mv ${CURDIR}/debian/control.save.998 ${CURDIR}/debian/control + else + if [ "$file" = "xCAT-genesis-scripts" ]; then + echo "Rename control file to build pkg: mv ${CURDIR}/debian/control-${target_arch} ${CURDIR}/debian/control" + cp ${CURDIR}/debian/control-${target_arch} ${CURDIR}/debian/control + elif [ "$file" = "xCAT" ]; then + # shipping bmcsetup and getipmi scripts as part of postscripts + files=("bmcsetup" "getipmi") + for f in "${files[@]}"; do + cp ${CURDIR}/../xCAT-genesis-scripts/usr/bin/$f ${CURDIR}/postscripts/$f + sed -i "s/xcat.genesis.$f/$f/g" ${CURDIR}/postscripts/$f + done + fi + CURDIR=$(pwd) + cp ${CURDIR}/debian/control ${CURDIR}/debian/control.save.998 + # Magic string used here + sed -i -e "s#>= 2.13-snap000000000000#= ${pkg_version}#g" ${CURDIR}/debian/control + dpkg-buildpackage -rfakeroot -uc -us -a$target_arch + mv ${CURDIR}/debian/control.save.998 ${CURDIR}/debian/control + if [ "$file" = "xCAT-genesis-scripts" ]; then + echo "Move control file back: mv ${CURDIR}/debian/control ${CURDIR}/debian/control-${target_arch}" + rm ${CURDIR}/debian/control + elif [ "$file" = "xCAT" ]; then + files=("bmcsetup" "getipmi") + for f in "${files[@]}"; do + rm -f ${CURDIR}/postscripts/$f + done + fi + fi + rc=$? + if [ $rc -gt 0 ]; then + echo "Error: $file build package failed exit code $rc" + exit $rc + fi + cd - + find $file -maxdepth 3 -type d -name "${file_low}*" | grep debian | xargs rm -rf + find $file -maxdepth 3 -type f -name "files" | grep debian | xargs rm -rf + mv ${file_low}* $PKGDIR/ + fi + done + done + + find $PKGDIR/* ! -name '*.deb' | xargs rm -f + fi + + if [ "$PROMOTE" = 1 ]; then + upload_dir="xcat-core" + tar_name="xcat-core-$ver.tar.bz2" + else + upload_dir="core-snap" + tar_name="core-debs-snap.tar.bz2" + fi + + echo "#################################" + echo "# Creating xcat-core repository #" + echo "#################################" + + #clean the repo directory + if [ -e $local_core_repo_path ]; then + rm -rf $local_core_repo_path + fi + mkdir -p $local_core_repo_path + cd $local_core_repo_path + mkdir conf + + for dist in $dists; do + # for all releases moving forward, support amd64 and ppc64el + tmp_out_arch="amd64 ppc64el" + if [ "$dist" = "saucy" ]; then + # for older releases of Ubuntu that does not support ppc64el + tmp_out_arch="amd64" + fi + cat << __EOF__ >> conf/distributions +Origin: xCAT internal repository +Label: xcat-core bazaar repository +Codename: $dist +Architectures: $tmp_out_arch +Components: main +Description: Repository automatically genereted conf +__EOF__ + + if [ "$GPGSIGN" = "0" ];then + #echo "GPGSIGN=$GPGSIGN specified, the repo will not be signed" + echo "" >> conf/distributions + else + keyid=$(gpg --list-keys --keyid-format long "$GPG_KEY_ID" | grep '^pub' | sed -e 's/.*\///' -e 's/ .*//') + echo "SignWith: $keyid" >> conf/distributions + echo "" >> conf/distributions + fi + done + + if [ -n "$GPG_HOME" ]; then + cat << __EOF__ > conf/options +verbose +basedir . +__EOF__ + else + cat << __EOF__ > conf/options +verbose +ask-passphrase +basedir . +__EOF__ + fi + + #import the deb packages into the repo + amd_files=`ls ../$package_dir_name/*.deb | grep -v "ppc64el"` + all_files=`ls ../$package_dir_name/*.deb` + for dist in $dists; do + deb_files=$all_files + if [ "$dist" = "saucy" ]; then + # for older releases of Ubuntu that does not support ppc64el + deb_files=$amd_files + fi + for file in $deb_files; do + reprepro -b ./ includedeb $dist $file; + done + done + #create the mklocalrepo script + cat << '__EOF__' > mklocalrepo.sh +. /etc/lsb-release +cd `dirname $0` +host_arch=`uname -m` +if [ "$host_arch" != "ppc64le" ];then + host_arch="amd64" +else + host_arch="ppc64el" +fi +echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-core.list +__EOF__ + + chmod 775 mklocalrepo.sh + + # + # Add a buildinfo file into the tar.bz2 file to track information about the build + # + BUILDINFO=$local_core_repo_path/buildinfo + echo "VERSION=$ver" > $BUILDINFO + echo "RELEASE=$xcat_release" >> $BUILDINFO + echo "BUILD_TIME=$build_time" >> $BUILDINFO + echo "BUILD_MACHINE=$build_machine" >> $BUILDINFO + echo "COMMIT_ID=$commit_id" >> $BUILDINFO + echo "COMMIT_ID_LONG=$commit_id_long" >> $BUILDINFO + + #create the xcat-core.list file + + cd ../ + if ! grep xcat /etc/group ; then + groupadd xcat + fi + + chgrp -R root xcat-core + chmod -R g+w xcat-core + + #build the tar ball + echo "Creating `pwd`/$tar_name ..." + tar -hjcf $tar_name xcat-core + chgrp root $tar_name + chmod g+w $tar_name + + if [ -n "$DEST" ]; then + ln -sf $(basename `pwd`)/$tar_name ../$tar_name + if [ $? != 0 ]; then + echo "ERROR: Failed to make symbol link $DEST/$tar_name" + fi + fi + + if [ ! -e core-snap ]; then + ln -s xcat-core core-snap + fi + + cd $old_pwd + exit 0 +fi + +if [ "$d_flag" ] +then + echo "################################" + echo "# Creating xcat-dep repository #" + echo "################################" + + #the path of ubuntu xcat-dep deb packages on GSA + GSA="/gsa/pokgsa/projects/x/xcat/build/ubuntu/xcat-dep" + if [ ! -d $GSA ]; then + echo "build-ubunturepo: It appears that you do not have GSA to access the xcat-dep pkgs." + exit 1; + fi + + #define the dep source code path, core build target path and dep build target path + if [ -z "$DEST" ]; then + local_dep_repo_path="$curdir/../../xcat-dep/xcat-dep" + else + local_dep_repo_path="$DEST/xcat-dep/xcat-dep" + fi + + # Sync from the GSA master copy of the dep rpms + echo "Creating directory $local_dep_repo_path" + mkdir -p $local_dep_repo_path/ + + echo "Syncing RPMs from $GSA/ to $local_dep_repo_path/../ ..." + rsync -ilrtpu --delete $GSA/ $local_dep_repo_path/../ + if [ $? -ne 0 ]; then + echo "Error from rsync, cannot continue!" + exit 1 + fi + + #clean all old files + if [ -e $local_dep_repo_path ];then + rm -rf $local_dep_repo_path + fi + mkdir -p $local_dep_repo_path + cd $local_dep_repo_path + mkdir conf + + + #create the conf/distributions file + for dist in $dists; do + tmp_out_arch="amd64 ppc64el" + if [ "$dist" = "saucy" ]; then + # for older releases of Ubuntu that does not support ppc64el + tmp_out_arch="amd64" + fi + cat << __EOF__ >> conf/distributions +Origin: xCAT internal repository +Label: xcat-dep bazaar repository +Codename: $dist +Architectures: $tmp_out_arch +Components: main +Description: Repository automatically genereted conf +__EOF__ + + if [ "$GPGSIGN" = "0" ];then + echo "GPGSIGN=$GPGSIGN specified, the repo will not be signed" + echo "" >> conf/distributions + else + keyid=$(gpg --list-keys --keyid-format long "$GPG_KEY_ID" | grep '^pub' | sed -e 's/.*\///' -e 's/ .*//') + echo "SignWith: $keyid" >> conf/distributions + echo "" >> conf/distributions + fi + + done + + + + if [ -n "$GPG_HOME" ]; then + cat << __EOF__ > conf/options +verbose +basedir . +__EOF__ + else + cat << __EOF__ > conf/options +verbose +ask-passphrase +basedir . +__EOF__ + fi + + #import the deb packages into the repo + amd_files=`ls ../debs/*.deb | grep -v "ppc64el"` + all_files=`ls ../debs/*.deb` + for dist in $dists; do + deb_files=$all_files + if [ "$dist" = "saucy" ]; then + # for older releases of Ubuntu that does not support ppc64el + deb_files=$amd_files + fi + for file in $deb_files; do + reprepro -b ./ includedeb $dist $file; + done + done + + cat << '__EOF__' > mklocalrepo.sh +. /etc/lsb-release +cd `dirname $0` +host_arch=`uname -m` +if [ "$host_arch" != "ppc64le" ];then + host_arch="amd64" +else + host_arch="ppc64el" +fi +echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-dep.list +__EOF__ + + chmod 775 mklocalrepo.sh + + cd .. + if ! grep xcat /etc/group ; then + groupadd xcat + fi + + chgrp -R root xcat-dep + chmod -R g+w xcat-dep + + #create the tar ball + dep_tar_name=xcat-dep-ubuntu-`date +%Y%m%d%H%M`.tar.bz2 + tar -hjcf $dep_tar_name xcat-dep + chgrp root $dep_tar_name + chmod g+w $dep_tar_name + + + USER="xcat" + SERVER="xcat.org" + FILES_PATH="files" + FRS="/var/www/${SERVER}/${FILES_PATH}" + APT_DIR="${FRS}/xcat" + APT_REPO_DIR="${APT_DIR}/repos/apt/devel" + + # Decide whether to upload the xcat-dep package or NOT (default is to NOT upload xcat-dep + if [ "$UP" != "1" ]; then + echo "Upload not specified, Done! (rerun with UP=1, to upload)" + cd $old_pwd + exit 0 + fi + + #upload the dep packages + i=0 + echo "Uploading debs from xcat-dep to ${APT_REPO_DIR}/xcat-dep/ ..." + while [ $((i+=1)) -le 5 ] && ! rsync -urLv --delete xcat-dep $USER@${SERVER}:${APT_REPO_DIR}/ + do : ; done + + #upload the tarball + i=0 + echo "Uploading $dep_tar_name to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..." + while [ $((i+=1)) -le 5 ] && ! rsync -v --force $dep_tar_name $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/ + do : ; done + + #upload the README file + cd debs + i=0 + echo "Uploading README to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..." + while [ $((i+=1)) -le 5 ] && ! rsync -v --force README $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/ + do : ; done + +fi + +cd $old_pwd +exit 0 diff --git a/builddebs.pl b/builddebs.pl old mode 100644 new mode 100755 index 35bd2c169..915c792b9 --- a/builddebs.pl +++ b/builddebs.pl @@ -101,14 +101,25 @@ my $ROOT = abs_path($FindBin::Bin); my $VERSION = do { open my $fh, '<', "$ROOT/Version" or die "Cannot read Version: $!\n"; my $v = <$fh>; chomp $v; $v }; my $EPOCH = source_date_epoch(); -my $RELEASE = $opts{release} || snap_release($EPOCH); +# A Release file, when present, is authoritative: buildrpms.pl writes one, and a +# pipeline that builds both must stamp the rpms and the debs with the same release. +my $FILE_RELEASE = do { + my $r; + if (-f "$ROOT/Release") { + open my $fh, '<', "$ROOT/Release" or die "Cannot read Release: $!\n"; + $r = <$fh>; + chomp $r if defined $r; + } + ($r && $r =~ /\S/) ? $r : undef; +}; +my $RELEASE = $opts{release} || $FILE_RELEASE || snap_release($EPOCH); my $PKGVER = deb_version($VERSION, $RELEASE); $ENV{SOURCE_DATE_EPOCH} = $EPOCH; # dpkg reads these for the changelog trailer. Fixed, so the packages do not carry # whoever happened to run the build. -$ENV{DEBFULLNAME} ||= 'xCAT Build'; -$ENV{DEBEMAIL} ||= 'xcat@xcat.org'; +$ENV{DEBFULLNAME} = 'xCAT Build'; +$ENV{DEBEMAIL} = 'xcat-build@xcat.org'; my $MAINTAINER = "$ENV{DEBFULLNAME} <$ENV{DEBEMAIL}>"; my $DEB_DATE = strftime('%a, %d %b %Y %H:%M:%S +0000', gmtime($EPOCH)); @@ -225,14 +236,24 @@ sub build_package { or die "FATAL: dpkg-buildpackage failed for $pkg ($arch)\n"; }); - # dpkg-buildpackage writes its output beside the package directory. + return; +} + +# collect_debs: move a finished package's .deb files out of the checkout root. +# +# This happens once the package's LAST architecture is built, never between them: +# dpkg-genbuildinfo reads the sibling .deb files that the same source package +# produced, so moving amd64 away before ppc64el runs makes the second build die with +# dpkg-genbuildinfo: error: cannot fstat file ../xcat_..._amd64.deb +sub collect_debs { + my ($pkg, $pkgdir) = @_; my $moved = 0; for my $deb (glob("$ROOT/*.deb")) { move($deb, "$pkgdir/" . basename($deb)) or die "Cannot move $deb into $pkgdir: $!\n"; $moved++; } - die "FATAL: $pkg ($arch) produced no .deb\n" unless $moved; + die "FATAL: $pkg produced no .deb\n" unless $moved; # The rest of the dpkg output is build residue, not an artifact. unlink glob("$ROOT/*.buildinfo"), glob("$ROOT/*.changes"), glob("$ROOT/*.dsc"), glob("$ROOT/*.tar.xz"), glob("$ROOT/*.tar.gz"); @@ -326,6 +347,14 @@ make_path($pkgdir); remove_tree($repo) if -d $repo && $opts{force}; make_path($repo); +# Clear dpkg output left in the checkout by an earlier run. dpkg-genbuildinfo reads +# the sibling artifacts of the source package it is building, so a stale .changes or +# .buildinfo from an aborted run makes the next build die with +# dpkg-genbuildinfo: error: cannot fstat file ../_.deb +# naming an architecture this run has not reached yet. +unlink glob("$ROOT/*.deb"), glob("$ROOT/*.buildinfo"), glob("$ROOT/*.changes"), + glob("$ROOT/*.dsc"), glob("$ROOT/*.tar.xz"), glob("$ROOT/*.orig.tar.gz"); + say "xcat-core $PKGVER -> $dest"; say "releases: @{[ join ' ', $opts{dists}->@* ]}"; @@ -333,6 +362,7 @@ for my $pkg ($opts{packages}->@*) { for my $arch (deb_package_arches($pkg)) { build_package($pkg, $arch, $pkgdir); } + collect_debs($pkg, $pkgdir); } my $count = assemble_repo($pkgdir, $repo); diff --git a/docs/source/developers/guides/code/builds.rst b/docs/source/developers/guides/code/builds.rst index bd3152cc7..de5e21d09 100644 --- a/docs/source/developers/guides/code/builds.rst +++ b/docs/source/developers/guides/code/builds.rst @@ -32,10 +32,14 @@ emitted, because a source-only run has no binary packages to advertise. .. note:: - ``buildcore.sh``, ``makerpm``, ``buildlocal.sh`` and ``build-ubunturepo`` - were removed in 2.19. ``buildrpms.pl`` replaces the first three, and its - ``--source-only`` replaces the old ``SRCONLY=1``; ``builddebs.pl`` replaces - ``build-ubunturepo``. + ``buildcore.sh``, ``makerpm`` and ``buildlocal.sh`` were removed in 2.19; + ``buildrpms.pl`` replaces all three, and its ``--source-only`` replaces the + old ``SRCONLY=1``. + + ``build-ubunturepo`` is superseded by ``builddebs.pl`` but is **still in the + tree for now**, as a differential oracle: it is the reference the new builder + is checked against, and it is removed once the CD pipelines have been moved + over. Do not add features to it. Debian and Ubuntu packages --------------------------