diff --git a/build-ubunturepo b/build-ubunturepo new file mode 100755 index 000000000..bc92165ee --- /dev/null +++ b/build-ubunturepo @@ -0,0 +1,710 @@ +#!/bin/bash +# Update GSA Ubuntu Repositories or create a local repository +# +# Author: Leonardo Tonetto (tonetto@linux.vnet.ibm.com) +# Revisor: Arif Ali (aali@ocf.co.uk) +# +# +# Getting Started: +# - Clone the xcat-core git repository under a directory named "xcat-core/src" +# - make sure reprepro is installed on the build machine +# - Run this script from the local git repository you just created. +# ./build-ubunturepo -c BUILDALL=1 + +# Usage: attr=value attr=value ... ./build-ubunturepo { -c | -d } +# PROMOTE=1 - if the attribute "PROMOTE" is specified, means an official dot release. This does not +# actually build xcat, just uploads the most recent snap build to http://xcat.org/files/xcat/ . +# If not specified, a snap build is assumed, which uploads to https://xcat.org/files/xcat/ +# PREGA=1 - use this option with PROMOTE=1 on a branch that already has a released dot release, but this +# build is a GA candidate build, not to be released yet. This will result in the tarball +# being uploaded to http://xcat.org/files/xcat/repos/apt +# (but the tarball file name will be like a released tarball, not a snap build). +# When you are ready to release this build, use PROMOTE=1 without PREGA +# BUILDALL=1 - build all rpms, whether they changed or not. Should be used for snap builds that are in +# prep for a release. +# GPGSIGN=0 - Do not sign the repo in the end of the build. The repo will be signed by default +# +# LOCAL_KEY=1 Use local keys to sign repo instead of WGET from GSA. By default use GSA. +# +# GPG_HOME= - Use the specified directory as GNUPGHOME for signing (no passphrase assumed). +# Bypasses GSA download and LOCAL_KEY. +# +# SETUP=1 Setup environment for build. By default do not setup environment. +# +# LOG= - provide an LOG file option to redirect some output into log file +# +# DEST= - provide a directory to contains the build result +# +# Running builds in parallel on one host: the build lock is scoped to the source checkout (see +# the "build-lock" block below), so two builds from DIFFERENT +# checkouts -- e.g. the devel and stable Ubuntu CD lanes -- run +# concurrently, while two builds of the SAME checkout still fail-fast +# (they build in-place and would corrupt each other). For parallel +# builds give each a separate checkout and a separate output tree +# (a distinct DEST). GPG_HOME may be SHARED between parallel builds +# -- it is used read-only for signing. +# +# For the dependency packages 1. All the xcat dependency deb packages should be uploaded to +# "pokgsa/projects/x/xcat/build/ubuntu/xcat-dep/debs/" on GSA +# 2. run ./build-ubunturepo -d +# +# 3. the built xcat-dep deb packages tarball can be found in "../../xcat-dep" +# related to the path of this script +############################ +printusage() +{ + printf "Usage: %s {-c | -d} \n" $(basename $0) >&2 + echo " -c : Build the xcat-core packages and create the repo" + echo " -d : Create the xcat-dep repo." +} +# For the purpose of getting the distribution name +if [[ ! -f /etc/lsb-release ]]; then + echo "ERROR: Could not find /etc/lsb-release, is this script executed on a Ubuntu machine?" + exit 1 +fi +. /etc/lsb-release + +export HOME=/root + + +# Process cmd line variable assignments, assigning each attr=val pair to a variable of same name +for i in $*; do + echo $i | grep '=' -q + if [ $? != 0 ];then + continue + fi + # upper case the variable name + varstring=`echo "$i"|cut -d '=' -f 1|tr '[a-z]' '[A-Z]'`=`echo "$i"|cut -d '=' -f 2` + export $varstring +done + +#Setup environment so the xcat-deps can be built on a FVT test machine +if [ "$SETUP" = "1" ];then + #Mount GSA + POKGSA="/gsa/pokgsa" + POKGSA2="/gsa/pokgsa-p2" + POKGSAIBM="pokgsa.ibm.com" + if [ ! -d $POKGSA ];then + mkdir -p $POKGSA + mount ${POKGSAIBM}:${POKGSA} ${POKGSA} + fi + if [ ! -d $POKGSA2 ];then + mkdir -p $POKGSA2 + mount ${POKGSAIBM}:${POKGSA2} ${POKGSA2} + fi + + # Verify needed packages installed + REPREPO="reprepro" + DEVSCRIPTS="devscripts" + DEBHELPER="debhelper" + QUILT="quilt" + + apt-get -y install $REPREPO $DEVSCRIPTS $DEBHELPER $QUILT + + echo "Finished setup for xcat-dep build. Rerun this script with SETUP=0 LOCAL_KEY=1 flags" + exit 1 +fi + +# Check the necessary packages before starting the build +declare -a packages=( "reprepro" "devscripts" "debhelper" "libsoap-lite-perl" "libdbi-perl" "quilt" "git") + +for package in ${packages[@]}; do + RC=`dpkg -l | grep $package >> /dev/null 2>&1; echo $?` + if [[ ${RC} != 0 ]]; then + echo "ERROR: Could not find $package, install using 'apt-get install $package' to continue" + exit 1 + fi +done + +# Supported distributions. Set DISTS="jammy noble resolute" to limit local validation builds. +dists="${DISTS:-saucy trusty utopic xenial bionic focal jammy noble resolute}" + +# GPG key used to sign the apt repo (reprepro SignWith). Defaults to the historic +# name. Override with GPG_KEY_ID= (space-free, since it is passed via +# the attr=value parser above), e.g. GPG_KEY_ID=xcat-build@xcat.org +GPG_KEY_ID="${GPG_KEY_ID:-xCAT Automatic Signing Key}" + +c_flag= # xcat-core (trunk-delvel) path +d_flag= # xcat-dep (trunk) path +r_flag= #genesis base rpm package path + +while getopts 'cdr:' OPTION +do + case $OPTION in + c) c_flag=1 + ;; + d) d_flag=1 + ;; + r) r_flag=1 + genesis_rpm_path="$OPTARG" + ;; + ?) printusage + exit 2 + ;; + esac +done +shift $(($OPTIND - 1)) + +if [ -z "$c_flag" -a -z "$d_flag" ];then + printusage + exit 2 +fi + +if [ "$c_flag" -a "$d_flag" ];then + printusage + exit 2 +fi + +if [ -z "$BUILDALL" ]; then + BUILDALL=1 +fi + +# Find where this script is located to set some build variables +old_pwd=`pwd` +cd `dirname $0` +curdir=`pwd` + +# Scope the build lock to THIS checkout. build-ubunturepo builds the packages in-place +# in its own source tree (it rewrites debian/changelog and debian/control, drops +# *.orig.tar.gz at the checkout root and runs dpkg-buildpackage inside the package +# dirs), so the resource two builds actually contend for is the checkout -- not the +# host. The historic single /var/lock/xcatbld.lock was host-global and fail-fast, so +# two builds from *different* checkouts (e.g. the devel and stable Ubuntu CD lanes on +# one build host) collided and the loser failed the pipeline even though they share +# nothing. Key the lock on the checkout path instead: builds of the SAME checkout +# still fail-fast (they would corrupt each other in-place), while builds of DISTINCT +# checkouts get distinct locks and run in parallel. The lock file stays on the local +# /var/lock (reliable flock; the checkout may live on NFS/virtiofs where flock is not) +# and the source tree is left byte-pristine. +# +# NOTE: the two marked regions below are extracted verbatim and exercised by the unit +# test xCAT-test/unit/build_ubunturepo_lock.t (which runs them with a chosen $curdir) +# -- keep the markers, and keep each region self-contained. +# BEGIN build-lock-id +lock_id_for() { printf '%s' "$1" | md5sum | cut -c1-12; } +LOCKFILE="/var/lock/xcatbld-$(lock_id_for "$curdir").lock" +# END build-lock-id +# BEGIN build-lock-acquire +exec 8>"$LOCKFILE" +if ! flock -n 8; then + echo "ERROR: Can't get lock $LOCKFILE for checkout $curdir. Another build is already using this checkout. Exiting...." + exit 1 +fi +# END build-lock-acquire + +# for the git case, query the current branch and set REL (changing master to devel if necessary) +function setbranch { + # Get the current branch name. safe.directory='*' so this still works when the + # build runs as root against a repo owned by another user (otherwise git errors + # with "dubious ownership", returns empty, and REL collapses to an unstable value). + branch=`git -c safe.directory='*' rev-parse --abbrev-ref HEAD 2>/dev/null` + if [ "$branch" = "master" ]; then + REL="devel" + elif [ "$branch" = "HEAD" ] || [ -z "$branch" ]; then + # Special handling when in a 'detached HEAD' state + branch=`git -c safe.directory='*' describe --abbrev=0 HEAD 2>/dev/null` + [[ -n "$branch" ]] && REL=`echo $branch|cut -d. -f 1,2` + else + REL=$branch + fi +} + +WGET_CMD="wget" +if [ ! -z ${LOG} ]; then + WGET_CMD="wget -o ${LOG}" +fi + +if [ "$GPGSIGN" = "0" ];then + echo "GPGSIGN=$GPGSIGN specified, skip gnupg key downloading" +elif [ -n "$GPG_HOME" ];then + echo "GPG_HOME=$GPG_HOME specified, using provided GNUPGHOME" + export GNUPGHOME="$GPG_HOME" +else + #sync the gpg key to the build machine local + gsa_url=http://pokgsa.ibm.com/projects/x/xcat/build/linux + mkdir -p $HOME/.gnupg + for key_name in pubring.gpg secring.gpg trustdb.gpg; do + if [ "$LOCAL_KEY" = "1" ];then + # Keys are already in the local $HOME/.gnupg directory + chmod 600 $HOME/.gnupg/$key_name + else + # Need to download keys from GSA + if [ ! -f $HOME/.gnupg/$key_name ] || [ `wc -c $HOME/.gnupg/$key_name|cut -f 1 -d' '` == 0 ]; then + rm -f $HOME/.gnupg/$key_name + ${WGET_CMD} -P $HOME/.gnupg $gsa_url/keys/$key_name + chmod 600 $HOME/.gnupg/$key_name + fi + fi + done +fi + +REL=xcat-core +if [ "$c_flag" ] +then + setbranch + # Sanitize REL into a stable, filesystem-safe token: replace any character that + # isn't [A-Za-z0-9._-] (e.g. the '/' in a branch like feat/ubuntu-e2e, which would + # otherwise create nested dirs) with '-', and never let it be empty. + REL=${REL//[^A-Za-z0-9._-]/-} + [ -z "$REL" ] && REL="local" + package_dir_name=debs$REL + + #define the dep source code path, core build target path and dep build target path + if [ -z "$DEST" ]; then + local_core_repo_path="$curdir/../../xcat-core" + PKGDIR="../../$package_dir_name" + else + local_core_repo_path="$DEST/$package_dir_name/xcat-core" + PKGDIR="$DEST/$package_dir_name/$package_dir_name" + fi + if [ ! -d "$PKGDIR" ];then + mkdir -p "$PKGDIR" + fi + + echo "#############################################################" + echo "Building xcat-core on branch ($REL) to $local_core_repo_path" + echo "#############################################################" + if [ "$PROMOTE" != 1 ]; then + code_change=0 + update_log='' + + if [ -z "$GITUP" ];then + update_log=../coregitup + echo "git pull > $update_log" + git pull > $update_log + else + update_log=$GITUP + fi + + if ! grep -q 'Already up-to-date' $update_log; then + code_change=1 + fi + ver=`cat Version` + short_ver=`cat Version|cut -d. -f 1,2` + short_short_ver=`cat Version|cut -d. -f 1` + commit_id_long=`git rev-parse HEAD` + commit_id="${commit_id_long:0:7}" + if [ -f Gitepoch ]; then + source_date_epoch=$(cat Gitepoch) + else + source_date_epoch=$(git log -1 --format=%ct HEAD 2>/dev/null || date +%s) + fi + export SOURCE_DATE_EPOCH="$source_date_epoch" + export DEBEMAIL="xcat-build@xcat.org" + export DEBFULLNAME="xCAT Build" + build_time=$(date -d "@$source_date_epoch" --utc '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || date -u) + build_machine=`hostname` + + if [ $code_change == 0 -a "$UP" != 1 -a "$BUILDALL" != 1 ]; then + echo "Nothing new detected. Exiting...." + exit 0 + fi + + echo "###############################" + echo "# Building xcat-core packages #" + echo "###############################" + + #the package type: local | snap | alpha + #the build introduce string + build_string="Snap_Build" + if [ -f Release ]; then + xcat_release=$(cat Release) + else + xcat_release="snap$(date -d "@$source_date_epoch" --utc '+%Y%m%d%H%M')" + fi + pkg_version="${ver}-${xcat_release}" + + packages="xCAT-client xCAT-genesis-scripts perl-xCAT xCAT-server xCAT xCATsn xCAT-test xCAT-buildkit xCAT-vlan xCAT-confluent xCAT-probe" + if [ -n "$PACKAGE" ]; then + match="" + for p in $packages; do + p_low=$(echo "$p" | tr '[A-Z]' '[a-z]') + pkg_low=$(echo "$PACKAGE" | tr '[A-Z]' '[a-z]') + if [ "$p_low" = "$pkg_low" ]; then + match="$p" + break + fi + done + if [ -z "$match" ]; then + echo "ERROR: Package '$PACKAGE' not found. Valid packages: $packages" + exit 1 + fi + packages="$match" + fi + target_archs=(amd64 ppc64el) + for file in $packages + do + file_low=`echo $file | tr '[A-Z]' '[a-z]'` + if [ "$file" = "xCAT" -o "$file" = "xCAT-genesis-scripts" -o "$file" = "xCATsn" ]; then + target_archs="amd64 ppc64el" + else + target_archs="all" + fi + for target_arch in $target_archs + do + tar_orig="${file_low}_${ver}.orig.tar.gz" + if grep -q "3.0 (quilt)" "${file}/debian/source/format" && [ ! -f "$tar_orig" ]; then + tar czf "$tar_orig" --exclude debian -C "$file" . + fi + + if grep -q $file $update_log || [ "$BUILDALL" == 1 -o "$file" = "perl-xCAT" ]; then + rm -f $PKGDIR/${file_low}_*.$target_arch.deb + cd $file + CURDIR=$(pwd) + + find . -name '*.dch' -delete + deterministic_date=$(date -R -d "@$SOURCE_DATE_EPOCH" --utc 2>/dev/null || date -R --utc) + sed -i "1s/(.*)/(${pkg_version})/" debian/changelog + sed -i "s/^ -- .*/ -- $DEBFULLNAME <$DEBEMAIL> $deterministic_date/" debian/changelog + if [ "$target_arch" = "all" ]; then + #xcat probe use some functions shipped by xCAT, for below reasons we need to copy files to xCAT-probe directory + #1 make xcat probe code to be self-contained + #2 don't maintain two files for each script + #3 symbolic link can't work during package + if [ $file_low = "xcat-probe" ]; then + mkdir -p ${CURDIR}/lib/perl/xCAT/ + cp -f ${CURDIR}/../perl-xCAT/xCAT/NetworkUtils.pm ${CURDIR}/lib/perl/xCAT/ + cp -f ${CURDIR}/../perl-xCAT/xCAT/GlobalDef.pm ${CURDIR}/lib/perl/xCAT/ + cp -f ${CURDIR}/../perl-xCAT/xCAT/ServiceNodeUtils.pm ${CURDIR}/lib/perl/xCAT/ + fi + CURDIR=$(pwd) + cp ${CURDIR}/debian/control ${CURDIR}/debian/control.save.998 + # Magic string used here + sed -i -e "s#>= 2.13-snap000000000000#= ${pkg_version}#g" ${CURDIR}/debian/control + dpkg-buildpackage -rfakeroot -uc -us + mv ${CURDIR}/debian/control.save.998 ${CURDIR}/debian/control + else + if [ "$file" = "xCAT-genesis-scripts" ]; then + echo "Rename control file to build pkg: mv ${CURDIR}/debian/control-${target_arch} ${CURDIR}/debian/control" + cp ${CURDIR}/debian/control-${target_arch} ${CURDIR}/debian/control + elif [ "$file" = "xCAT" ]; then + # shipping bmcsetup and getipmi scripts as part of postscripts + files=("bmcsetup" "getipmi") + for f in "${files[@]}"; do + cp ${CURDIR}/../xCAT-genesis-scripts/usr/bin/$f ${CURDIR}/postscripts/$f + sed -i "s/xcat.genesis.$f/$f/g" ${CURDIR}/postscripts/$f + done + fi + CURDIR=$(pwd) + cp ${CURDIR}/debian/control ${CURDIR}/debian/control.save.998 + # Magic string used here + sed -i -e "s#>= 2.13-snap000000000000#= ${pkg_version}#g" ${CURDIR}/debian/control + dpkg-buildpackage -rfakeroot -uc -us -a$target_arch + mv ${CURDIR}/debian/control.save.998 ${CURDIR}/debian/control + if [ "$file" = "xCAT-genesis-scripts" ]; then + echo "Move control file back: mv ${CURDIR}/debian/control ${CURDIR}/debian/control-${target_arch}" + rm ${CURDIR}/debian/control + elif [ "$file" = "xCAT" ]; then + files=("bmcsetup" "getipmi") + for f in "${files[@]}"; do + rm -f ${CURDIR}/postscripts/$f + done + fi + fi + rc=$? + if [ $rc -gt 0 ]; then + echo "Error: $file build package failed exit code $rc" + exit $rc + fi + cd - + find $file -maxdepth 3 -type d -name "${file_low}*" | grep debian | xargs rm -rf + find $file -maxdepth 3 -type f -name "files" | grep debian | xargs rm -rf + mv ${file_low}* $PKGDIR/ + fi + done + done + + find $PKGDIR/* ! -name '*.deb' | xargs rm -f + fi + + if [ "$PROMOTE" = 1 ]; then + upload_dir="xcat-core" + tar_name="xcat-core-$ver.tar.bz2" + else + upload_dir="core-snap" + tar_name="core-debs-snap.tar.bz2" + fi + + echo "#################################" + echo "# Creating xcat-core repository #" + echo "#################################" + + #clean the repo directory + if [ -e $local_core_repo_path ]; then + rm -rf $local_core_repo_path + fi + mkdir -p $local_core_repo_path + cd $local_core_repo_path + mkdir conf + + for dist in $dists; do + # for all releases moving forward, support amd64 and ppc64el + tmp_out_arch="amd64 ppc64el" + if [ "$dist" = "saucy" ]; then + # for older releases of Ubuntu that does not support ppc64el + tmp_out_arch="amd64" + fi + cat << __EOF__ >> conf/distributions +Origin: xCAT internal repository +Label: xcat-core bazaar repository +Codename: $dist +Architectures: $tmp_out_arch +Components: main +Description: Repository automatically genereted conf +__EOF__ + + if [ "$GPGSIGN" = "0" ];then + #echo "GPGSIGN=$GPGSIGN specified, the repo will not be signed" + echo "" >> conf/distributions + else + keyid=$(gpg --list-keys --keyid-format long "$GPG_KEY_ID" | grep '^pub' | sed -e 's/.*\///' -e 's/ .*//') + echo "SignWith: $keyid" >> conf/distributions + echo "" >> conf/distributions + fi + done + + if [ -n "$GPG_HOME" ]; then + cat << __EOF__ > conf/options +verbose +basedir . +__EOF__ + else + cat << __EOF__ > conf/options +verbose +ask-passphrase +basedir . +__EOF__ + fi + + #import the deb packages into the repo + amd_files=`ls ../$package_dir_name/*.deb | grep -v "ppc64el"` + all_files=`ls ../$package_dir_name/*.deb` + for dist in $dists; do + deb_files=$all_files + if [ "$dist" = "saucy" ]; then + # for older releases of Ubuntu that does not support ppc64el + deb_files=$amd_files + fi + for file in $deb_files; do + reprepro -b ./ includedeb $dist $file; + done + done + #create the mklocalrepo script + cat << '__EOF__' > mklocalrepo.sh +. /etc/lsb-release +cd `dirname $0` +host_arch=`uname -m` +if [ "$host_arch" != "ppc64le" ];then + host_arch="amd64" +else + host_arch="ppc64el" +fi +echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-core.list +__EOF__ + + chmod 775 mklocalrepo.sh + + # + # Add a buildinfo file into the tar.bz2 file to track information about the build + # + BUILDINFO=$local_core_repo_path/buildinfo + echo "VERSION=$ver" > $BUILDINFO + echo "RELEASE=$xcat_release" >> $BUILDINFO + echo "BUILD_TIME=$build_time" >> $BUILDINFO + echo "BUILD_MACHINE=$build_machine" >> $BUILDINFO + echo "COMMIT_ID=$commit_id" >> $BUILDINFO + echo "COMMIT_ID_LONG=$commit_id_long" >> $BUILDINFO + + #create the xcat-core.list file + + cd ../ + if ! grep xcat /etc/group ; then + groupadd xcat + fi + + chgrp -R root xcat-core + chmod -R g+w xcat-core + + #build the tar ball + echo "Creating `pwd`/$tar_name ..." + tar -hjcf $tar_name xcat-core + chgrp root $tar_name + chmod g+w $tar_name + + if [ -n "$DEST" ]; then + ln -sf $(basename `pwd`)/$tar_name ../$tar_name + if [ $? != 0 ]; then + echo "ERROR: Failed to make symbol link $DEST/$tar_name" + fi + fi + + if [ ! -e core-snap ]; then + ln -s xcat-core core-snap + fi + + cd $old_pwd + exit 0 +fi + +if [ "$d_flag" ] +then + echo "################################" + echo "# Creating xcat-dep repository #" + echo "################################" + + #the path of ubuntu xcat-dep deb packages on GSA + GSA="/gsa/pokgsa/projects/x/xcat/build/ubuntu/xcat-dep" + if [ ! -d $GSA ]; then + echo "build-ubunturepo: It appears that you do not have GSA to access the xcat-dep pkgs." + exit 1; + fi + + #define the dep source code path, core build target path and dep build target path + if [ -z "$DEST" ]; then + local_dep_repo_path="$curdir/../../xcat-dep/xcat-dep" + else + local_dep_repo_path="$DEST/xcat-dep/xcat-dep" + fi + + # Sync from the GSA master copy of the dep rpms + echo "Creating directory $local_dep_repo_path" + mkdir -p $local_dep_repo_path/ + + echo "Syncing RPMs from $GSA/ to $local_dep_repo_path/../ ..." + rsync -ilrtpu --delete $GSA/ $local_dep_repo_path/../ + if [ $? -ne 0 ]; then + echo "Error from rsync, cannot continue!" + exit 1 + fi + + #clean all old files + if [ -e $local_dep_repo_path ];then + rm -rf $local_dep_repo_path + fi + mkdir -p $local_dep_repo_path + cd $local_dep_repo_path + mkdir conf + + + #create the conf/distributions file + for dist in $dists; do + tmp_out_arch="amd64 ppc64el" + if [ "$dist" = "saucy" ]; then + # for older releases of Ubuntu that does not support ppc64el + tmp_out_arch="amd64" + fi + cat << __EOF__ >> conf/distributions +Origin: xCAT internal repository +Label: xcat-dep bazaar repository +Codename: $dist +Architectures: $tmp_out_arch +Components: main +Description: Repository automatically genereted conf +__EOF__ + + if [ "$GPGSIGN" = "0" ];then + echo "GPGSIGN=$GPGSIGN specified, the repo will not be signed" + echo "" >> conf/distributions + else + keyid=$(gpg --list-keys --keyid-format long "$GPG_KEY_ID" | grep '^pub' | sed -e 's/.*\///' -e 's/ .*//') + echo "SignWith: $keyid" >> conf/distributions + echo "" >> conf/distributions + fi + + done + + + + if [ -n "$GPG_HOME" ]; then + cat << __EOF__ > conf/options +verbose +basedir . +__EOF__ + else + cat << __EOF__ > conf/options +verbose +ask-passphrase +basedir . +__EOF__ + fi + + #import the deb packages into the repo + amd_files=`ls ../debs/*.deb | grep -v "ppc64el"` + all_files=`ls ../debs/*.deb` + for dist in $dists; do + deb_files=$all_files + if [ "$dist" = "saucy" ]; then + # for older releases of Ubuntu that does not support ppc64el + deb_files=$amd_files + fi + for file in $deb_files; do + reprepro -b ./ includedeb $dist $file; + done + done + + cat << '__EOF__' > mklocalrepo.sh +. /etc/lsb-release +cd `dirname $0` +host_arch=`uname -m` +if [ "$host_arch" != "ppc64le" ];then + host_arch="amd64" +else + host_arch="ppc64el" +fi +echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-dep.list +__EOF__ + + chmod 775 mklocalrepo.sh + + cd .. + if ! grep xcat /etc/group ; then + groupadd xcat + fi + + chgrp -R root xcat-dep + chmod -R g+w xcat-dep + + #create the tar ball + dep_tar_name=xcat-dep-ubuntu-`date +%Y%m%d%H%M`.tar.bz2 + tar -hjcf $dep_tar_name xcat-dep + chgrp root $dep_tar_name + chmod g+w $dep_tar_name + + + USER="xcat" + SERVER="xcat.org" + FILES_PATH="files" + FRS="/var/www/${SERVER}/${FILES_PATH}" + APT_DIR="${FRS}/xcat" + APT_REPO_DIR="${APT_DIR}/repos/apt/devel" + + # Decide whether to upload the xcat-dep package or NOT (default is to NOT upload xcat-dep + if [ "$UP" != "1" ]; then + echo "Upload not specified, Done! (rerun with UP=1, to upload)" + cd $old_pwd + exit 0 + fi + + #upload the dep packages + i=0 + echo "Uploading debs from xcat-dep to ${APT_REPO_DIR}/xcat-dep/ ..." + while [ $((i+=1)) -le 5 ] && ! rsync -urLv --delete xcat-dep $USER@${SERVER}:${APT_REPO_DIR}/ + do : ; done + + #upload the tarball + i=0 + echo "Uploading $dep_tar_name to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..." + while [ $((i+=1)) -le 5 ] && ! rsync -v --force $dep_tar_name $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/ + do : ; done + + #upload the README file + cd debs + i=0 + echo "Uploading README to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..." + while [ $((i+=1)) -le 5 ] && ! rsync -v --force README $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/ + do : ; done + +fi + +cd $old_pwd +exit 0 diff --git a/builddebs.pl b/builddebs.pl old mode 100644 new mode 100755 index 35bd2c169..915c792b9 --- a/builddebs.pl +++ b/builddebs.pl @@ -101,14 +101,25 @@ my $ROOT = abs_path($FindBin::Bin); my $VERSION = do { open my $fh, '<', "$ROOT/Version" or die "Cannot read Version: $!\n"; my $v = <$fh>; chomp $v; $v }; my $EPOCH = source_date_epoch(); -my $RELEASE = $opts{release} || snap_release($EPOCH); +# A Release file, when present, is authoritative: buildrpms.pl writes one, and a +# pipeline that builds both must stamp the rpms and the debs with the same release. +my $FILE_RELEASE = do { + my $r; + if (-f "$ROOT/Release") { + open my $fh, '<', "$ROOT/Release" or die "Cannot read Release: $!\n"; + $r = <$fh>; + chomp $r if defined $r; + } + ($r && $r =~ /\S/) ? $r : undef; +}; +my $RELEASE = $opts{release} || $FILE_RELEASE || snap_release($EPOCH); my $PKGVER = deb_version($VERSION, $RELEASE); $ENV{SOURCE_DATE_EPOCH} = $EPOCH; # dpkg reads these for the changelog trailer. Fixed, so the packages do not carry # whoever happened to run the build. -$ENV{DEBFULLNAME} ||= 'xCAT Build'; -$ENV{DEBEMAIL} ||= 'xcat@xcat.org'; +$ENV{DEBFULLNAME} = 'xCAT Build'; +$ENV{DEBEMAIL} = 'xcat-build@xcat.org'; my $MAINTAINER = "$ENV{DEBFULLNAME} <$ENV{DEBEMAIL}>"; my $DEB_DATE = strftime('%a, %d %b %Y %H:%M:%S +0000', gmtime($EPOCH)); @@ -225,14 +236,24 @@ sub build_package { or die "FATAL: dpkg-buildpackage failed for $pkg ($arch)\n"; }); - # dpkg-buildpackage writes its output beside the package directory. + return; +} + +# collect_debs: move a finished package's .deb files out of the checkout root. +# +# This happens once the package's LAST architecture is built, never between them: +# dpkg-genbuildinfo reads the sibling .deb files that the same source package +# produced, so moving amd64 away before ppc64el runs makes the second build die with +# dpkg-genbuildinfo: error: cannot fstat file ../xcat_..._amd64.deb +sub collect_debs { + my ($pkg, $pkgdir) = @_; my $moved = 0; for my $deb (glob("$ROOT/*.deb")) { move($deb, "$pkgdir/" . basename($deb)) or die "Cannot move $deb into $pkgdir: $!\n"; $moved++; } - die "FATAL: $pkg ($arch) produced no .deb\n" unless $moved; + die "FATAL: $pkg produced no .deb\n" unless $moved; # The rest of the dpkg output is build residue, not an artifact. unlink glob("$ROOT/*.buildinfo"), glob("$ROOT/*.changes"), glob("$ROOT/*.dsc"), glob("$ROOT/*.tar.xz"), glob("$ROOT/*.tar.gz"); @@ -326,6 +347,14 @@ make_path($pkgdir); remove_tree($repo) if -d $repo && $opts{force}; make_path($repo); +# Clear dpkg output left in the checkout by an earlier run. dpkg-genbuildinfo reads +# the sibling artifacts of the source package it is building, so a stale .changes or +# .buildinfo from an aborted run makes the next build die with +# dpkg-genbuildinfo: error: cannot fstat file ../_.deb +# naming an architecture this run has not reached yet. +unlink glob("$ROOT/*.deb"), glob("$ROOT/*.buildinfo"), glob("$ROOT/*.changes"), + glob("$ROOT/*.dsc"), glob("$ROOT/*.tar.xz"), glob("$ROOT/*.orig.tar.gz"); + say "xcat-core $PKGVER -> $dest"; say "releases: @{[ join ' ', $opts{dists}->@* ]}"; @@ -333,6 +362,7 @@ for my $pkg ($opts{packages}->@*) { for my $arch (deb_package_arches($pkg)) { build_package($pkg, $arch, $pkgdir); } + collect_debs($pkg, $pkgdir); } my $count = assemble_repo($pkgdir, $repo); diff --git a/docs/source/developers/guides/code/builds.rst b/docs/source/developers/guides/code/builds.rst index bd3152cc7..de5e21d09 100644 --- a/docs/source/developers/guides/code/builds.rst +++ b/docs/source/developers/guides/code/builds.rst @@ -32,10 +32,14 @@ emitted, because a source-only run has no binary packages to advertise. .. note:: - ``buildcore.sh``, ``makerpm``, ``buildlocal.sh`` and ``build-ubunturepo`` - were removed in 2.19. ``buildrpms.pl`` replaces the first three, and its - ``--source-only`` replaces the old ``SRCONLY=1``; ``builddebs.pl`` replaces - ``build-ubunturepo``. + ``buildcore.sh``, ``makerpm`` and ``buildlocal.sh`` were removed in 2.19; + ``buildrpms.pl`` replaces all three, and its ``--source-only`` replaces the + old ``SRCONLY=1``. + + ``build-ubunturepo`` is superseded by ``builddebs.pl`` but is **still in the + tree for now**, as a differential oracle: it is the reference the new builder + is checked against, and it is removed once the CD pipelines have been moved + over. Do not add features to it. Debian and Ubuntu packages --------------------------