From cb340bd4efc137e30cd8350efac9b745c06e33c6 Mon Sep 17 00:00:00 2001 From: Victor Hu Date: Thu, 6 Jun 2019 09:58:50 -0400 Subject: [PATCH 01/12] Add a checkrc to break out when non zero return code. Add more debug to make this build work in our own clone environmemt --- builddep.sh | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/builddep.sh b/builddep.sh index 9f0f4fb0c..9c2fd55a0 100755 --- a/builddep.sh +++ b/builddep.sh @@ -12,7 +12,7 @@ # - createrepo command needs to be present on the build machine # # Usage: builddep.sh [attr=value attr=value ...] -# DESTDIR= - the dir to place the dep tarball in. The default is ../../../xcat-dep, +# DESTDIR= - the dir to place the dep tarball in. The default is ../../xcat-dep, # relative to where this script is located. # UP=0 or UP=1 - override the default upload behavior # FRSYUM=0 - put the directory of individual rpms in the project web area instead @@ -87,13 +87,37 @@ if [ -z "$DESTDIR" ]; then # This is really a hack here because it depends on the build # environment structure. However, it's not expected that # users are building the xcat-dep packages - DESTDIR=../../xcat-dep + if [[ $XCATCOREDIR == *"xcat2_autobuild_daily_builds"* ]]; then + # This shows we are in the daily build environment path, create the + # deps package at the top level of the build directory + DESTDIR=../../xcat-dep + else + # This means we are building in some other clone of xcat-core, + # so just place the destination one level up. + DESTDIR=../xcat-dep + fi fi +echo "INFO: xcat-dep package will be created here: $XCATCOREDIR/$DESTDIR" + +# Create a function to check the return code, +# if non-zero, we should stop or unexpected things may happen +function checkrc { + if [[ $? != 0 ]]; then + echo "[checkrc] non-zero return code, exiting..." + exit 1 + fi +} + # Sync from the GSA master copy of the dep rpms mkdir -p $DESTDIR/xcat-dep +checkrc + +# Copy over the xcat-dep from master staging area on GSA to the local directory here echo "Syncing RPMs from $GSA/ to $DESTDIR/xcat-dep ..." rsync -ilrtpu --delete $GSA/ $DESTDIR/xcat-dep +checkrc +ls -ltr $DESTDIR/xcat-dep cd $DESTDIR/xcat-dep # add a comment to indicate the latest xcat-dep tar ball name From 7274ce1ddc0dc857a5ce74efe63d2bc7d8e22248 Mon Sep 17 00:00:00 2001 From: Victor Hu Date: Thu, 6 Jun 2019 10:06:41 -0400 Subject: [PATCH 02/12] Check rc, we are forcing home to /root and it will fail and keep going --- builddep.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/builddep.sh b/builddep.sh index 9c2fd55a0..85df1524e 100755 --- a/builddep.sh +++ b/builddep.sh @@ -98,6 +98,7 @@ if [ -z "$DESTDIR" ]; then fi fi +echo "INFO: xcat-dep package name: $DFNAME" echo "INFO: xcat-dep package will be created here: $XCATCOREDIR/$DESTDIR" # Create a function to check the return code, @@ -117,7 +118,7 @@ checkrc echo "Syncing RPMs from $GSA/ to $DESTDIR/xcat-dep ..." rsync -ilrtpu --delete $GSA/ $DESTDIR/xcat-dep checkrc -ls -ltr $DESTDIR/xcat-dep +ls $DESTDIR/xcat-dep cd $DESTDIR/xcat-dep # add a comment to indicate the latest xcat-dep tar ball name @@ -126,6 +127,7 @@ sed -i -e "s#REPLACE_LATEST_SNAP_LINE#The latest xcat-dep tar ball is ${DFNAME}# if [ "$OSNAME" != "AIX" ]; then # Get gpg keys in place mkdir -p $HOME/.gnupg + checkrc for i in pubring.gpg secring.gpg trustdb.gpg; do if [ ! -f $HOME/.gnupg/$i ] || [ `wc -c $HOME/.gnupg/$i|cut -f 1 -d' '` == 0 ]; then rm -f $HOME/.gnupg/$i From 4135fd665b40b9c444890553cf4921b562f763b6 Mon Sep 17 00:00:00 2001 From: Victor Hu Date: Thu, 6 Jun 2019 10:10:50 -0400 Subject: [PATCH 03/12] Make the gnupg directory a variable so we can more easily change it --- builddep.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/builddep.sh b/builddep.sh index 85df1524e..c6040b15e 100755 --- a/builddep.sh +++ b/builddep.sh @@ -124,20 +124,21 @@ cd $DESTDIR/xcat-dep # add a comment to indicate the latest xcat-dep tar ball name sed -i -e "s#REPLACE_LATEST_SNAP_LINE#The latest xcat-dep tar ball is ${DFNAME}#g" README +GNU_KEYDIR="$HOME/.gnupg" +MACROS=$HOME/.rpmmacros if [ "$OSNAME" != "AIX" ]; then # Get gpg keys in place - mkdir -p $HOME/.gnupg + mkdir -p ${GNU_KEYDIR} checkrc for i in pubring.gpg secring.gpg trustdb.gpg; do - if [ ! -f $HOME/.gnupg/$i ] || [ `wc -c $HOME/.gnupg/$i|cut -f 1 -d' '` == 0 ]; then - rm -f $HOME/.gnupg/$i - cp $GSA/../keys/$i $HOME/.gnupg - chmod 600 $HOME/.gnupg/$i + if [ ! -f ${GNU_KEYDIR}/$i ] || [ `wc -c ${GNU_KEYDIR}/$i|cut -f 1 -d' '` == 0 ]; then + rm -f ${GNU_KEYDIR}/$i + cp $GSA/../keys/$i ${GNU_KEYDIR} + chmod 600 ${GNU_KEYDIR}/$i fi done # Tell rpm to use gpg to sign - MACROS=$HOME/.rpmmacros if ! $GREP -q '%_signature gpg' $MACROS 2>/dev/null; then echo '%_signature gpg' >> $MACROS fi From ffa2ced11660e3298e97dc3333af053c1dce1036 Mon Sep 17 00:00:00 2001 From: Victor Hu Date: Thu, 6 Jun 2019 10:23:28 -0400 Subject: [PATCH 04/12] Hard stop if the keys are already there so caller needs to decide to remove it or not --- builddep.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/builddep.sh b/builddep.sh index c6040b15e..875fb1002 100755 --- a/builddep.sh +++ b/builddep.sh @@ -47,7 +47,6 @@ if [ "$OSNAME" == "AIX" ]; then else DFNAME=xcat-dep-`date +%Y%m%d%H%M`.tar.bz2 GSA=/gsa/pokgsa/projects/x/xcat/build/linux/xcat-dep - export HOME=/root # This is so rpm and gpg will know home, even in sudo fi if [ ! -d $GSA ]; then @@ -90,11 +89,11 @@ if [ -z "$DESTDIR" ]; then if [[ $XCATCOREDIR == *"xcat2_autobuild_daily_builds"* ]]; then # This shows we are in the daily build environment path, create the # deps package at the top level of the build directory - DESTDIR=../../xcat-dep + DESTDIR=../../xcat-dep-build else # This means we are building in some other clone of xcat-core, # so just place the destination one level up. - DESTDIR=../xcat-dep + DESTDIR=../xcat-dep-build fi fi @@ -128,8 +127,13 @@ GNU_KEYDIR="$HOME/.gnupg" MACROS=$HOME/.rpmmacros if [ "$OSNAME" != "AIX" ]; then # Get gpg keys in place + if [[ -d ${GNU_KEYDIR} ]]; then + echo "WARNING: The gnupg key dir: $GNU_KEYDIR exists, it will be overwitten. Stop." + echo "WARNING: To continue, remove it and rerun the script." + exit 1 + fi mkdir -p ${GNU_KEYDIR} - checkrc + checkrc for i in pubring.gpg secring.gpg trustdb.gpg; do if [ ! -f ${GNU_KEYDIR}/$i ] || [ `wc -c ${GNU_KEYDIR}/$i|cut -f 1 -d' '` == 0 ]; then rm -f ${GNU_KEYDIR}/$i From 33411cd7c884a238ecfe8746d0a6b5aad9395600 Mon Sep 17 00:00:00 2001 From: Victor Hu Date: Thu, 6 Jun 2019 10:25:40 -0400 Subject: [PATCH 05/12] Check the key dir early on before we start doing anything --- builddep.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/builddep.sh b/builddep.sh index 875fb1002..5e67b76a4 100755 --- a/builddep.sh +++ b/builddep.sh @@ -65,6 +65,14 @@ for pkg in ${REQPKG[*]}; do fi done +GNU_KEYDIR="$HOME/.gnupg" +MACROS=$HOME/.rpmmacros +if [[ -d ${GNU_KEYDIR} ]]; then + echo "WARNING: The gnupg key dir: $GNU_KEYDIR exists, it will be overwitten. Stop." + echo "WARNING: To continue, remove it and rerun the script." + exit 1 +fi + # set grep to quiet by default GREP="grep -q" if [ "$VERBOSE" = "1" -o "$VERBOSE" = "yes" ]; then @@ -123,15 +131,8 @@ cd $DESTDIR/xcat-dep # add a comment to indicate the latest xcat-dep tar ball name sed -i -e "s#REPLACE_LATEST_SNAP_LINE#The latest xcat-dep tar ball is ${DFNAME}#g" README -GNU_KEYDIR="$HOME/.gnupg" -MACROS=$HOME/.rpmmacros if [ "$OSNAME" != "AIX" ]; then # Get gpg keys in place - if [[ -d ${GNU_KEYDIR} ]]; then - echo "WARNING: The gnupg key dir: $GNU_KEYDIR exists, it will be overwitten. Stop." - echo "WARNING: To continue, remove it and rerun the script." - exit 1 - fi mkdir -p ${GNU_KEYDIR} checkrc for i in pubring.gpg secring.gpg trustdb.gpg; do From b7b338c4369dd23354e0383fff6b72240915f521 Mon Sep 17 00:00:00 2001 From: Victor Hu Date: Thu, 6 Jun 2019 13:16:23 -0400 Subject: [PATCH 06/12] Change the message for key slightly --- builddep.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builddep.sh b/builddep.sh index 5e67b76a4..5c2c71395 100755 --- a/builddep.sh +++ b/builddep.sh @@ -68,8 +68,8 @@ done GNU_KEYDIR="$HOME/.gnupg" MACROS=$HOME/.rpmmacros if [[ -d ${GNU_KEYDIR} ]]; then - echo "WARNING: The gnupg key dir: $GNU_KEYDIR exists, it will be overwitten. Stop." - echo "WARNING: To continue, remove it and rerun the script." + echo "ERROR: The gnupg key dir: $GNU_KEYDIR exists, it will be overwitten. Stop." + echo "ERROR: To continue, remove it and rerun the script." exit 1 fi From 271a496263deb8710545829d1a9fb004ad716993 Mon Sep 17 00:00:00 2001 From: Victor Hu Date: Thu, 6 Jun 2019 13:43:39 -0400 Subject: [PATCH 07/12] Create more variables for the directory locations, change mklocalrepo to excute if needed --- builddep.sh | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/builddep.sh b/builddep.sh index 5c2c71395..793002d35 100755 --- a/builddep.sh +++ b/builddep.sh @@ -88,12 +88,13 @@ else YUMDIR=htdocs fi -cd `dirname $0` +SCRIPT=$(readlink -f "$0") +SCRIPTPATH=$(dirname "$SCRIPT") +echo "INFO: Running script from here: $SCRIPTPATH ..." + +cd $SCRIPTPATH XCATCOREDIR=`/bin/pwd` if [ -z "$DESTDIR" ]; then - # This is really a hack here because it depends on the build - # environment structure. However, it's not expected that - # users are building the xcat-dep packages if [[ $XCATCOREDIR == *"xcat2_autobuild_daily_builds"* ]]; then # This shows we are in the daily build environment path, create the # deps package at the top level of the build directory @@ -105,8 +106,8 @@ if [ -z "$DESTDIR" ]; then fi fi -echo "INFO: xcat-dep package name: $DFNAME" -echo "INFO: xcat-dep package will be created here: $XCATCOREDIR/$DESTDIR" +echo "INFO: Target package name: $DFNAME" +echo "INFO: Target package will be created here: $XCATCOREDIR/$DESTDIR" # Create a function to check the return code, # if non-zero, we should stop or unexpected things may happen @@ -117,16 +118,17 @@ function checkrc { fi } +WORKING_TARGET_DIR="${DESTDIR}/xcat-dep" # Sync from the GSA master copy of the dep rpms -mkdir -p $DESTDIR/xcat-dep +mkdir -p ${WORKING_TARGET_DIR} checkrc # Copy over the xcat-dep from master staging area on GSA to the local directory here -echo "Syncing RPMs from $GSA/ to $DESTDIR/xcat-dep ..." -rsync -ilrtpu --delete $GSA/ $DESTDIR/xcat-dep +echo "Syncing RPMs from $GSA/ to ${WORKING_TARGET_DIR} ..." +rsync -ilrtpu --delete $GSA/ ${WORKING_TARGET_DIR} checkrc -ls $DESTDIR/xcat-dep -cd $DESTDIR/xcat-dep +ls ${WORKING_TARGET_DIR} +cd ${WORKING_TARGET_DIR} # add a comment to indicate the latest xcat-dep tar ball name sed -i -e "s#REPLACE_LATEST_SNAP_LINE#The latest xcat-dep tar ball is ${DFNAME}#g" README @@ -172,6 +174,14 @@ if [ "$OSNAME" != "AIX" ]; then # Modify xcat-dep.repo files to point to the correct place echo "===> Modifying the xcat-dep.repo files to point to the correct location..." + + # make sure the mklocalrepo.sh script has execute permission (to ensure quality control) + echo "===> Making sure that the mklocalrepo.sh file contains execute permission ..." + ls -ltr ${SCRIPTPATH}/${WORKING_TARGET_DIR}/mklocalrepo.sh + if [[ ! -x "${SCRIPTPATH}/${WORKING_TARGET_DIR}/mklocalrepo.sh" ]]; then + echo "==> Adding execute ..." + chmod +x ${SCRIPTPATH}/${WORKING_TARGET_DIR}/mklocalrepo.sh + fi fi if [ "$OSNAME" == "AIX" ]; then From 9ab31d623870ce50e2525877d966c8eff66550c4 Mon Sep 17 00:00:00 2001 From: Victor Hu Date: Thu, 6 Jun 2019 13:46:14 -0400 Subject: [PATCH 08/12] Remove dev files if present --- builddep.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/builddep.sh b/builddep.sh index 793002d35..cedc1f25c 100755 --- a/builddep.sh +++ b/builddep.sh @@ -175,13 +175,17 @@ if [ "$OSNAME" != "AIX" ]; then # Modify xcat-dep.repo files to point to the correct place echo "===> Modifying the xcat-dep.repo files to point to the correct location..." - # make sure the mklocalrepo.sh script has execute permission (to ensure quality control) echo "===> Making sure that the mklocalrepo.sh file contains execute permission ..." ls -ltr ${SCRIPTPATH}/${WORKING_TARGET_DIR}/mklocalrepo.sh if [[ ! -x "${SCRIPTPATH}/${WORKING_TARGET_DIR}/mklocalrepo.sh" ]]; then echo "==> Adding execute ..." chmod +x ${SCRIPTPATH}/${WORKING_TARGET_DIR}/mklocalrepo.sh fi + + echo "==> Checking if 'replacelinks' is in the xcat-deps, removing if there ..." + if [[ -f ${SCRIPTPATH}/${WORKING_TARGET_DIR}/replacelinks ]]; then + rm -f ${SCRIPTPATH}/${WORKING_TARGET_DIR}/replacelinks + fi fi if [ "$OSNAME" == "AIX" ]; then From d25fae052cfa853e367a486f4a1f8dbade41fe52 Mon Sep 17 00:00:00 2001 From: Victor Hu Date: Thu, 6 Jun 2019 13:49:12 -0400 Subject: [PATCH 09/12] Formatted the output a little better --- builddep.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builddep.sh b/builddep.sh index cedc1f25c..6ccf9f9ff 100755 --- a/builddep.sh +++ b/builddep.sh @@ -178,11 +178,11 @@ if [ "$OSNAME" != "AIX" ]; then echo "===> Making sure that the mklocalrepo.sh file contains execute permission ..." ls -ltr ${SCRIPTPATH}/${WORKING_TARGET_DIR}/mklocalrepo.sh if [[ ! -x "${SCRIPTPATH}/${WORKING_TARGET_DIR}/mklocalrepo.sh" ]]; then - echo "==> Adding execute ..." + echo "===> --- found not execute, changing +x ..." chmod +x ${SCRIPTPATH}/${WORKING_TARGET_DIR}/mklocalrepo.sh fi - echo "==> Checking if 'replacelinks' is in the xcat-deps, removing if there ..." + echo "===> Checking if 'replacelinks' is in the xcat-deps, removing if there ..." if [[ -f ${SCRIPTPATH}/${WORKING_TARGET_DIR}/replacelinks ]]; then rm -f ${SCRIPTPATH}/${WORKING_TARGET_DIR}/replacelinks fi From de2cb793d10aebb59879b7d89c6b31c7d008a219 Mon Sep 17 00:00:00 2001 From: Victor Hu Date: Thu, 6 Jun 2019 14:02:10 -0400 Subject: [PATCH 10/12] Determine XCATCOREDIR on the fly --- builddep.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/builddep.sh b/builddep.sh index 6ccf9f9ff..251ae76d0 100755 --- a/builddep.sh +++ b/builddep.sh @@ -89,11 +89,10 @@ else fi SCRIPT=$(readlink -f "$0") -SCRIPTPATH=$(dirname "$SCRIPT") -echo "INFO: Running script from here: $SCRIPTPATH ..." +XCATCOREDIR=$(dirname "$SCRIPT") +echo "INFO: Running script from here: $XCATCOREDIR ..." -cd $SCRIPTPATH -XCATCOREDIR=`/bin/pwd` +cd $XCATCOREDIR if [ -z "$DESTDIR" ]; then if [[ $XCATCOREDIR == *"xcat2_autobuild_daily_builds"* ]]; then # This shows we are in the daily build environment path, create the @@ -176,15 +175,15 @@ if [ "$OSNAME" != "AIX" ]; then echo "===> Modifying the xcat-dep.repo files to point to the correct location..." echo "===> Making sure that the mklocalrepo.sh file contains execute permission ..." - ls -ltr ${SCRIPTPATH}/${WORKING_TARGET_DIR}/mklocalrepo.sh - if [[ ! -x "${SCRIPTPATH}/${WORKING_TARGET_DIR}/mklocalrepo.sh" ]]; then + ls -ltr ${XCATCOREDIR}/${WORKING_TARGET_DIR}/mklocalrepo.sh + if [[ ! -x "${XCATCOREDIR}/${WORKING_TARGET_DIR}/mklocalrepo.sh" ]]; then echo "===> --- found not execute, changing +x ..." - chmod +x ${SCRIPTPATH}/${WORKING_TARGET_DIR}/mklocalrepo.sh + chmod +x ${XCATCOREDIR}/${WORKING_TARGET_DIR}/mklocalrepo.sh fi echo "===> Checking if 'replacelinks' is in the xcat-deps, removing if there ..." - if [[ -f ${SCRIPTPATH}/${WORKING_TARGET_DIR}/replacelinks ]]; then - rm -f ${SCRIPTPATH}/${WORKING_TARGET_DIR}/replacelinks + if [[ -f ${XCATCOREDIR}/${WORKING_TARGET_DIR}/replacelinks ]]; then + rm -f ${XCATCOREDIR}/${WORKING_TARGET_DIR}/replacelinks fi fi @@ -280,7 +279,9 @@ chmod -R g+w * echo "===> Building the tarball..." # -# Want to stay above xcat-dep so we can rsync the whole directory +# Want to stay one level above xcat-dep so that the script +# can rsync the directory up to xcat.org. +# # DO NOT CHANGE DIRECTORY AFTER THIS POINT!! # cd .. From f059ec0998b94d68ecefc852ae0fd147e49026e0 Mon Sep 17 00:00:00 2001 From: Victor Hu Date: Thu, 6 Jun 2019 14:33:03 -0400 Subject: [PATCH 11/12] Stop using cd .. , cd .. and change to the exact directory --- builddep.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/builddep.sh b/builddep.sh index 251ae76d0..6ffa3d706 100755 --- a/builddep.sh +++ b/builddep.sh @@ -277,15 +277,16 @@ fi chgrp -R -h $SYSGRP * chmod -R g+w * -echo "===> Building the tarball..." +TARBALL_WORKING_DIR="${XCATCOREDIR}/${DESTDIR}" +echo "===> Building the tarball at: ${TARBALL_WORKING_DIR} ..." # # Want to stay one level above xcat-dep so that the script # can rsync the directory up to xcat.org. # # DO NOT CHANGE DIRECTORY AFTER THIS POINT!! # -cd .. -pwd + +cd ${TARBALL_WORKING_DIR} verbosetar="" if [ -n "$VERBOSEMODE" ]; then From 1abf93e6f4b24bb79f1f73fc7f90451f8086e411 Mon Sep 17 00:00:00 2001 From: Victor Hu Date: Thu, 6 Jun 2019 19:18:04 -0400 Subject: [PATCH 12/12] Change the destination directory to builddep.sh --- builddep.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builddep.sh b/builddep.sh index 6ffa3d706..97090a5a2 100755 --- a/builddep.sh +++ b/builddep.sh @@ -12,7 +12,7 @@ # - createrepo command needs to be present on the build machine # # Usage: builddep.sh [attr=value attr=value ...] -# DESTDIR= - the dir to place the dep tarball in. The default is ../../xcat-dep, +# DESTDIR= - the dir to place the dep tarball in. The default is ../../xcat-dep-build, # relative to where this script is located. # UP=0 or UP=1 - override the default upload behavior # FRSYUM=0 - put the directory of individual rpms in the project web area instead