diff --git a/build-ubunturepo b/build-ubunturepo index f2b36320a..61d2398dc 100755 --- a/build-ubunturepo +++ b/build-ubunturepo @@ -17,15 +17,16 @@ # Usage: attr=value attr=value ... ./build-ubunturepo { -c | -d } # PROMOTE=1 - if the attribute "PROMOTE" is specified, means an official dot release. This does not -# actually build xcat, just uploads the most recent snap build to http://xcat.org/files/xcat/ . -# If not specified, a snap build is assumed, which uploads to https://xcat.org/files/yum/ -# or https://xcat.org/files/aix/. -# PREGA=1 - use this option with PROMOTE=1 on a branch that already has a released dot release, but this build is -# a GA candidate build, not to be released yet. This will result in the tarball being uploaded to -# http://xcat.org/files/yum/ or http://xcat.org/files/aix/ -# (but the tarball file name will be like a released tarball, not a snap build). When you are ready to -# release this build, use PROMOTE=1 without PREGA -# BUILDALL=1 - build all rpms, whether they changed or not. Should be used for snap builds that are in prep for a release. +# actually build xcat, just uploads the most recent snap build to http://xcat.org/files/xcat/ . +# If not specified, a snap build is assumed, which uploads to https://xcat.org/files/yum/ +# or https://xcat.org/files/aix/. +# PREGA=1 - use this option with PROMOTE=1 on a branch that already has a released dot release, but this +# build is a GA candidate build, not to be released yet. This will result in the tarball +# being uploaded to http://xcat.org/files/xcat/repos/apt +# (but the tarball file name will be like a released tarball, not a snap build). +# When you are ready to release this build, use PROMOTE=1 without PREGA +# BUILDALL=1 - build all rpms, whether they changed or not. Should be used for snap builds that are in +# prep for a release. # UP=0 or UP=1 - override the default upload behavior # LOG= - provide an LOG file option to redirect some output into log file # @@ -43,8 +44,18 @@ printusage() echo " -d : Create the xcat dep repo. Building the xcat dep packages can refer \"build-debs-all\" from svn" } # For the purpose of getting the distribution name +if [[ ! -f /etc/lsb-release ]]; then + echo "ERROR: Could not find /etc/lsb-release, is this script executed on a Ubuntu machine?" + exit 1 +fi . /etc/lsb-release +REPREPRO=`which reprepro >> /dev/null 2>&1; echo $?` +if [[ ${REPREPRO} != 0 ]]; then + echo "ERROR: Could not find reprepro, verify that reprepro is installed on this machine. Cannot continue!" + exit 1 +fi + # Process cmd line variable assignments, assigning each attr=val pair to a variable of same name for i in $*; do echo $i | grep '=' @@ -93,9 +104,14 @@ fi uploader="litingt" -user="xcat" -target_machine="xcat.org" +USER="xcat" +SERVER="xcat.org" +FILES_PATH="files" +FRS="/var/www/${SERVER}/${FILES_PATH}" release="github.com/xcat2/xcat-core/releases" +APT_DIR="${FRS}/xcat" +APT_REPO_DIR="${APT_DIR}/repos/apt" + # Find where this script is located to set some build variables old_pwd=`pwd` cd `dirname $0` @@ -347,53 +363,24 @@ __EOF__ ln -s xcat-core core-snap fi - # - # remove the changes to changelog since the files get changed and created in xcat-core.. - # left over files not commited to git will prevent switching from branch to branch ... - echo "Cleaning up modified changelog files..." - cleanup_oldpwd=`pwd` - # go to the xcat-core git repo directory - # - cd $local_core_repo_path - cd ../xcatbuild/xcat-core - find ./ -name "changelog" -print | xargs git checkout - cd $cleanup_oldpwd - - # Decide whether to upload or not - if [ -n "$UP" ] && [ "$UP" == 0 ]; then - echo "No need to upload" + # Decide whether to upload or not (default NOT to upload) + if [ "$UP" != "1" ]; then + echo "Upload not specified, Done! (rerun with UP=1, to upload)" cd $old_pwd exit 0 fi #upload the deb packages - if [ "$REL" = "devel" -o "$PREGA" != 1 ]; then - i=0 - echo "Uploading RPMs from $upload_dir to ${sf_dir}/ubuntu/${REL}/ ..." - while [ $((i+=1)) -le 5 ] && ! rsync -urLv --delete $upload_dir $user@$target_machine:${sf_dir}/ubuntu/${REL}/ - do : ; done - fi + i=0 + echo "Uploading RPMs from $upload_dir to ${APT_REPO_DIR}/${REL}/ ..." + while [ $((i+=1)) -le 5 ] && ! rsync -urLv --delete $upload_dir $USER@${SERVER}:${APT_REPO_DIR}/${REL}/ + do : ; done #upload the tar ball - #for the GA build, upload to http://xcat.org/files/xcat/.x_Ubuntu/ and https://github.com/xcat2/xcat-core/releases - #for other scenario, upload to http://xcat.org/files/ubuntu/ - if [ "$PROMOTE" = 1 -a "$REL" != "devel" -a "$PREGA" != 1 ]; then - i=0 - echo "Uploading $tar_name to ${sf_dir}/xcat/${REL}.x_Ubuntu/ ..." - while [ $((i+=1)) -le 5 ] && ! rsync -v --force $tar_name $user@$target_machine:${sf_dir}/xcat/${REL}.x_Ubuntu/ - do : ; done - - #at the same time upload the GA build to github. - i=0 - echo "Uploading $tar_name to https://github.com/xcat2/xcat-core/releases ..." - while [ $((i+=1)) -le 5 ] && ! rsync -v --force $tar_name $uploader@$release/ - do : ; done - else - i=0 - echo "Uploading $tar_name to ${sf_dir}/ubuntu/${REL}/ ..." - while [ $((i+=1)) -le 5 ] && ! rsync -v --force $tar_name $user@$target_machine:${sf_dir}/ubuntu/${REL}/ - do : ; done - fi + i=0 + echo "Uploading $tar_name to ${APT_DIR}/xcat-core/${REL}/Ubuntu/core-snap/ ..." + while [ $((i+=1)) -le 5 ] && ! rsync -v --force $tar_name $USER@${SERVER}:${APT_DIR}/xcat-core/${REL}/Ubuntu/core-snap/ + do : ; done cd $old_pwd fi @@ -404,20 +391,24 @@ then echo "# Creating xcat-dep repository #" echo "################################" - #the path of ubuntu xcat-dep deb packages on GSA GSA="/gsa/pokgsa/projects/x/xcat/build/ubuntu/xcat-dep" - if [ ! -d $GSA ]; then echo "build-ubunturepo: It appears that you do not have gsa installed to access the xcat-dep pkgs." exit 1; fi # Sync from the GSA master copy of the dep rpms + echo "Creating directory $local_dep_repo_path" + mkdir -p $local_dep_repo_path/ + echo "Syncing RPMs from $GSA/ to $local_dep_repo_path/../ ..." rsync -ilrtpu --delete $GSA/ $local_dep_repo_path/../ - + if [ $? -ne 0 ]; then + echo "Error from rsync, cannot continue!" + exit 1 + fi #clean all old files if [ -e $local_dep_repo_path ];then @@ -492,37 +483,35 @@ __EOF__ chmod -R g+w xcat-dep #create the tar ball - dep_tar_name=xcat-dep-ubuntu-snap`date +%Y%m%d`.tar.bz + dep_tar_name=xcat-dep-ubuntu-`date +%Y%m%d%H%M`.tar.bz tar -hjcf $dep_tar_name xcat-dep chgrp root $dep_tar_name chmod g+w $dep_tar_name - # remove the changes to changelog since the builds are occuring in xcat-core, will prevent - # switching from branch to branch without a new clone of the repository - find ./ -name "changelog" -print | xargs git checkout - - if [ -n "$UP" ] && [ "$UP" == 0 ];then - echo "No need to upload the dep packages" + # Decide whether to upload or not (default NOT to upload) + if [ "$UP" != "1" ]; then + echo "Upload not specified, Done! (rerun with UP=1, to upload)" cd $old_pwd exit 0 fi #upload the dep packages i=0 - echo "Uploading debs from xcat-dep to ${sf_dir}/ubuntu/xcat-dep/ ..." - while [ $((i+=1)) -le 5 ] && ! rsync -urLv --delete xcat-dep $user@$target_machine:${sf_dir}/ubuntu/ + echo "Uploading debs from xcat-dep to ${APT_REPO_DIR}/xcat-dep/ ..." + while [ $((i+=1)) -le 5 ] && ! rsync -urLv --delete xcat-dep $USER@${SERVER}:${APT_REPO_DIR}/ do : ; done + #upload the tarball i=0 - echo "Uploading $dep_tar_name to ${sf_dir}/xcat-dep/2.x_Ubuntu/ ..." - while [ $((i+=1)) -le 5 ] && ! rsync -v --force $dep_tar_name $user@$target_machine:${sf_dir}/xcat-dep/2.x_Ubuntu/ + echo "Uploading $dep_tar_name to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..." + while [ $((i+=1)) -le 5 ] && ! rsync -v --force $dep_tar_name $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/ do : ; done #upload the README file cd debs i=0 - echo "Uploading README to ${sf_dir}/xcat-dep/2.x_Ubuntu/ ..." - while [ $((i+=1)) -le 5 ] && ! rsync -v --force README $user@$targe_machine:${sf_dir}/xcat-dep/2.x_Ubuntu/ + echo "Uploading README to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..." + while [ $((i+=1)) -le 5 ] && ! rsync -v --force README $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/ do : ; done cd $old_pwd diff --git a/buildcore.sh b/buildcore.sh index b185f3323..a936c8185 100755 --- a/buildcore.sh +++ b/buildcore.sh @@ -17,11 +17,11 @@ # Before running buildcore.sh, you must change the local git repo to the branch you want built, using: git checkout # PROMOTE=1 - if the attribute "PROMOTE" is specified, means an official dot release. This does not actually build # xcat, just uploads the most recent snap build to http://xcat.org/files/xcat/ . -# If not specified, a snap build is assumed, which uploads to http://xcat.org/files/yum/ +# If not specified, a snap build is assumed, which uploads to http://xcat.org/files/xcat/repos/yum/ # or http//xcat.org/files/aix/. # PREGA=1 - use this option with PROMOTE=1 on a branch that already has a released dot release, but this build is # a GA candidate build, not to be released yet. This will result in the tarball being uploaded to -# http://xcar.org/files/yum/ or http://xcat.org/files/aix/ +# http://xcar.org/files/xcat/repos/yum/ # (but the tarball file name will be like a released tarball, not a snap build). When you are ready to # release this build, use PROMOTE=1 without PREGA # BUILDALL=1 - build all rpms, whether they changed or not. Should be used for snap builds that are in prep for a release. diff --git a/builddep.sh b/builddep.sh index 90269963e..5a3f7e931 100755 --- a/builddep.sh +++ b/builddep.sh @@ -1,69 +1,80 @@ # The shell is commented out so that it will run in bash on linux and ksh on aix -# !/bin/sh -# Package up all the xCAT open source dependencies, setting up yum repos and -# also tar it all up. This assumes that individual rpms have already been built for -# all relevant architectures from the src & spec files in svn. - -# When running this script to package xcat-dep: -# - Make sure createrepo is installed on the build machine - +# !/bin/sh +# +# Package up all the xCAT open source dependencies +# - creating the yum repos +# - tar up the deps package +# +# This script assumes that the individual rpms have already been compiled +# for the relevant architectures from the src & spec files in git. +# +# Dependencies: +# - createrepo command needs to be present on the build machine +# # Usage: builddep.sh [attr=value attr=value ...] -# DESTDIR= - the dir to place the dep tarball in. The default is ../../../xcat-dep, relative -# to where this script is located. -# UP=0 or UP=1 - override the default upload behavior -# FRSYUM=0 - put the directory of individual rpms in the project web area instead of the FRS area. -# VERBOSE=1 - to see lots of verbose output +# DESTDIR= - the dir to place the dep tarball in. The default is ../../../xcat-dep, +# relative to where this script is located. +# UP=0 or UP=1 - override the default upload behavior +# FRSYUM=0 - put the directory of individual rpms in the project web area instead +# of the FRS area. +# VERBOSE=1 - Set to 1 to see more VERBOSE output # you can change this if you need to USER=xcat TARGET_MACHINE=xcat.org -FRS=/var/www/xcat.org/files +FRS=/var/www/xcat.org/files/xcat OSNAME=$(uname) +UP=0 # Process cmd line variable assignments, assigning each attr=val pair to a variable of same name for i in $*; do # upper case the variable name varstring=`echo "$i"|cut -d '=' -f 1|tr [a-z] [A-Z]`=`echo "$i"|cut -d '=' -f 2` export $varstring done -if [ "$VERBOSE" = "1" -o "$VERBOSE" = "yes" ]; then - set -x - VERBOSEMODE=1 -fi if [ "$OSNAME" == "AIX" ]; then + DFNAME=dep-aix-`date +%Y%m%d%H%M`.tar.gz GSA=/gsa/pokgsa/projects/x/xcat/build/aix/xcat-dep else + DFNAME=xcat-dep-`date +%Y%m%d%H%M`.tar.bz2 GSA=/gsa/pokgsa/projects/x/xcat/build/linux/xcat-dep export HOME=/root # This is so rpm and gpg will know home, even in sudo fi +if [ ! -d $GSA ]; then + echo "ERROR: This script is intended to be used by xCAT development..." + echo "ERROR: The GSA directory ($GSA) directory does not appear to be mounted, cannot continue!" + exit 1 +fi + +# set grep to quiet by default +GREP="grep -q" +if [ "$VERBOSE" = "1" -o "$VERBOSE" = "yes" ]; then + set -x + VERBOSEMODE=1 + GREP="grep" +fi + # this is needed only when we are transitioning the yum over to frs -YUMREPOURL1="http://xcat.org/yum" -YUMREPOURL2="http://xcat.org/files/yum" +# YUMREPOURL1="http://xcat.org/yum" +# YUMREPOURL2="http://xcat.org/files/yum" if [ "$FRSYUM" != 0 ]; then - YUMDIR=$FRS - YUMREPOURL="$YUMREPOURL2" + YUMDIR="$FRS/repos" + # YUMREPOURL="$YUMREPOURL2" else YUMDIR=htdocs - YUMREPOURL="$YUMREPOURL1" + # YUMREPOURL="$YUMREPOURL1" fi -if [ -n "$VERBOSEMODE" ]; then - GREP=grep -else - GREP="grep -q" -fi - -if [ ! -d $GSA ]; then - echo "builddep: It appears that you do not have gsa installed to access the xcat-dep pkgs." - exit 1; -fi cd `dirname $0` XCATCOREDIR=`/bin/pwd` if [ -z "$DESTDIR" ]; then - DESTDIR=../../../xcat-dep + # This is really a hack here because it depends on the build + # environment structure. However, it's not expected that + # users are building the xcat-dep packages + DESTDIR=../../xcat-dep fi # Sync from the GSA master copy of the dep rpms @@ -72,6 +83,9 @@ echo "Syncing RPMs from $GSA/ to $DESTDIR/xcat-dep ..." rsync -ilrtpu --delete $GSA/ $DESTDIR/xcat-dep cd $DESTDIR/xcat-dep +# add a comment to indicate the latest xcat-dep tar ball name +sed -i -e "s#REPLACE_LATEST_SNAP_LINE#The latest xcat-dep tar ball is ${DFNAME}#g" README + if [ "$OSNAME" != "AIX" ]; then # Get gpg keys in place mkdir -p $HOME/.gnupg @@ -93,11 +107,11 @@ if [ "$OSNAME" != "AIX" ]; then fi # Sign the rpms that are not already signed. The "standard input reopened" warnings are normal. - echo "Signing RPMs..." + echo "===> Signing RPMs..." $XCATCOREDIR/build-utils/rpmsign.exp `find . -type f -name '*.rpm'` | grep -v -E '(already contains identical signature|was already signed|rpm --quiet --resign|WARNING: standard input reopened)' # Create the repodata dirs - echo "Creating repodata directories..." + echo "===> Creating repodata directories..." for i in `find -mindepth 2 -maxdepth 2 -type d `; do if [ -n "$VERBOSEMODE" ]; then createrepo --checksum sha $i # specifying checksum so the repo will work on rhel5 @@ -112,14 +126,19 @@ if [ "$OSNAME" != "AIX" ]; then done # Modify xCAT-dep.repo files to point to the correct place - if [ "$FRSYUM" != 0 ]; then - newurl="$YUMREPOURL2" - oldurl="$YUMREPOURL1" - else - newurl="$YUMREPOURL1" - oldurl="$YUMREPOURL2" - fi - sed -i -e "s|=$oldurl|=$newurl|g" `find . -name "xCAT-dep.repo" ` + echo "===> Modifying the xCAT-dep.repo files to point to the correct location..." + # 10/01/2015 - vkhu + # The URLs have been updated in GSA, this section is not needed at the moment + # + #if [ "$FRSYUM" != 0 ]; then + # newurl="$YUMREPOURL2" + # oldurl="$YUMREPOURL1" + #else + # newurl="$YUMREPOURL1" + # oldurl="$YUMREPOURL2" + #fi + # + #sed -i -e "s|=$oldurl|=$newurl|g" `find . -name "xCAT-dep.repo" ` fi if [ "$OSNAME" == "AIX" ]; then @@ -199,72 +218,74 @@ fi # Get the permissions and group correct if [ "$OSNAME" == "AIX" ]; then + # AIX SYSGRP=system + YUM=aix + FRSDIR='2.x_AIX' else + # Linux SYSGRP=root + YUM=yum + FRSDIR='2.x_Linux' fi chgrp -R -h $SYSGRP * chmod -R g+w * -# Build the tarball -#VER=`cat $XCATCOREDIR/Version` +echo "===> Building the tarball..." +# +# Want to stay above xcat-dep so we can rsync the whole directory +# DO NOT CHANGE DIRECTORY AFTER THIS POINT!! +# cd .. +pwd + +verbosetar="" if [ -n "$VERBOSEMODE" ]; then verbosetar="-v" -else - verbosetar="" fi + +echo "===> Creating $DFNAME ..." if [ "$OSNAME" == "AIX" ]; then - DFNAME=dep-aix-`date +%Y%m%d%H%M`.tar.gz - echo "Creating $DFNAME ..." tar $verbosetar -cf ${DFNAME%.gz} xcat-dep rm -f $DFNAME gzip ${DFNAME%.gz} else - DFNAME=xcat-dep-`date +%Y%m%d%H%M`.tar.bz2 - echo "Creating $DFNAME ..." + # Linux tar $verbosetar -jcf $DFNAME xcat-dep fi -#cd xcat-dep <-- now we want to stay above xcat-dep, so we can rsync the whole dir - -if [ "$UP" == 0 ]; then - exit 0; +if [[ ${UP} -eq 0 ]]; then + echo "Upload not being done, set UP=1 to upload to xcat.org" + exit 0; fi -if [ "$OSNAME" == "AIX" ]; then - YUM=aix - FRSDIR='2.x_AIX' -else - YUM=yum - FRSDIR='2.x_Linux' -fi - -# Upload the dir structure to SF yum area. Currently we do not have it preserving permissions -# because that gives errors when different users try to do it. -i=0 +# Upload the directory structure to xcat.org yum area (xcat/repos/yum). if [ "$FRSYUM" != 0 ]; then - links="-L" # FRS does not support rsyncing sym links + links="-L" # FRS does not support rsyncing sym links else links="-l" fi -echo "Uploading RPMs from xcat-dep to $YUMDIR/$YUM/ ..." + +i=0 +echo "Uploading the xcat-deps RPMs from xcat-dep to RPMs from xcat-dep to $YUMDIR/$YUM/ ..." while [ $((i+=1)) -le 5 ] && ! rsync $links -ruv --delete xcat-dep $USER@$TARGET_MACHINE:$YUMDIR/$YUM/ do : ; done -# Upload the tarball to the SF FRS Area +# Upload the tarball to the xcat.org FRS Area i=0 echo "Uploading $DFNAME to $FRS/xcat-dep/$FRSDIR/ ..." while [ $((i+=1)) -le 5 ] && ! rsync -v $DFNAME $USER@$TARGET_MACHINE:$FRS/xcat-dep/$FRSDIR/ do : ; done -# Upload the README to the SF FRS Area -cd xcat-dep -# add a comment to indicate the latest xcat-dep tar ball name -sed "6 iThe latest xcat-dep tar ball is ${DFNAME}" -i README - +# Upload the README to the xcat.org FRS Area i=0 +cd xcat-dep echo "Uploading README to $FRS/xcat-dep/$FRSDIR/ ..." while [ $((i+=1)) -le 5 ] && ! rsync -v README $USER@$TARGET_MACHINE:$FRS/xcat-dep/$FRSDIR/ do : ; done +# For some reason the README is not updated +echo "Uploading README to $YUMDIR/$YUM/ ..." +while [ $((i+=1)) -le 5 ] && ! rsync -v README $USER@$TARGET_MACHINE:$YUMDIR/$YUM/ +do : ; done +