From 580e82c50789801d9e3b262d0164c3053f04edf0 Mon Sep 17 00:00:00 2001 From: huweihua Date: Thu, 1 Oct 2015 05:44:34 -0400 Subject: [PATCH 1/5] provide new usage of mellanox ib drives isntallation --- .../scripts/Mellanox/mlnxofed_ib_install.v2 | 325 ++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100644 xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 diff --git a/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 b/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 new file mode 100644 index 000000000..178760473 --- /dev/null +++ b/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 @@ -0,0 +1,325 @@ +#!/bin/sh +# +# This is the second version of Mellanox IB driver installation sample script +# This is also the version we recommend to use right now. +# What are changed in this version +# 1. The usage interface is changed, using command line arguments instead of using environment attribute +# 2. The way of downloading Mellanox OFED files. +# 3. Some structure of code +# +# For AIX: +# TBD +# For Linux: +# - For full-disk installs: +# - Copy Mellanox OFED ISO to node +# - Install IB rpms +# - For diskless images: +# - Copy the packages to the images. +# - Install IB rpms +# +#usage: +# +# Copy the xCAT mlnxofed_ib_install.v2 script file to postscripts directory: +# cp /opt/xcat/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 /install/postscripts/mlnxofed_ib_install +# +# Prepare Mellanox OFED ISO file and save it into any subdirectory under /install. but for easy to use and comply with xcat regular rule, recommend using a meaningful directory, such as /install/post/otherpkgs///ofed/ +# +# 1. install the ofed driver for diskfull node +# [NOTE] step 1.1-1.2 only needed by redhat and sles +# 1.1 copy the pkglist to the custom directory: +# cp /opt/xcat/share/xcat/install//compute...pkglist /install/custom/install//compute...pkglist +# Edit your /install/custom/install//compute...pkglist and add: +# #INCLUDE:/opt/xcat/share/xcat/ib/netboot//ib...pkglist# +# 1.2 Make the related osimage use the customized pkglist. +# chdef -t osimage -o --install-compute pkglist=/install/custom/install//compute...pkglist +# 1.3 set mlnxofed_ib_install as postbootscripts for the target node. assign necessary attribute for mlnxofed_ib_install at same time. +# chdef -p postbootscripts="mlnxofed_ib_install -ofeddir -ofedname " +# [NOTE] The default options input into Mellanox are '--without-32bit --without-fw-update --force' +# you can appoint the options by yourslef with '-passmlnxofedoptions' option of mlnxofed_ib_install +# In order to distinguish which options are tranfered to Mellanox and which options are belong to mlnxofed_ib_install, any options wanted to transfered to Mellanox must follow behind -passmlnxofedoptions and end with "-end-", for example: +# chdef -p postbootscripts="mlnxofed_ib_install -ofeddir -passmlnxofedoptions --without-32bit --add-kernel-support --force -end- -ofedname < OFED ISO file name>" +# 1.4 do the diskfull installation +# nodeset osimage= ...... +# +# 2. install the ofed driver for diskless images: +# [NOTE] step 2.1 only needed by redhat and sles +# 2.1 copy the pkglist to the custom directory: +# cp /opt/xcat/share/xcat/netboot//compute...pkglist /install/custom/netboot//compute...pkglist +# Edit your /install/custom/netboot//.pkglist and add: +# #INCLUDE:/opt/xcat/share/xcat/ib/netboot//ib...pkglist# +# 2.2 Add to postinstall scripts +# Edit your /install/custom/netboot//.postinstall and add: +# /install/postscripts/mlnxofed_ib_install -ofeddir -ofedname -nodesetstate genimage -installroot $1 +# 2.3 Make sure the related osimage use the customized pkglist and customized compute.postinsall +# lsdef -t osimage -o --netboot-compute +# if not, change it: +# chdef -t osimage -o --netboot-compute pkglist=/install/custom/netboot//compute...pkglist postinstall=/install/custom/netboot//.postinstall +# 2.4 run genimage +# genimage --netboot-compute +# + +#set -x + +OS="$(uname)" +if [ "$OS" = "Linux" ]; then + str_dir_name="${0%/*}" + . "$str_dir_name/xcatlib.sh" +fi + +#-------------------------------------------------------- +declare -a MLNXOFED_OPTS + +function usage() { + echo "Usage: mlnxofed_ib_install [-attribute]" + echo " attribute include:" + echo " -h: print this help message" + echo " -ofeddir: the directory where OFED file is saved. this is necessary attribute" + echo " -ofedname: the name of OFED file. this is necessary attribute" + echo " -passmlnxofedoptions: the options inputted into mlnxofedinstall script, defualt value are --without-32bit --without-fw-update --force" + echo " -installroot: the image root path. this is necessary attribute in diskless scenario" + echo " -nodesetstate: nodeset status, the value are install, boot or genimage" +} + +while [ "$#" -gt "0" ] +do + case "$1" in + "-ofeddir") + shift + OFED_DIR="$1" + ;; + "-passmlnxofedoptions") + shift + while [ "-end-" != "$1" ] + do + MLNXOFED_OPTS=("${MLNXOFED_OPTS[@]}" "$1") + shift + done + ;; + "-ofedname") + shift + OFED_NAME="$1" + ;; + "-installroot") + shift + IMGROOTPATH="$1" + ;; + "-nodesetstate") + shift + NODESETSTATE="$1" + ;; + "-h") + usage + exit 0 + ;; + *) + echo "unsupport attribute $1" + exit 0 + ;; + esac + shift +done + +if [ -z "$OFED_DIR" ]; then + echo "[Error] Without Mellanox OFED file path, please assign correct path" + exit 1 +fi + +if [ -z "$OFED_NAME" ]; then + echo "[Error] Without Mellanox OFED file name, please assign correct name" + exit 1 +fi + +if [ "$NODESETSTATE" = "genimage" -a -z "$IMGROOTPATH" ]; then + echo "[Error] this is for diskless installation, please assign correct diskless image root path" + exit 1 +fi + +[ "${#MLNXOFED_OPTS}" = 0 ] && MLNXOFED_OPTS=(--without-32bit --without-fw-update --force) + +echo "Mellanox OFED file path is $OFED_DIR" +echo "Mellanox OFED file is $OFED_NAME" +echo "Mellanox OFED options are ${MLNXOFED_OPTS[@]}" +echo "image root path is $IMGROOTPATH" +echo "NODESETSTATE is $NODESETSTATE" + +if [ "$OS" != "AIX" ]; then + if [ "$NODESETSTATE" = "install" -o "$NODESETSTATE" = "boot" ]; then + + #if the host is ubuntn, need to do some network check and configuration + if grep -q Ubuntu /etc/os-release 2>/dev/null + then + nodename=`hostname` + echo "$nodename 's operating system is Ubuntu." + echo "If you want to install Mellanox_OFED in $nodename, $nodename must have ability to access www.ports.ubuntu.com" + echo -n "checking $nodename 's ability to access www.ports.ubuntu.com..........." + if ping -c 3 www.ports.ubuntu.com > /dev/null;then + echo "[OK]" + else + echo "[Failed]" + echo "please make your $nodename has ability to access www.ports.ubuntu.com" + exit 1 + fi + + cp /etc/apt/sources.list /etc/apt/sources.list.bak + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty main/d" /etc/apt/sources.list + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty-updates main/d" /etc/apt/sources.list + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty universe/d" /etc/apt/sources.list + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty-updates universe/d" /etc/apt/sources.list + + cat <<-EOF >>/etc/apt/sources.list +deb http://ports.ubuntu.com/ubuntu-ports/ trusty main +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main +deb http://ports.ubuntu.com/ubuntu-ports/ trusty universe +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates universe +EOF + + apt-get clean all + sleep 1 + apt-get update + fi + + # Being run from a stateful install postscript + # Copy rpms directly from the xCAT management node and install + rm -rf -- /tmp/ofed + mkdir -p /tmp/ofed/mountpoint + cd /tmp/ofed/ + echo "Downloading Mellanox OFED file $OFED_NAME form http://$MASTER/$OFED_DIR .........." + num=`echo "$OFED_DIR" | awk -F"/" '{if($NF==""){print NF-2}else{print NF-1}}'` + wget -l inf -N --waitretry=10 --random-wait --retry-connrefused -t 10 -T 60 -nH --cut-dirs=$num --no-parent "http://$MASTER/$OFED_DIR/$OFED_NAME" 2> /tmp/wget.log + if [ ! -f "/tmp/ofed/$OFED_NAME" ]; then + echo "[Failed]" + rm -rf -- /tmp/ofed + exit 1 + fi + echo "[OK]" + + echo "Mounting Mellanox OFED file $OFED_NAME ........." + mount -o loop "/tmp/ofed/$OFED_NAME" /tmp/ofed/mountpoint + if [ ! -f "/tmp/ofed/mountpoint/mlnxofedinstall" ]; then + echo "[Failed]" + rm -rf -- /tmp/ofed + exit 1 + fi + echo "[OK]" + + echo "Start Mellanox OFED installation ........." + env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall ${MLNXOFED_OPTS[@]} + umount /tmp/ofed/mountpoint + rm -rf -- /tmp/ofed + + #force openibd load all modules in need, restart again + sleep 1 + service openibd restart + fi + + if [[ $NODESETSTATE == "genimage" ]]; then + rm -rf -- "$IMGROOTPATH/tmp/ofed" + mkdir -p "$IMGROOTPATH/tmp/ofed/mountpoint" + cp -r "$OFED_DIR/$OFED_NAME" "$IMGROOTPATH/tmp/ofed/" + + echo "Mounting Mellanox OFED file $OFED_NAME ........." + mount -o loop "$IMGROOTPATH/tmp/ofed/$OFED_NAME" "$IMGROOTPATH/tmp/ofed/mountpoint" + if [ ! -f "$IMGROOTPATH/tmp/ofed/mountpoint/mlnxofedinstall" ]; then + echo "[Failed]" + rm -rf -- "$IMGROOTPATH/tmp/ofed" + exit 1 + fi + echo "[OK]" + + echo "Start Mellanox OFED installation ........." + + # Being called from .postinstall script + # Assume we are on the same machine + if [ -f /etc/SuSE-release ]; then + mount --bind /dev "$IMGROOTPATH/dev/" + mount --bind /sys "$IMGROOTPATH/sys" + mount --bind /proc "$IMGROOTPATH/proc" + chroot "$IMGROOTPATH" rpm -e --noscripts --allmatches mlnx-ofa_kernel-kmp-default 2>/dev/null + chroot "$IMGROOTPATH" rpm -e --nodeps --allmatches libibverbs 2>/dev/null + chroot "$IMGROOTPATH" env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall ${MLNXOFED_OPTS[@]} + umount "$IMGROOTPATH/dev/" + umount "$IMGROOTPATH/sys" + umount "$IMGROOTPATH/proc" + elif grep -q Ubuntu /etc/os-release 2>/dev/null; then + nodename=`hostname` + echo "$nodename 's operating system is Ubuntu." + echo "If you want to install Mellanox_OFED in $nodename, $nodename must have ability to access www.ports.ubuntu.com" + echo -n "checking $nodename 's ability to access www.ports.ubuntu.com..........." + if ping -c 3 www.ports.ubuntu.com > /dev/null; then + echo "[OK]" + else + echo "[Failed]" + echo "please make your $nodename has ability to access www.ports.ubuntu.com" + umount "$IMGROOTPATH/tmp/ofed/mountpoint" + rm -rf -- "$IMGROOTPATH/tmp/ofed" + exit 1 + fi + + sourceslist="$IMGROOTPATH/etc/apt/sources.list" + cp "$sourceslist" "${sourceslist}.bak" + + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty main/d" $sourceslist + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty-updates main/d" $sourceslist + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty universe/d" $sourceslist + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty-updates universe/d" $sourceslist + + cat <<-EOF >>"$sourceslist" +deb http://ports.ubuntu.com/ubuntu-ports/ trusty main +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main +deb http://ports.ubuntu.com/ubuntu-ports/ trusty universe +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates universe +EOF + + chroot "$IMGROOTPATH" apt-get clean all + sleep 1 + chroot "$IMGROOTPATH" apt-get update + mount --bind /dev "$IMGROOTPATH/dev/" + mount --bind /proc "$IMGROOTPATH/proc/" + mount --bind /sys "$IMGROOTPATH/sys/" + + mv "${IMGROOTPATH}/bin/uname" "${IMGROOTPATH}/bin/uname.nouse" + cat <<-EOF >"${IMGROOTPATH}/bin/uname" + #!/bin/sh + case "\$1" in + "-m") + ARCH="\$(dpkg --print-architecture || rpm -q kernel-\$("\$0" -r) --qf '%{arch}')" + case "\$ARCH" in + "amd64") + ARCH="x86_64" + ;; + "ppc64el") + ARCH="ppc64le" + ;; + esac + echo "\$ARCH" + ;; + "-r") + cd /lib/modules && for d in * ; do : ; done && echo \$d + ;; + "-s"|"") + echo "Linux" + ;; + esac + exit 0 + EOF + + chmod 0755 "${IMGROOTPATH}/bin/uname" + + chroot "$IMGROOTPATH" sh -c 'apt-get install -y linux-headers-$(uname -r)' + chroot "$IMGROOTPATH" env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall ${MLNXOFED_OPTS[@]} + + mv "${IMGROOTPATH}/bin/uname.nouse" "${IMGROOTPATH}/bin/uname" + umount "$IMGROOTPATH/dev" + umount "$IMGROOTPATH/proc" + umount "$IMGROOTPATH/sys" + else + chroot "$IMGROOTPATH" rpm -e --nodeps --allmatches libibverbs 2>/dev/null + chroot "$IMGROOTPATH" env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall ${MLNXOFED_OPTS[@]} + + fi + + umount "$IMGROOTPATH/tmp/ofed/mountpoint" + rm -rf -- "$IMGROOTPATH/tmp/ofed" + fi +fi + From c9a3a3ab21439c8dccb6eb31b7f3d7ad299d67e2 Mon Sep 17 00:00:00 2001 From: huweihua Date: Wed, 14 Oct 2015 03:48:06 -0400 Subject: [PATCH 2/5] add ib pkglist for support rhels7.2 on ppc64le --- .../xcat/ib/netboot/rh/ib.rhels7.ppc64le.pkglist | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 xCAT-server/share/xcat/ib/netboot/rh/ib.rhels7.ppc64le.pkglist diff --git a/xCAT-server/share/xcat/ib/netboot/rh/ib.rhels7.ppc64le.pkglist b/xCAT-server/share/xcat/ib/netboot/rh/ib.rhels7.ppc64le.pkglist new file mode 100644 index 000000000..167a390e5 --- /dev/null +++ b/xCAT-server/share/xcat/ib/netboot/rh/ib.rhels7.ppc64le.pkglist @@ -0,0 +1,14 @@ +pciutils-libs +pciutils +tcl +tk +tcsh +gcc-gfortran +lsof +libnl +libxml2-python +python-devel +redhat-rpm-config +rpm-build +kernel-devel + From 600fa8fa00a40c93f4d3dea747c92efd12d7823b Mon Sep 17 00:00:00 2001 From: huweihua Date: Sun, 18 Oct 2015 05:29:15 -0400 Subject: [PATCH 3/5] reduce the number of options --- .../scripts/Mellanox/mlnxofed_ib_install.v2 | 51 ++++++++----------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 b/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 index 178760473..99f7f7ff6 100644 --- a/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 +++ b/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 @@ -19,13 +19,13 @@ # #usage: # -# Copy the xCAT mlnxofed_ib_install.v2 script file to postscripts directory: +# Copy the xCAT mlnxofed_ib_install.v2 script file to postscripts directory and rename to mlnxofed_ib_install: # cp /opt/xcat/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 /install/postscripts/mlnxofed_ib_install # -# Prepare Mellanox OFED ISO file and save it into any subdirectory under /install. but for easy to use and comply with xcat regular rule, recommend using a meaningful directory, such as /install/post/otherpkgs///ofed/ +# Prepare Mellanox OFED ISO file and save it into any subdirectory under /install. # # 1. install the ofed driver for diskfull node -# [NOTE] step 1.1-1.2 only needed by redhat and sles +# [NOTE] step 1.1-1.2 are only needed by redhat and sles # 1.1 copy the pkglist to the custom directory: # cp /opt/xcat/share/xcat/install//compute...pkglist /install/custom/install//compute...pkglist # Edit your /install/custom/install//compute...pkglist and add: @@ -33,23 +33,23 @@ # 1.2 Make the related osimage use the customized pkglist. # chdef -t osimage -o --install-compute pkglist=/install/custom/install//compute...pkglist # 1.3 set mlnxofed_ib_install as postbootscripts for the target node. assign necessary attribute for mlnxofed_ib_install at same time. -# chdef -p postbootscripts="mlnxofed_ib_install -ofeddir -ofedname " +# chdef -p postbootscripts="mlnxofed_ib_install -p /install//" # [NOTE] The default options input into Mellanox are '--without-32bit --without-fw-update --force' -# you can appoint the options by yourslef with '-passmlnxofedoptions' option of mlnxofed_ib_install -# In order to distinguish which options are tranfered to Mellanox and which options are belong to mlnxofed_ib_install, any options wanted to transfered to Mellanox must follow behind -passmlnxofedoptions and end with "-end-", for example: -# chdef -p postbootscripts="mlnxofed_ib_install -ofeddir -passmlnxofedoptions --without-32bit --add-kernel-support --force -end- -ofedname < OFED ISO file name>" +# you can appoint the options by yourslef with '-m' option of mlnxofed_ib_install +# In order to distinguish which options are tranfered to Mellanox and which options are belong to mlnxofed_ib_install, any options wanted to transfered to Mellanox must follow behind -m and end with "-end-", for example: +# chdef -p postbootscripts="mlnxofed_ib_install -p /install// -m --without-32bit --add-kernel-support --force -end-" # 1.4 do the diskfull installation # nodeset osimage= ...... # # 2. install the ofed driver for diskless images: -# [NOTE] step 2.1 only needed by redhat and sles +# [NOTE] step 2.1 is only needed by redhat and sles # 2.1 copy the pkglist to the custom directory: # cp /opt/xcat/share/xcat/netboot//compute...pkglist /install/custom/netboot//compute...pkglist # Edit your /install/custom/netboot//.pkglist and add: # #INCLUDE:/opt/xcat/share/xcat/ib/netboot//ib...pkglist# # 2.2 Add to postinstall scripts # Edit your /install/custom/netboot//.postinstall and add: -# /install/postscripts/mlnxofed_ib_install -ofeddir -ofedname -nodesetstate genimage -installroot $1 +# /install/postscripts/mlnxofed_ib_install -p /install// -n genimage -i $1 # 2.3 Make sure the related osimage use the customized pkglist and customized compute.postinsall # lsdef -t osimage -o --netboot-compute # if not, change it: @@ -73,21 +73,20 @@ function usage() { echo "Usage: mlnxofed_ib_install [-attribute]" echo " attribute include:" echo " -h: print this help message" - echo " -ofeddir: the directory where OFED file is saved. this is necessary attribute" - echo " -ofedname: the name of OFED file. this is necessary attribute" - echo " -passmlnxofedoptions: the options inputted into mlnxofedinstall script, defualt value are --without-32bit --without-fw-update --force" - echo " -installroot: the image root path. this is necessary attribute in diskless scenario" - echo " -nodesetstate: nodeset status, the value are install, boot or genimage" + echo " -p: the path where OFED file is saved. this is necessary attribute" + echo " -m: the options inputted into mlnxofedinstall script, defualt value are --without-32bit --without-fw-update --force" + echo " -i: the image root path. this is necessary attribute in diskless scenario" + echo " -n: nodeset status, the value are one of install, boot or genimage" } while [ "$#" -gt "0" ] do case "$1" in - "-ofeddir") + "-p") shift - OFED_DIR="$1" + OFED_PATH="$1" ;; - "-passmlnxofedoptions") + "-m") shift while [ "-end-" != "$1" ] do @@ -95,15 +94,11 @@ do shift done ;; - "-ofedname") - shift - OFED_NAME="$1" - ;; - "-installroot") + "-i") shift IMGROOTPATH="$1" ;; - "-nodesetstate") + "-n") shift NODESETSTATE="$1" ;; @@ -119,16 +114,11 @@ do shift done -if [ -z "$OFED_DIR" ]; then +if [ -z "$OFED_PATH" ]; then echo "[Error] Without Mellanox OFED file path, please assign correct path" exit 1 fi -if [ -z "$OFED_NAME" ]; then - echo "[Error] Without Mellanox OFED file name, please assign correct name" - exit 1 -fi - if [ "$NODESETSTATE" = "genimage" -a -z "$IMGROOTPATH" ]; then echo "[Error] this is for diskless installation, please assign correct diskless image root path" exit 1 @@ -136,6 +126,9 @@ fi [ "${#MLNXOFED_OPTS}" = 0 ] && MLNXOFED_OPTS=(--without-32bit --without-fw-update --force) +OFED_DIR=${OFED_PATH%/*} +OFED_NAME=${OFED_PATH##*/} + echo "Mellanox OFED file path is $OFED_DIR" echo "Mellanox OFED file is $OFED_NAME" echo "Mellanox OFED options are ${MLNXOFED_OPTS[@]}" From 73958c40c241b787d9c48363511fe67f07be02d5 Mon Sep 17 00:00:00 2001 From: huweihua Date: Mon, 19 Oct 2015 08:28:15 -0400 Subject: [PATCH 4/5] modify code depending on gongjie's comments --- .../scripts/Mellanox/mlnxofed_ib_install.v2 | 78 +++++++++++-------- 1 file changed, 47 insertions(+), 31 deletions(-) diff --git a/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 b/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 index 99f7f7ff6..b25725dba 100644 --- a/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 +++ b/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # This is the second version of Mellanox IB driver installation sample script # This is also the version we recommend to use right now. @@ -63,7 +63,7 @@ OS="$(uname)" if [ "$OS" = "Linux" ]; then str_dir_name="${0%/*}" - . "$str_dir_name/xcatlib.sh" + . "$str_dir_name/xcatlib.sh" 2>/dev/null fi #-------------------------------------------------------- @@ -114,17 +114,17 @@ do shift done -if [ -z "$OFED_PATH" ]; then +if [ ! -f "$OFED_PATH" ]; then echo "[Error] Without Mellanox OFED file path, please assign correct path" exit 1 fi -if [ "$NODESETSTATE" = "genimage" -a -z "$IMGROOTPATH" ]; then +if [ "$NODESETSTATE" = "genimage" -a ! -d "$IMGROOTPATH" ]; then echo "[Error] this is for diskless installation, please assign correct diskless image root path" exit 1 fi -[ "${#MLNXOFED_OPTS}" = 0 ] && MLNXOFED_OPTS=(--without-32bit --without-fw-update --force) +[ "${#MLNXOFED_OPTS[@]}" = 0 ] && MLNXOFED_OPTS=(--without-32bit --without-fw-update --force) OFED_DIR=${OFED_PATH%/*} OFED_NAME=${OFED_PATH##*/} @@ -135,9 +135,42 @@ echo "Mellanox OFED options are ${MLNXOFED_OPTS[@]}" echo "image root path is $IMGROOTPATH" echo "NODESETSTATE is $NODESETSTATE" + +function cleanup() +{ + if [ "$NODESETSTATE" != "genimage" ]; then + if mount |grep "/tmp/ofed/mountpoint"; then + umount /tmp/ofed/mountpoint + fi + if [ -d "/tmp/ofed" ]; then + rm -rf -- /tmp/ofed + fi + else + if mount |grep "$IMGROOTPATH/tmp/ofed/mountpoint"; then + umount "$IMGROOTPATH/tmp/ofed/mountpoint" + fi + if mount |grep "$IMGROOTPATH/dev/"; then + umount "$IMGROOTPATH/dev/" + fi + if mount |grep "$IMGROOTPATH/sys"; then + umount "$IMGROOTPATH/sys" + fi + if mount |grep "$IMGROOTPATH/proc"; then + umount "$IMGROOTPATH/proc" + fi + if [ -d "$IMGROOTPATH/tmp/ofed" ]; then + rm -rf -- "$IMGROOTPATH/tmp/ofed" + fi + fi + +} + +trap 'cleanup' 0 + + if [ "$OS" != "AIX" ]; then if [ "$NODESETSTATE" = "install" -o "$NODESETSTATE" = "boot" ]; then - + #if the host is ubuntn, need to do some network check and configuration if grep -q Ubuntu /etc/os-release 2>/dev/null then @@ -145,7 +178,7 @@ if [ "$OS" != "AIX" ]; then echo "$nodename 's operating system is Ubuntu." echo "If you want to install Mellanox_OFED in $nodename, $nodename must have ability to access www.ports.ubuntu.com" echo -n "checking $nodename 's ability to access www.ports.ubuntu.com..........." - if ping -c 3 www.ports.ubuntu.com > /dev/null;then + if ping -c 3 ports.ubuntu.com > /dev/null;then echo "[OK]" else echo "[Failed]" @@ -181,7 +214,6 @@ EOF wget -l inf -N --waitretry=10 --random-wait --retry-connrefused -t 10 -T 60 -nH --cut-dirs=$num --no-parent "http://$MASTER/$OFED_DIR/$OFED_NAME" 2> /tmp/wget.log if [ ! -f "/tmp/ofed/$OFED_NAME" ]; then echo "[Failed]" - rm -rf -- /tmp/ofed exit 1 fi echo "[OK]" @@ -190,15 +222,12 @@ EOF mount -o loop "/tmp/ofed/$OFED_NAME" /tmp/ofed/mountpoint if [ ! -f "/tmp/ofed/mountpoint/mlnxofedinstall" ]; then echo "[Failed]" - rm -rf -- /tmp/ofed exit 1 fi echo "[OK]" echo "Start Mellanox OFED installation ........." - env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall ${MLNXOFED_OPTS[@]} - umount /tmp/ofed/mountpoint - rm -rf -- /tmp/ofed + env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall "${MLNXOFED_OPTS[@]}" #force openibd load all modules in need, restart again sleep 1 @@ -208,13 +237,11 @@ EOF if [[ $NODESETSTATE == "genimage" ]]; then rm -rf -- "$IMGROOTPATH/tmp/ofed" mkdir -p "$IMGROOTPATH/tmp/ofed/mountpoint" - cp -r "$OFED_DIR/$OFED_NAME" "$IMGROOTPATH/tmp/ofed/" - echo "Mounting Mellanox OFED file $OFED_NAME ........." - mount -o loop "$IMGROOTPATH/tmp/ofed/$OFED_NAME" "$IMGROOTPATH/tmp/ofed/mountpoint" + echo "Mounting Mellanox OFED file $OFED_DIR/$OFED_NAME ........." + mount -o loop "$OFED_DIR/$OFED_NAME" "$IMGROOTPATH/tmp/ofed/mountpoint" if [ ! -f "$IMGROOTPATH/tmp/ofed/mountpoint/mlnxofedinstall" ]; then echo "[Failed]" - rm -rf -- "$IMGROOTPATH/tmp/ofed" exit 1 fi echo "[OK]" @@ -229,22 +256,17 @@ EOF mount --bind /proc "$IMGROOTPATH/proc" chroot "$IMGROOTPATH" rpm -e --noscripts --allmatches mlnx-ofa_kernel-kmp-default 2>/dev/null chroot "$IMGROOTPATH" rpm -e --nodeps --allmatches libibverbs 2>/dev/null - chroot "$IMGROOTPATH" env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall ${MLNXOFED_OPTS[@]} - umount "$IMGROOTPATH/dev/" - umount "$IMGROOTPATH/sys" - umount "$IMGROOTPATH/proc" + chroot "$IMGROOTPATH" env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall "${MLNXOFED_OPTS[@]}" elif grep -q Ubuntu /etc/os-release 2>/dev/null; then nodename=`hostname` echo "$nodename 's operating system is Ubuntu." echo "If you want to install Mellanox_OFED in $nodename, $nodename must have ability to access www.ports.ubuntu.com" echo -n "checking $nodename 's ability to access www.ports.ubuntu.com..........." - if ping -c 3 www.ports.ubuntu.com > /dev/null; then + if ping -c 3 ports.ubuntu.com > /dev/null; then echo "[OK]" else echo "[Failed]" echo "please make your $nodename has ability to access www.ports.ubuntu.com" - umount "$IMGROOTPATH/tmp/ofed/mountpoint" - rm -rf -- "$IMGROOTPATH/tmp/ofed" exit 1 fi @@ -299,20 +321,14 @@ EOF chmod 0755 "${IMGROOTPATH}/bin/uname" chroot "$IMGROOTPATH" sh -c 'apt-get install -y linux-headers-$(uname -r)' - chroot "$IMGROOTPATH" env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall ${MLNXOFED_OPTS[@]} + chroot "$IMGROOTPATH" env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall "${MLNXOFED_OPTS[@]}" mv "${IMGROOTPATH}/bin/uname.nouse" "${IMGROOTPATH}/bin/uname" - umount "$IMGROOTPATH/dev" - umount "$IMGROOTPATH/proc" - umount "$IMGROOTPATH/sys" else chroot "$IMGROOTPATH" rpm -e --nodeps --allmatches libibverbs 2>/dev/null - chroot "$IMGROOTPATH" env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall ${MLNXOFED_OPTS[@]} + chroot "$IMGROOTPATH" env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall "${MLNXOFED_OPTS[@]}" fi - - umount "$IMGROOTPATH/tmp/ofed/mountpoint" - rm -rf -- "$IMGROOTPATH/tmp/ofed" fi fi From 2424460fb5f8010d3b50fb774e8abd5f821bb2e4 Mon Sep 17 00:00:00 2001 From: huweihua Date: Tue, 20 Oct 2015 01:44:02 -0400 Subject: [PATCH 5/5] modify depending on the second round comments --- .../scripts/Mellanox/mlnxofed_ib_install.v2 | 61 +++++++++++-------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 b/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 index b25725dba..222aceaa6 100644 --- a/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 +++ b/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 @@ -115,12 +115,12 @@ do done if [ ! -f "$OFED_PATH" ]; then - echo "[Error] Without Mellanox OFED file path, please assign correct path" + echo "[Error] Without Mellanox OFED file path, please assign correct path" >&2 exit 1 fi if [ "$NODESETSTATE" = "genimage" -a ! -d "$IMGROOTPATH" ]; then - echo "[Error] this is for diskless installation, please assign correct diskless image root path" + echo "[Error] this is for diskless installation, please assign correct diskless image root path" >&2 exit 1 fi @@ -168,21 +168,20 @@ function cleanup() trap 'cleanup' 0 -if [ "$OS" != "AIX" ]; then +if [ "$OS" = "Linux" ]; then if [ "$NODESETSTATE" = "install" -o "$NODESETSTATE" = "boot" ]; then #if the host is ubuntn, need to do some network check and configuration if grep -q Ubuntu /etc/os-release 2>/dev/null then - nodename=`hostname` - echo "$nodename 's operating system is Ubuntu." - echo "If you want to install Mellanox_OFED in $nodename, $nodename must have ability to access www.ports.ubuntu.com" - echo -n "checking $nodename 's ability to access www.ports.ubuntu.com..........." + echo "$HOSTNAME 's operating system is Ubuntu." + echo "If you want to install Mellanox_OFED in $HOSTNAME, $HOSTNAME must have ability to access ports.ubuntu.com" + echo -n "checking $HOSTNAME 's ability to access ports.ubuntu.com..........." if ping -c 3 ports.ubuntu.com > /dev/null;then echo "[OK]" else - echo "[Failed]" - echo "please make your $nodename has ability to access www.ports.ubuntu.com" + echo "[Failed]" >&2 + echo "[Error] please make your $HOSTNAME has ability to access ports.ubuntu.com" >&2 exit 1 fi @@ -205,23 +204,31 @@ EOF fi # Being run from a stateful install postscript - # Copy rpms directly from the xCAT management node and install + # Copy ISO directly from the xCAT management node and install rm -rf -- /tmp/ofed mkdir -p /tmp/ofed/mountpoint + if [ "$?" != "0" ] ;then + echo "[Error] We are in trouble to mkdir /tmp/ofed/mountpoint, please check your node" >&2 + exit 1 + fi + cd /tmp/ofed/ echo "Downloading Mellanox OFED file $OFED_NAME form http://$MASTER/$OFED_DIR .........." - num=`echo "$OFED_DIR" | awk -F"/" '{if($NF==""){print NF-2}else{print NF-1}}'` - wget -l inf -N --waitretry=10 --random-wait --retry-connrefused -t 10 -T 60 -nH --cut-dirs=$num --no-parent "http://$MASTER/$OFED_DIR/$OFED_NAME" 2> /tmp/wget.log + wget -l inf -N --waitretry=10 --random-wait --retry-connrefused -t 10 -T 60 -nH --no-parent "http://$MASTER/$OFED_DIR/$OFED_NAME" 2> /tmp/wget.log + if [ "$?" != "0" ]; then + echo "[Error] Downloading Mellanox OFED file $OFED_NAME failed" >&2 + exit 1 + fi if [ ! -f "/tmp/ofed/$OFED_NAME" ]; then - echo "[Failed]" + echo "[Failed]" >&2 exit 1 fi echo "[OK]" echo "Mounting Mellanox OFED file $OFED_NAME ........." mount -o loop "/tmp/ofed/$OFED_NAME" /tmp/ofed/mountpoint - if [ ! -f "/tmp/ofed/mountpoint/mlnxofedinstall" ]; then - echo "[Failed]" + if [ ! -f "/tmp/ofed/mountpoint/mlnxofedinstall" -o ! -x "/tmp/ofed/mountpoint/mlnxofedinstall" ]; then + echo "[Failed]" >&2 exit 1 fi echo "[OK]" @@ -234,14 +241,18 @@ EOF service openibd restart fi - if [[ $NODESETSTATE == "genimage" ]]; then + if [[ "$NODESETSTATE" == "genimage" ]]; then rm -rf -- "$IMGROOTPATH/tmp/ofed" mkdir -p "$IMGROOTPATH/tmp/ofed/mountpoint" - + if [ "$?" != "0" ] ;then + echo "[Error] We are in trouble to mkdir $IMGROOTPATH/tmp/ofed/mountpoint, please check your node" >&2 + exit 1 + fi + echo "Mounting Mellanox OFED file $OFED_DIR/$OFED_NAME ........." mount -o loop "$OFED_DIR/$OFED_NAME" "$IMGROOTPATH/tmp/ofed/mountpoint" - if [ ! -f "$IMGROOTPATH/tmp/ofed/mountpoint/mlnxofedinstall" ]; then - echo "[Failed]" + if [ ! -f "$IMGROOTPATH/tmp/ofed/mountpoint/mlnxofedinstall" -o ! -x "$IMGROOTPATH/tmp/ofed/mountpoint/mlnxofedinstall" ]; then + echo "[Failed]" >&2 exit 1 fi echo "[OK]" @@ -258,21 +269,21 @@ EOF chroot "$IMGROOTPATH" rpm -e --nodeps --allmatches libibverbs 2>/dev/null chroot "$IMGROOTPATH" env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall "${MLNXOFED_OPTS[@]}" elif grep -q Ubuntu /etc/os-release 2>/dev/null; then - nodename=`hostname` - echo "$nodename 's operating system is Ubuntu." - echo "If you want to install Mellanox_OFED in $nodename, $nodename must have ability to access www.ports.ubuntu.com" - echo -n "checking $nodename 's ability to access www.ports.ubuntu.com..........." + echo "$HOSTNAME 's operating system is Ubuntu." + echo "If you want to install Mellanox_OFED in $HOSTNAME, $HOSTNAME must have ability to access ports.ubuntu.com" + echo -n "checking $HOSTNAME 's ability to access ports.ubuntu.com..........." if ping -c 3 ports.ubuntu.com > /dev/null; then echo "[OK]" else - echo "[Failed]" - echo "please make your $nodename has ability to access www.ports.ubuntu.com" + echo "[Failed]" >&2 + echo "please make your $HOSTNAME has ability to access ports.ubuntu.com" >&2 exit 1 fi sourceslist="$IMGROOTPATH/etc/apt/sources.list" cp "$sourceslist" "${sourceslist}.bak" + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty main/d" $sourceslist sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty main/d" $sourceslist sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty-updates main/d" $sourceslist sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty universe/d" $sourceslist