diff --git a/xCAT-server/share/xcat/ib/netboot/rh/ib.rhels7.ppc64le.pkglist b/xCAT-server/share/xcat/ib/netboot/rh/ib.rhels7.ppc64le.pkglist new file mode 100644 index 000000000..167a390e5 --- /dev/null +++ b/xCAT-server/share/xcat/ib/netboot/rh/ib.rhels7.ppc64le.pkglist @@ -0,0 +1,14 @@ +pciutils-libs +pciutils +tcl +tk +tcsh +gcc-gfortran +lsof +libnl +libxml2-python +python-devel +redhat-rpm-config +rpm-build +kernel-devel + diff --git a/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 b/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 new file mode 100644 index 000000000..222aceaa6 --- /dev/null +++ b/xCAT-server/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 @@ -0,0 +1,345 @@ +#!/bin/bash +# +# This is the second version of Mellanox IB driver installation sample script +# This is also the version we recommend to use right now. +# What are changed in this version +# 1. The usage interface is changed, using command line arguments instead of using environment attribute +# 2. The way of downloading Mellanox OFED files. +# 3. Some structure of code +# +# For AIX: +# TBD +# For Linux: +# - For full-disk installs: +# - Copy Mellanox OFED ISO to node +# - Install IB rpms +# - For diskless images: +# - Copy the packages to the images. +# - Install IB rpms +# +#usage: +# +# Copy the xCAT mlnxofed_ib_install.v2 script file to postscripts directory and rename to mlnxofed_ib_install: +# cp /opt/xcat/share/xcat/ib/scripts/Mellanox/mlnxofed_ib_install.v2 /install/postscripts/mlnxofed_ib_install +# +# Prepare Mellanox OFED ISO file and save it into any subdirectory under /install. +# +# 1. install the ofed driver for diskfull node +# [NOTE] step 1.1-1.2 are only needed by redhat and sles +# 1.1 copy the pkglist to the custom directory: +# cp /opt/xcat/share/xcat/install//compute...pkglist /install/custom/install//compute...pkglist +# Edit your /install/custom/install//compute...pkglist and add: +# #INCLUDE:/opt/xcat/share/xcat/ib/netboot//ib...pkglist# +# 1.2 Make the related osimage use the customized pkglist. +# chdef -t osimage -o --install-compute pkglist=/install/custom/install//compute...pkglist +# 1.3 set mlnxofed_ib_install as postbootscripts for the target node. assign necessary attribute for mlnxofed_ib_install at same time. +# chdef -p postbootscripts="mlnxofed_ib_install -p /install//" +# [NOTE] The default options input into Mellanox are '--without-32bit --without-fw-update --force' +# you can appoint the options by yourslef with '-m' option of mlnxofed_ib_install +# In order to distinguish which options are tranfered to Mellanox and which options are belong to mlnxofed_ib_install, any options wanted to transfered to Mellanox must follow behind -m and end with "-end-", for example: +# chdef -p postbootscripts="mlnxofed_ib_install -p /install// -m --without-32bit --add-kernel-support --force -end-" +# 1.4 do the diskfull installation +# nodeset osimage= ...... +# +# 2. install the ofed driver for diskless images: +# [NOTE] step 2.1 is only needed by redhat and sles +# 2.1 copy the pkglist to the custom directory: +# cp /opt/xcat/share/xcat/netboot//compute...pkglist /install/custom/netboot//compute...pkglist +# Edit your /install/custom/netboot//.pkglist and add: +# #INCLUDE:/opt/xcat/share/xcat/ib/netboot//ib...pkglist# +# 2.2 Add to postinstall scripts +# Edit your /install/custom/netboot//.postinstall and add: +# /install/postscripts/mlnxofed_ib_install -p /install// -n genimage -i $1 +# 2.3 Make sure the related osimage use the customized pkglist and customized compute.postinsall +# lsdef -t osimage -o --netboot-compute +# if not, change it: +# chdef -t osimage -o --netboot-compute pkglist=/install/custom/netboot//compute...pkglist postinstall=/install/custom/netboot//.postinstall +# 2.4 run genimage +# genimage --netboot-compute +# + +#set -x + +OS="$(uname)" +if [ "$OS" = "Linux" ]; then + str_dir_name="${0%/*}" + . "$str_dir_name/xcatlib.sh" 2>/dev/null +fi + +#-------------------------------------------------------- +declare -a MLNXOFED_OPTS + +function usage() { + echo "Usage: mlnxofed_ib_install [-attribute]" + echo " attribute include:" + echo " -h: print this help message" + echo " -p: the path where OFED file is saved. this is necessary attribute" + echo " -m: the options inputted into mlnxofedinstall script, defualt value are --without-32bit --without-fw-update --force" + echo " -i: the image root path. this is necessary attribute in diskless scenario" + echo " -n: nodeset status, the value are one of install, boot or genimage" +} + +while [ "$#" -gt "0" ] +do + case "$1" in + "-p") + shift + OFED_PATH="$1" + ;; + "-m") + shift + while [ "-end-" != "$1" ] + do + MLNXOFED_OPTS=("${MLNXOFED_OPTS[@]}" "$1") + shift + done + ;; + "-i") + shift + IMGROOTPATH="$1" + ;; + "-n") + shift + NODESETSTATE="$1" + ;; + "-h") + usage + exit 0 + ;; + *) + echo "unsupport attribute $1" + exit 0 + ;; + esac + shift +done + +if [ ! -f "$OFED_PATH" ]; then + echo "[Error] Without Mellanox OFED file path, please assign correct path" >&2 + exit 1 +fi + +if [ "$NODESETSTATE" = "genimage" -a ! -d "$IMGROOTPATH" ]; then + echo "[Error] this is for diskless installation, please assign correct diskless image root path" >&2 + exit 1 +fi + +[ "${#MLNXOFED_OPTS[@]}" = 0 ] && MLNXOFED_OPTS=(--without-32bit --without-fw-update --force) + +OFED_DIR=${OFED_PATH%/*} +OFED_NAME=${OFED_PATH##*/} + +echo "Mellanox OFED file path is $OFED_DIR" +echo "Mellanox OFED file is $OFED_NAME" +echo "Mellanox OFED options are ${MLNXOFED_OPTS[@]}" +echo "image root path is $IMGROOTPATH" +echo "NODESETSTATE is $NODESETSTATE" + + +function cleanup() +{ + if [ "$NODESETSTATE" != "genimage" ]; then + if mount |grep "/tmp/ofed/mountpoint"; then + umount /tmp/ofed/mountpoint + fi + if [ -d "/tmp/ofed" ]; then + rm -rf -- /tmp/ofed + fi + else + if mount |grep "$IMGROOTPATH/tmp/ofed/mountpoint"; then + umount "$IMGROOTPATH/tmp/ofed/mountpoint" + fi + if mount |grep "$IMGROOTPATH/dev/"; then + umount "$IMGROOTPATH/dev/" + fi + if mount |grep "$IMGROOTPATH/sys"; then + umount "$IMGROOTPATH/sys" + fi + if mount |grep "$IMGROOTPATH/proc"; then + umount "$IMGROOTPATH/proc" + fi + if [ -d "$IMGROOTPATH/tmp/ofed" ]; then + rm -rf -- "$IMGROOTPATH/tmp/ofed" + fi + fi + +} + +trap 'cleanup' 0 + + +if [ "$OS" = "Linux" ]; then + if [ "$NODESETSTATE" = "install" -o "$NODESETSTATE" = "boot" ]; then + + #if the host is ubuntn, need to do some network check and configuration + if grep -q Ubuntu /etc/os-release 2>/dev/null + then + echo "$HOSTNAME 's operating system is Ubuntu." + echo "If you want to install Mellanox_OFED in $HOSTNAME, $HOSTNAME must have ability to access ports.ubuntu.com" + echo -n "checking $HOSTNAME 's ability to access ports.ubuntu.com..........." + if ping -c 3 ports.ubuntu.com > /dev/null;then + echo "[OK]" + else + echo "[Failed]" >&2 + echo "[Error] please make your $HOSTNAME has ability to access ports.ubuntu.com" >&2 + exit 1 + fi + + cp /etc/apt/sources.list /etc/apt/sources.list.bak + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty main/d" /etc/apt/sources.list + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty-updates main/d" /etc/apt/sources.list + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty universe/d" /etc/apt/sources.list + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty-updates universe/d" /etc/apt/sources.list + + cat <<-EOF >>/etc/apt/sources.list +deb http://ports.ubuntu.com/ubuntu-ports/ trusty main +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main +deb http://ports.ubuntu.com/ubuntu-ports/ trusty universe +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates universe +EOF + + apt-get clean all + sleep 1 + apt-get update + fi + + # Being run from a stateful install postscript + # Copy ISO directly from the xCAT management node and install + rm -rf -- /tmp/ofed + mkdir -p /tmp/ofed/mountpoint + if [ "$?" != "0" ] ;then + echo "[Error] We are in trouble to mkdir /tmp/ofed/mountpoint, please check your node" >&2 + exit 1 + fi + + cd /tmp/ofed/ + echo "Downloading Mellanox OFED file $OFED_NAME form http://$MASTER/$OFED_DIR .........." + wget -l inf -N --waitretry=10 --random-wait --retry-connrefused -t 10 -T 60 -nH --no-parent "http://$MASTER/$OFED_DIR/$OFED_NAME" 2> /tmp/wget.log + if [ "$?" != "0" ]; then + echo "[Error] Downloading Mellanox OFED file $OFED_NAME failed" >&2 + exit 1 + fi + if [ ! -f "/tmp/ofed/$OFED_NAME" ]; then + echo "[Failed]" >&2 + exit 1 + fi + echo "[OK]" + + echo "Mounting Mellanox OFED file $OFED_NAME ........." + mount -o loop "/tmp/ofed/$OFED_NAME" /tmp/ofed/mountpoint + if [ ! -f "/tmp/ofed/mountpoint/mlnxofedinstall" -o ! -x "/tmp/ofed/mountpoint/mlnxofedinstall" ]; then + echo "[Failed]" >&2 + exit 1 + fi + echo "[OK]" + + echo "Start Mellanox OFED installation ........." + env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall "${MLNXOFED_OPTS[@]}" + + #force openibd load all modules in need, restart again + sleep 1 + service openibd restart + fi + + if [[ "$NODESETSTATE" == "genimage" ]]; then + rm -rf -- "$IMGROOTPATH/tmp/ofed" + mkdir -p "$IMGROOTPATH/tmp/ofed/mountpoint" + if [ "$?" != "0" ] ;then + echo "[Error] We are in trouble to mkdir $IMGROOTPATH/tmp/ofed/mountpoint, please check your node" >&2 + exit 1 + fi + + echo "Mounting Mellanox OFED file $OFED_DIR/$OFED_NAME ........." + mount -o loop "$OFED_DIR/$OFED_NAME" "$IMGROOTPATH/tmp/ofed/mountpoint" + if [ ! -f "$IMGROOTPATH/tmp/ofed/mountpoint/mlnxofedinstall" -o ! -x "$IMGROOTPATH/tmp/ofed/mountpoint/mlnxofedinstall" ]; then + echo "[Failed]" >&2 + exit 1 + fi + echo "[OK]" + + echo "Start Mellanox OFED installation ........." + + # Being called from .postinstall script + # Assume we are on the same machine + if [ -f /etc/SuSE-release ]; then + mount --bind /dev "$IMGROOTPATH/dev/" + mount --bind /sys "$IMGROOTPATH/sys" + mount --bind /proc "$IMGROOTPATH/proc" + chroot "$IMGROOTPATH" rpm -e --noscripts --allmatches mlnx-ofa_kernel-kmp-default 2>/dev/null + chroot "$IMGROOTPATH" rpm -e --nodeps --allmatches libibverbs 2>/dev/null + chroot "$IMGROOTPATH" env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall "${MLNXOFED_OPTS[@]}" + elif grep -q Ubuntu /etc/os-release 2>/dev/null; then + echo "$HOSTNAME 's operating system is Ubuntu." + echo "If you want to install Mellanox_OFED in $HOSTNAME, $HOSTNAME must have ability to access ports.ubuntu.com" + echo -n "checking $HOSTNAME 's ability to access ports.ubuntu.com..........." + if ping -c 3 ports.ubuntu.com > /dev/null; then + echo "[OK]" + else + echo "[Failed]" >&2 + echo "please make your $HOSTNAME has ability to access ports.ubuntu.com" >&2 + exit 1 + fi + + sourceslist="$IMGROOTPATH/etc/apt/sources.list" + cp "$sourceslist" "${sourceslist}.bak" + + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty main/d" $sourceslist + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty main/d" $sourceslist + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty-updates main/d" $sourceslist + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty universe/d" $sourceslist + sed -i "/deb http:\\/\\/ports.ubuntu.com\\/ubuntu-ports\\/ trusty-updates universe/d" $sourceslist + + cat <<-EOF >>"$sourceslist" +deb http://ports.ubuntu.com/ubuntu-ports/ trusty main +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main +deb http://ports.ubuntu.com/ubuntu-ports/ trusty universe +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates universe +EOF + + chroot "$IMGROOTPATH" apt-get clean all + sleep 1 + chroot "$IMGROOTPATH" apt-get update + mount --bind /dev "$IMGROOTPATH/dev/" + mount --bind /proc "$IMGROOTPATH/proc/" + mount --bind /sys "$IMGROOTPATH/sys/" + + mv "${IMGROOTPATH}/bin/uname" "${IMGROOTPATH}/bin/uname.nouse" + cat <<-EOF >"${IMGROOTPATH}/bin/uname" + #!/bin/sh + case "\$1" in + "-m") + ARCH="\$(dpkg --print-architecture || rpm -q kernel-\$("\$0" -r) --qf '%{arch}')" + case "\$ARCH" in + "amd64") + ARCH="x86_64" + ;; + "ppc64el") + ARCH="ppc64le" + ;; + esac + echo "\$ARCH" + ;; + "-r") + cd /lib/modules && for d in * ; do : ; done && echo \$d + ;; + "-s"|"") + echo "Linux" + ;; + esac + exit 0 + EOF + + chmod 0755 "${IMGROOTPATH}/bin/uname" + + chroot "$IMGROOTPATH" sh -c 'apt-get install -y linux-headers-$(uname -r)' + chroot "$IMGROOTPATH" env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall "${MLNXOFED_OPTS[@]}" + + mv "${IMGROOTPATH}/bin/uname.nouse" "${IMGROOTPATH}/bin/uname" + else + chroot "$IMGROOTPATH" rpm -e --nodeps --allmatches libibverbs 2>/dev/null + chroot "$IMGROOTPATH" env -i "PATH=${PATH}" /tmp/ofed/mountpoint/mlnxofedinstall "${MLNXOFED_OPTS[@]}" + + fi + fi +fi +