start:xcatd_systemd_has_no_legacy_init_script
description:systemd installations do not retain the legacy xcatd init script
label:mn_only,ci_test,xcatd
cmd:test -d /run/systemd/system && test ! -f /etc/init.d/xcatd
check:rc==0
end


start:xcatd_start
description:stop then start xcatd daemon, and check all the 6 processes are running
label:mn_only,ci_test,xcatd
cmd:if [ -d "/tmp/xcatd_start" ]; then mv /tmp/xcatd_start /tmp/xcatd_start.org; fi; mkdir -p /tmp/xcatd_start
check:rc==0
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
check:rc==0
check:output=~xcatd service|xcatd.service
check:output=~Active: active \(running\)|xcatd service is running
cmd:ps axjf |grep -v grep |grep "xcatd:" | tee /tmp/xcatd_start/original_xcatd_processes_status
check:rc==0
cmd:cat /tmp/xcatd_start/original_xcatd_processes_status |wc -l
#check:output=~6
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl stop xcatd; else service xcatd stop; fi
check:rc==0
cmd:sleep 3
cmd:ps axjf |grep -v grep |grep "xcatd:" | tee /tmp/xcatd_start/after_stop_xcatd_processes_status
check:rc==0
cmd:cat /tmp/xcatd_start/after_stop_xcatd_processes_status|wc -l
check:output=~0
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl start xcatd; else service xcatd start; fi
check:rc==0
cmd:sleep 3
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
check:rc==0
check:output=~xcatd service|xcatd.service
check:output=~Active: active \(running\)|xcatd service is running
cmd:ps axjf |grep -v grep |grep "xcatd:" | tee /tmp/xcatd_start/after_start_xcatd_processes_status
check:rc==0
cmd:cat  /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
check:output=~6
cmd:grep "xcatd: SSL listener" /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
check:output=~1
cmd:grep "xcatd: DB Access" /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
check:output=~1
cmd:grep "xcatd: UDP listener" /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
check:output=~1
cmd:grep "xcatd: Discovery worker" /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
check:output=~1
cmd:grep "xcatd: install monitor" /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
check:output=~1
cmd:grep "xcatd: Command log writer" /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
check:output=~1
cmd: rm -rf /tmp/xcatd_start; if [ -d "/tmp/xcatd_start.org" ]; then mv /tmp/xcatd_start.org /tmp/xcatd_start; fi
check:rc==0
end


start:xcatd_stop
description:stop xcatd daemon, then check all the 6 processes are stopped
label:mn_only,ci_test,xcatd
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
check:rc==0
check:output=~xcatd service|xcatd.service
check:output=~xcatd service is running|active \(running\)
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl stop xcatd; else service xcatd stop; fi
check:rc==0
cmd:sleep 3
cmd:ps axjf |grep -v grep |grep "xcatd:" | wc -l
check:rc==0
check:output=~0
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
check:output=~xcatd service|xcatd.service
check:output=~xcatd service is not running|inactive \(dead\)
cmd:ps -ef | grep xcatd
check:output!~"xcatd: SSL listener"
check:output!~"xcatd: DB Access"
check:output!~"xcatd: UDP listener"
check:output!~"xcatd: install monitor"
check:output!~"xcatd: Discovery worke"
check:output!~"xcatd: Command log writer"
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl start xcatd; else service xcatd start; fi
check:rc==0
end


start:xcatd_restart
description:restart xcatd daemon
label:mn_only,xcatd
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
check:output=~xcatd service|xcatd.service
check:output=~xcatd service is running|active \(running\)
check:rc==0
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl restart xcatd; else service xcatd restart; fi
check:rc==0
cmd:sleep 10
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
check:output=~xcatd service|xcatd.service
check:output=~xcatd service is running|active \(running\)
check:rc==0
end



start:run_command_with_XCATBYPASS
description:with XCATBYPASS=YES, the command works either xcatd deamon running or not
label:mn_only,ci_test,xcatd
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
check:rc==0
check:output=~xcatd service|xcatd.service
check:output=~xcatd service is running|active \(running\)
cmd:XCATBYPASS=YES tabdump site
check:rc==0
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl stop xcatd; else service xcatd stop; fi
check:rc==0
cmd:sleep 3
cmd:ps aux|tee /tmp/run_command_with_XCATBYPASS.log
cmd:awk '{print $11}' /tmp/run_command_with_XCATBYPASS.log|grep -E ^xcatd
check:rc!=0
cmd:rm -rf /tmp/run_command_with_XCATBYPASS.log
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
cmd:tabdump site
check:rc!=0
cmd:XCATBYPASS=YES tabdump site
check:rc==0
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl start xcatd; else service xcatd start; fi
check:rc==0
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
check:rc==0
check:output=~xcatd service|xcatd.service
check:output=~xcatd service is running|active \(running\)
end

start:disable_root_permission_in_policy_table
description:disable root permission, only with XCATBYPASS=YES, the command works
label:mn_only,ci_test,xcatd
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
check:rc==0
check:output=~ xcatd service|xcatd.service
check:output=~xcatd service is running|active \(running\)
cmd:chtab name=root policy.rule=disable
check:rc==0
cmd:lsdef
check:output=~Error: (\[.*?\]: )?Permission denied for request
cmd:XCATBYPASS=YES lsdef
check:rc==0
cmd:XCATBYPASS=YES chtab name=root policy.rule=allow
check:rc==0
cmd:lsdef
check:rc==0
end


start:assign_certain_command_permission
description:assign root permission to certain command, only this command works
label:mn_only,ci_test,xcatd
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
check:rc==0
check:output=~ xcatd service |xcatd.service
check:output=~xcatd service is running|active \(running\)
cmd:lsdef
check:rc==0
cmd:tabdump policy
check:rc==0
cmd:chtab name=root policy.commands=lsdef policy.rule=allow
check:rc==0
cmd:lsdef
check:rc==0
cmd:tabdump policy
check:output=~Error: (\[.*?\]: )?Permission denied for request
cmd:chtab name=root policy.commands= policy.rule=allow
check:rc==0
end

start:reload_xcatd_with_XCATBYPASS
description:with XCATBYPASS=YES, there is no error when restart xcatd deamon. This case is add test case for issue 2727 : run xCAT in "XCATBYPASS" mode and simply reload xCAT to check if there is any errors. 
label:mn_only,ci_test,xcatd
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
check:rc==0
check:output=~xcatd service|xcatd.service
check:output=~xcatd service is running|active \(running\)
cmd:XCATBYPASS=YES lsxcatd -a
check:rc==0
check:output!~Error|ERROR
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then XCATBYPASS=YES systemctl status xcatd; else XCATBYPASS=YES service xcatd status; fi
check:rc==0
check:output!~Error|ERROR
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then XCATBYPASS=YES systemctl restart xcatd; else XCATBYPASS=YES service xcatd restart; fi
check:rc==0
check:output!~Error|ERROR 
end

start:xcatd_install_monitor_respawns
description:a killed install monitor comes back and reclaims xcatiport, without restarting xcatd
label:mn_only,ci_test,xcatd
#the process titles are matched anchored: an unanchored "pgrep -f" also matches this
#test's own command line, and the kill below would then take out the test itself
cmd:mkdir -p /tmp/xcatd_monitor_respawn; pgrep -f "^xcatd: install monitor$" > /tmp/xcatd_monitor_respawn/before.pid; test -s /tmp/xcatd_monitor_respawn/before.pid
check:rc==0
cmd:pgrep -f "^xcatd: SSL listener$" > /tmp/xcatd_monitor_respawn/listener.pid; test -s /tmp/xcatd_monitor_respawn/listener.pid
check:rc==0
cmd:kill -9 $(cat /tmp/xcatd_monitor_respawn/before.pid)
check:rc==0
#it must return on its own. the respawn floor is 5s by default, so poll well past that
cmd:for i in $(seq 1 30); do sleep 2; pgrep -f "^xcatd: install monitor$" > /tmp/xcatd_monitor_respawn/after.pid; test -s /tmp/xcatd_monitor_respawn/after.pid && break; done; test -s /tmp/xcatd_monitor_respawn/after.pid
check:rc==0
cmd:test "$(cat /tmp/xcatd_monitor_respawn/before.pid)" != "$(cat /tmp/xcatd_monitor_respawn/after.pid)"
check:rc==0
#and it must have reclaimed the port, not merely be running again
cmd:XCATIPORT=$(lsdef -t site -i xcatiport 2>/dev/null | grep xcatiport | cut -d= -f2); XCATIPORT=${XCATIPORT:-3002}; ss -lntp | grep ":$XCATIPORT " | grep -q "pid=$(cat /tmp/xcatd_monitor_respawn/after.pid)"
check:rc==0
#xcatd itself must not have been restarted -- surviving without a restart is the point
cmd:pgrep -f "^xcatd: SSL listener$" | diff -q - /tmp/xcatd_monitor_respawn/listener.pid
check:rc==0
cmd:rm -rf /tmp/xcatd_monitor_respawn
check:rc==0
#if the respawn did not happen, this MN is left with a dead xcatiport, and every later case
#in the bundle that provisions a node would fail for that reason instead of its own. a failed
#check does not stop the remaining cmds of a case, so restore the daemon unconditionally here
cmd:pgrep -f "^xcatd: install monitor$" >/dev/null || { if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then XCATBYPASS=YES systemctl restart xcatd; else XCATBYPASS=YES service xcatd restart; fi; sleep 5; }
check:rc==0
end
