mirror of
https://github.com/xcat2/xcat-core.git
synced 2026-09-09 05:00:44 +00:00
Daniel Hilst
616162e424
The new xcatd_install_monitor_respawns case kills the install monitor to prove it comes back. When it does not come back, the case simply ends there, and the MN is left running a daemon whose xcatiport is dead. xcattest does not stop a case at the first failed check -- the only "last" statements are inside the check loops, so every remaining cmd still runs -- but it has no teardown either, and the next case in the bundle that provisions a node would then fail because nothing is listening for install status, not because of anything it did. One real failure would read as a cascade of unrelated ones. Restore the daemon at the end of the case, and only if the monitor is actually missing, so a passing run stays a no-op rather than restarting xcatd for nothing. Verified against a live MN: with the monitor present the step exits 0 and leaves the running pid untouched. Signed-off-by: Daniel Hilst <392820+dhilst@users.noreply.github.com>
221 lines
10 KiB
Plaintext
221 lines
10 KiB
Plaintext
start:xcatd_systemd_has_no_legacy_init_script
|
|
description:systemd installations do not retain the legacy xcatd init script
|
|
label:mn_only,ci_test,xcatd
|
|
cmd:test -d /run/systemd/system && test ! -f /etc/init.d/xcatd
|
|
check:rc==0
|
|
end
|
|
|
|
|
|
start:xcatd_start
|
|
description:stop then start xcatd daemon, and check all the 6 processes are running
|
|
label:mn_only,ci_test,xcatd
|
|
cmd:if [ -d "/tmp/xcatd_start" ]; then mv /tmp/xcatd_start /tmp/xcatd_start.org; fi; mkdir -p /tmp/xcatd_start
|
|
check:rc==0
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
|
|
check:rc==0
|
|
check:output=~xcatd service|xcatd.service
|
|
check:output=~Active: active \(running\)|xcatd service is running
|
|
cmd:ps axjf |grep -v grep |grep "xcatd:" | tee /tmp/xcatd_start/original_xcatd_processes_status
|
|
check:rc==0
|
|
cmd:cat /tmp/xcatd_start/original_xcatd_processes_status |wc -l
|
|
#check:output=~6
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl stop xcatd; else service xcatd stop; fi
|
|
check:rc==0
|
|
cmd:sleep 3
|
|
cmd:ps axjf |grep -v grep |grep "xcatd:" | tee /tmp/xcatd_start/after_stop_xcatd_processes_status
|
|
check:rc==0
|
|
cmd:cat /tmp/xcatd_start/after_stop_xcatd_processes_status|wc -l
|
|
check:output=~0
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl start xcatd; else service xcatd start; fi
|
|
check:rc==0
|
|
cmd:sleep 3
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
|
|
check:rc==0
|
|
check:output=~xcatd service|xcatd.service
|
|
check:output=~Active: active \(running\)|xcatd service is running
|
|
cmd:ps axjf |grep -v grep |grep "xcatd:" | tee /tmp/xcatd_start/after_start_xcatd_processes_status
|
|
check:rc==0
|
|
cmd:cat /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
|
|
check:output=~6
|
|
cmd:grep "xcatd: SSL listener" /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
|
|
check:output=~1
|
|
cmd:grep "xcatd: DB Access" /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
|
|
check:output=~1
|
|
cmd:grep "xcatd: UDP listener" /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
|
|
check:output=~1
|
|
cmd:grep "xcatd: Discovery worker" /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
|
|
check:output=~1
|
|
cmd:grep "xcatd: install monitor" /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
|
|
check:output=~1
|
|
cmd:grep "xcatd: Command log writer" /tmp/xcatd_start/after_start_xcatd_processes_status|wc -l
|
|
check:output=~1
|
|
cmd: rm -rf /tmp/xcatd_start; if [ -d "/tmp/xcatd_start.org" ]; then mv /tmp/xcatd_start.org /tmp/xcatd_start; fi
|
|
check:rc==0
|
|
end
|
|
|
|
|
|
start:xcatd_stop
|
|
description:stop xcatd daemon, then check all the 6 processes are stopped
|
|
label:mn_only,ci_test,xcatd
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
|
|
check:rc==0
|
|
check:output=~xcatd service|xcatd.service
|
|
check:output=~xcatd service is running|active \(running\)
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl stop xcatd; else service xcatd stop; fi
|
|
check:rc==0
|
|
cmd:sleep 3
|
|
cmd:ps axjf |grep -v grep |grep "xcatd:" | wc -l
|
|
check:rc==0
|
|
check:output=~0
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
|
|
check:output=~xcatd service|xcatd.service
|
|
check:output=~xcatd service is not running|inactive \(dead\)
|
|
cmd:ps -ef | grep xcatd
|
|
check:output!~"xcatd: SSL listener"
|
|
check:output!~"xcatd: DB Access"
|
|
check:output!~"xcatd: UDP listener"
|
|
check:output!~"xcatd: install monitor"
|
|
check:output!~"xcatd: Discovery worke"
|
|
check:output!~"xcatd: Command log writer"
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl start xcatd; else service xcatd start; fi
|
|
check:rc==0
|
|
end
|
|
|
|
|
|
start:xcatd_restart
|
|
description:restart xcatd daemon
|
|
label:mn_only,xcatd
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
|
|
check:output=~xcatd service|xcatd.service
|
|
check:output=~xcatd service is running|active \(running\)
|
|
check:rc==0
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl restart xcatd; else service xcatd restart; fi
|
|
check:rc==0
|
|
cmd:sleep 10
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
|
|
check:output=~xcatd service|xcatd.service
|
|
check:output=~xcatd service is running|active \(running\)
|
|
check:rc==0
|
|
end
|
|
|
|
|
|
|
|
start:run_command_with_XCATBYPASS
|
|
description:with XCATBYPASS=YES, the command works either xcatd deamon running or not
|
|
label:mn_only,ci_test,xcatd
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
|
|
check:rc==0
|
|
check:output=~xcatd service|xcatd.service
|
|
check:output=~xcatd service is running|active \(running\)
|
|
cmd:XCATBYPASS=YES tabdump site
|
|
check:rc==0
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl stop xcatd; else service xcatd stop; fi
|
|
check:rc==0
|
|
cmd:sleep 3
|
|
cmd:ps aux|tee /tmp/run_command_with_XCATBYPASS.log
|
|
cmd:awk '{print $11}' /tmp/run_command_with_XCATBYPASS.log|grep -E ^xcatd
|
|
check:rc!=0
|
|
cmd:rm -rf /tmp/run_command_with_XCATBYPASS.log
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
|
|
cmd:tabdump site
|
|
check:rc!=0
|
|
cmd:XCATBYPASS=YES tabdump site
|
|
check:rc==0
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl start xcatd; else service xcatd start; fi
|
|
check:rc==0
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
|
|
check:rc==0
|
|
check:output=~xcatd service|xcatd.service
|
|
check:output=~xcatd service is running|active \(running\)
|
|
end
|
|
|
|
start:disable_root_permission_in_policy_table
|
|
description:disable root permission, only with XCATBYPASS=YES, the command works
|
|
label:mn_only,ci_test,xcatd
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
|
|
check:rc==0
|
|
check:output=~ xcatd service|xcatd.service
|
|
check:output=~xcatd service is running|active \(running\)
|
|
cmd:chtab name=root policy.rule=disable
|
|
check:rc==0
|
|
cmd:lsdef
|
|
check:output=~Error: (\[.*?\]: )?Permission denied for request
|
|
cmd:XCATBYPASS=YES lsdef
|
|
check:rc==0
|
|
cmd:XCATBYPASS=YES chtab name=root policy.rule=allow
|
|
check:rc==0
|
|
cmd:lsdef
|
|
check:rc==0
|
|
end
|
|
|
|
|
|
start:assign_certain_command_permission
|
|
description:assign root permission to certain command, only this command works
|
|
label:mn_only,ci_test,xcatd
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
|
|
check:rc==0
|
|
check:output=~ xcatd service |xcatd.service
|
|
check:output=~xcatd service is running|active \(running\)
|
|
cmd:lsdef
|
|
check:rc==0
|
|
cmd:tabdump policy
|
|
check:rc==0
|
|
cmd:chtab name=root policy.commands=lsdef policy.rule=allow
|
|
check:rc==0
|
|
cmd:lsdef
|
|
check:rc==0
|
|
cmd:tabdump policy
|
|
check:output=~Error: (\[.*?\]: )?Permission denied for request
|
|
cmd:chtab name=root policy.commands= policy.rule=allow
|
|
check:rc==0
|
|
end
|
|
|
|
start:reload_xcatd_with_XCATBYPASS
|
|
description:with XCATBYPASS=YES, there is no error when restart xcatd deamon. This case is add test case for issue 2727 : run xCAT in "XCATBYPASS" mode and simply reload xCAT to check if there is any errors.
|
|
label:mn_only,ci_test,xcatd
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then systemctl status xcatd; else service xcatd status; fi
|
|
check:rc==0
|
|
check:output=~xcatd service|xcatd.service
|
|
check:output=~xcatd service is running|active \(running\)
|
|
cmd:XCATBYPASS=YES lsxcatd -a
|
|
check:rc==0
|
|
check:output!~Error|ERROR
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then XCATBYPASS=YES systemctl status xcatd; else XCATBYPASS=YES service xcatd status; fi
|
|
check:rc==0
|
|
check:output!~Error|ERROR
|
|
cmd:if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then XCATBYPASS=YES systemctl restart xcatd; else XCATBYPASS=YES service xcatd restart; fi
|
|
check:rc==0
|
|
check:output!~Error|ERROR
|
|
end
|
|
|
|
start:xcatd_install_monitor_respawns
|
|
description:a killed install monitor comes back and reclaims xcatiport, without restarting xcatd
|
|
label:mn_only,ci_test,xcatd
|
|
#the process titles are matched anchored: an unanchored "pgrep -f" also matches this
|
|
#test's own command line, and the kill below would then take out the test itself
|
|
cmd:mkdir -p /tmp/xcatd_monitor_respawn; pgrep -f "^xcatd: install monitor$" > /tmp/xcatd_monitor_respawn/before.pid; test -s /tmp/xcatd_monitor_respawn/before.pid
|
|
check:rc==0
|
|
cmd:pgrep -f "^xcatd: SSL listener$" > /tmp/xcatd_monitor_respawn/listener.pid; test -s /tmp/xcatd_monitor_respawn/listener.pid
|
|
check:rc==0
|
|
cmd:kill -9 $(cat /tmp/xcatd_monitor_respawn/before.pid)
|
|
check:rc==0
|
|
#it must return on its own. the respawn floor is 5s by default, so poll well past that
|
|
cmd:for i in $(seq 1 30); do sleep 2; pgrep -f "^xcatd: install monitor$" > /tmp/xcatd_monitor_respawn/after.pid; test -s /tmp/xcatd_monitor_respawn/after.pid && break; done; test -s /tmp/xcatd_monitor_respawn/after.pid
|
|
check:rc==0
|
|
cmd:test "$(cat /tmp/xcatd_monitor_respawn/before.pid)" != "$(cat /tmp/xcatd_monitor_respawn/after.pid)"
|
|
check:rc==0
|
|
#and it must have reclaimed the port, not merely be running again
|
|
cmd:XCATIPORT=$(lsdef -t site -i xcatiport 2>/dev/null | grep xcatiport | cut -d= -f2); XCATIPORT=${XCATIPORT:-3002}; ss -lntp | grep ":$XCATIPORT " | grep -q "pid=$(cat /tmp/xcatd_monitor_respawn/after.pid)"
|
|
check:rc==0
|
|
#xcatd itself must not have been restarted -- surviving without a restart is the point
|
|
cmd:pgrep -f "^xcatd: SSL listener$" | diff -q - /tmp/xcatd_monitor_respawn/listener.pid
|
|
check:rc==0
|
|
cmd:rm -rf /tmp/xcatd_monitor_respawn
|
|
check:rc==0
|
|
#if the respawn did not happen, this MN is left with a dead xcatiport, and every later case
|
|
#in the bundle that provisions a node would fail for that reason instead of its own. a failed
|
|
#check does not stop the remaining cmds of a case, so restore the daemon unconditionally here
|
|
cmd:pgrep -f "^xcatd: install monitor$" >/dev/null || { if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then XCATBYPASS=YES systemctl restart xcatd; else XCATBYPASS=YES service xcatd restart; fi; sleep 5; }
|
|
check:rc==0
|
|
end
|