From e9c7690af470fd8452ddec88e2845336bdfbef67 Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Thu, 7 Mar 2019 16:03:42 +0000
Subject: [PATCH] Enable masakari instance monitor tests

Add helpers and a test to test a guest dying and being resurrected
by masakari instance monitor.
---
 .../.test_zaza_utilities_openstack.py.swp     | Bin 0 -> 61440 bytes
 .../test_zaza_utilities_openstack.py          |  49 +++++++++
 zaza/charm_tests/masakari/tests.py            |  95 ++++++++++++++++--
 zaza/configure/masakari.py                    |  17 ++++
 zaza/utilities/.openstack.py.swp              | Bin 0 -> 16384 bytes
 zaza/utilities/exceptions.py                  |   6 ++
 zaza/utilities/openstack.py                   |  32 ++++++
 7 files changed, 192 insertions(+), 7 deletions(-)
 create mode 100644 unit_tests/utilities/.test_zaza_utilities_openstack.py.swp
 create mode 100644 zaza/utilities/.openstack.py.swp

diff --git a/unit_tests/utilities/.test_zaza_utilities_openstack.py.swp b/unit_tests/utilities/.test_zaza_utilities_openstack.py.swp
new file mode 100644
index 0000000000000000000000000000000000000000..752be8519224f1d649a3b5a336e28f65409a5e48
GIT binary patch
literal 61440
zcmeI5378yLb*LM&IK)c`_6v(0C`OnbCF+)Dv|3n+$MV<~Aj!;(WXnP{HQhBcW%cH+
z?ir1Ra7akl{2q@j<^jfFf>|5_BtS?oyB(AFg|G)6keJoHfX(&+c_A3ye{R*STU#$P
zl8w#Rt?yS)SJmClJ@?i<=ia(CyJ7#X-PX3^h8&+q<Z?^rKld4bcF4-TWGt8K%y#PS
zj-2TInXh*m%Y|9nYrBp5c=v0c<43rDsnI&Tezs-T%L|UT{%ZSbdwr+wwo7ftYkTWE
zZMW*SUCzkwlAJDu$4ZT+Q}^0-`Eaqh+&!h?nV|%R5{OBl)1E7AJ1w_yd`va<H6u^3
z&Uoa$*d@asLkSEeFqFVh0z(N5B`}o0Py#~<3?=aEErE9PwA?im@uZ-bcJTe6#P`X;
z@98er{YG&7vf%sO!MBd*;lb}c!S^Y_34Q)IgX5P6-v<)sPYI5fg70@F&Yv0_-yeKG
zA}ElK|Fq!vf#7>CasKq+_`%@&Ly7Yb4URuO_<rnz{Xca44-1a#Z#QxNA;I}#@O?$%
zeDE`;<<Ll+4}uN9h7uS`U?_p11cnkAN?<5~p#+8!7)oF$fuRJ35?GZ4>`E>tw&`Dp
zA<O)~9mMk0-^%6w4PFZeU_BJz=NQUAgLgv{&Vf5Hu0IGLfOo+g;5v9Q9EGc4KRgcp
zABOsWz)#^exE0<3*T5C96;6k{G5BwVzk(OQK^TP}V<3MKZUP53!nbiQyc1pu2jH=A
z7Y>OJz#qZ0;4FA7JP1CIGvf{5KoP!;(f($*39f|$a5<b0qwqWMIUE2tz>DEUP=!5k
z9{dae-U;7=zl9~(3R~bi*zvD}>){Zb2mgg{;mhzPcm?c%-EazgjCyzzsQS<~YDGSt
zQ=Kc$&NoY5)3FabEib1}sxx(`({45D<*Mt{+r_HO3W~_ZrAEC}Y1#8~uI02lt$JzE
zu67)2hxLrU@u*;4GMq})Kb7khMY-kJZKqUs+Oqs3GnU|7uUJ%pg6p!gqEDL4h<yB9
zRw^yW^GaTsRU~VBB@<-Ujs{3(c{Ge5I{~j}3aTKn_*2ChHATp%+8~Q_)rQ@6>+^0i
znLicAoDOrF5tqu0xipoNL{#1&8EZCLDIHVs$cet0k&Cjkq0aOOk1E$1EDzdk&%5Q_
z{587PWLIiRKF1Cn%c(yS|K;6Ee!G?5u+`qM_1tZhLV5h$a$%Ew?s#F_TzR%o9&^f@
zx176i)7J5G^XpPE9E}|riL^U!WV>}RJgdYIJxiXCt{{`MZryGzOGbm%ULdkU(5^G`
zC4~%1wsmf!Jy+R0TbP^MQYmbz%xx^pIvbqA#>%-{Hjg=5$F^<Vm=QywWPWo`m&re#
zF3pH;loBM+Kh+H!)MrKxMQS8^g_n*^iWC2gG+T{Dx8k(62MLwh%T0&GM&_z^d*oPl
zWt%um_~yd0=a%iN?x!RUIYzu=Bk3A9QOI8-twyI!CY&SuU$?6So@sTQzQNi~-LAJw
zZiN7T6KOTBbjot1Z`itBlY**Xq$~N3hY2OT-8wp=lF&yaL33?X=xLdc#Jeg|^y@Kp
z_+b31be+bo_G>mS!Q`)z9-`@At8TNj$PJQ3iR5>aDQYqm|M!TuV7DBpgMlh7D)?Br
zN^&NiSVAtQCgK6p^$|O-YDMQ=Rafk^N|$TmiK(L_xmd7BPcL!ONc25K^;0TpbL?Q`
ze5^`!oRRMl>79K=4*%?3PhsS`#YLI{x?JjXl8qn?;Wnu&t<xsI8ES1#re&z^CrS+J
zuTZn)*FjZ??uCxFq@WGzJa;SgNK|i>v@+C1jz`L-E=wzi9L^|D%UuixTAZY33x6iG
zAm}aRqTtWO1$3%GFXTt$|B?zxT&T+|m+8L~mxLu%QHhc$IT*d{G@aJy3=4kuvcPv*
z7WnRC9k91f^=xa4p3~mf<W4TSR&nMyD!)Zj*KW_{%2nI*taRwn4hkgE6fdwnN6zo$
zTSDaKT8)||Ym1BF+G5MCDUmuy%1%?3GJCzQ3Kt?<wr7Pu{N!>$OlDQHO2v~)b(V7a
zBI)AFZhKk(meGs;+imE-D%5G)tvUKPLs-^7ud%9b&22LRB)3+l&MnYtFF4lI+%n6g
zogx=ZJ5KbFHP`Sj!F(vYb8gwH+V%MkcW!IGL2p@SENM{}HP@4llPuRO#B$lLnM?NW
zo3SpNn3|f{GqY=Q+S)s1?cBTP!d)}F_U_@=Mb^Zg1J+Y_?YVHB<+x;`WjRNhGTydG
z5rJTlR3t+cY?78c%bl{4&avfsu?re?GGL8NOk2CAN307brgu%RBaVIbszls!uIjk-
z+1Bi`Wj9HoY|mC5O0*<t+Vd@kbM1!Z)~%%$Ys%~M=u=~^y+lW!TSGkZS!lPL+t;sO
zT3RaFO`9d@#YSs>J#mQyz4cGqwR3XM^kiY2CGryFK%-;T>}9Jl+qQ|<wxlMk##~Iu
zcAG1ggDdNew$<^Z7AV}DO9p)nlWdh6A{bV}JJu3Hi$)@{3hFT`s0o!veP*4t-@n6I
z<j)$Fx*rJwjjVmL%J5F|*K+3<+Sd4nv2E5)yWXhF5XX92oB9y@|L54{=VQl<{f{}V
zKDS`&zaRbwEWibD57&GQ-U^36!u%l&KSK!&B`}o0Py#~<3?(p>z)%812@EALl)z8|
zzy1<XCUo#0(-BJxINn=}uGeVg&HR^~Pq;{m^Emg+#U<PlrTGpXn$db=(e|e=*ID_+
zn)r$GYmGnEcYJ!D(`rv%)v>E)8qNg7UDOiKZ*k78SJLxha!z?!bms9U+3j`<f08~$
z(tD>1lb6pFrl<B7CT3=)wl6ficAn$i3TXo4<4R3I>7-01yIm#jE%oGkR|#piS2|8r
za6A`Y5vLNakRpCD1XWtFy%HvW)f>ep!_0zH@qeyW_Wu-f;7?*+T<rg1pt2vp-hU3v
z!c*XJ@FQ&f+u<MJk6<gD3U}f^_%K`x75F{)HgouIfoq`+m%!QZL+0>*03U<v;RsBD
z%<bQW^Wk>55stul@F@5p^ZB>Ki$Ui1FM!{NGvN#5`xdwgqzoUTEZ4#CGnBwk0z(N5
zB`}o015g6`)-K;_EFpLA(fpO2D?5eR*)5w{mLii^g+Z^gD_hH3wv1P{&W%+HW1H1D
zhH+Lw<{y}e$*(<>>lIb6bJ6S^XENQRVo?esuUL-FkT5=`VO(XF+9XGv{;;ToFiR3!
zNd|2)UW$x5<HJ^Ov0Rc0*cca(K0I!6Au`g3$IM|f;4_0WS#OmbmYO{4NH!K_$VqY1
z)t;1dPFIi#>n{?L&@r)PT*-Dy?T)O*$TKx3qsB@;eHEv>j%jtG<AY{+Q#yh1;-DEG
zFPaSd)tQms@j>!CHfVk~rt>>CaDK-k`Ry*?v0fQAI@Il|&}itY;7OxFYm9v7&pA*y
zr&c(pGIP!)+t1m({haA5@@wPmKwq9`67k%^hK+>{TW7{LZr`+F`{r#|WULs|^VP-h
z4`zLtRj)b5U<>1{EicwG;xNTF%}K%}8}*XS=3!z$`X*SZ)09L@%v_i3a@$>uB&jA@
zBPQgO?J~)WRivl<i+ku6E*3*{3hY+<BAXf83Tp;*ZcVOMZf0upGNKZD_RdV~oY}R1
zGNNOl8Cvv<QlZ+Df*(_Y4|8>*tWRA^y()p6Hku`SjumYsY-Fd_Y>$o^)mU*=NVnHo
zR<Tkjf6;NMynQROG)HaFbGLQH-b-5T3pcKHBAHN`!#^JAS}pVcpJjlwk+tVy|Lf=5
zKZL#ie5k=%_-*(Ew*G72)$lx+gGpEe4}x!C_rD!p4)gFB_%t^E3*h;103Hppw*Pi`
z13Uw^!admeABC5|5|m*QE`*cdBiQ<X4o!FpJOu8*-hUIw`u??W8vF#?|FiHJcm+(u
zD4YaeVIBW<(16YG1J>^U9b6B83T2pub?{x*?td6w4Q)6d&VsM8e*Y#o4Abz3@F@5`
z^7}a44$lUWr>y7yBy0KK22HpQbo)BNy-2zX8Qi0#v!uWKwdp~xOLfC1rtOz%4(__3
zlMk1G=}hVG0dHx^UN!4>7Pj^9kL*UD&$x}5YBrroKl!HQX6~44tX;BRh~;7|B@@pc
zOSD)sWVeGQMe>tHMzZ3lC3iL!_-KKcoScx`R$Af0d)ov3o>g<_TWUq!@#$-K)nt8H
z!MMH?esZa=EKQQ*rVfoZ$?<*~6_&_o(h3GbyR|%iB}1XX`>}i7GcTf+&j}hEg|)IG
zJqtq(JQm6i^^NtyVfSRL$8;aI0%hweRFhOUV@-6(H054b-T$<$BtD5!d4Wa4PG%3+
zTiddFHX{nttF59(yFcF>SD;t-_Vl8tlzi?VomfhR4tv3)tHIL;o_dB=&tr-gl(|cu
zZ*@v?2OF(4YOKQE;eYYuhp|k<i2QSX;*nWq9gF;J;_;<rSvtOTmVPcmu3KjXNv9$(
z*F|$Bu_bj|=6Q9TZpGZ9BW{YsOt;QUF;=$31HA|Rlg4l9|1P@XO$c)BR&u(0{*hjt
zQ5c4cM*JUz_|qW46(S3{rnc-KFwcU78RxkGN-ZA6;Gqu529FuY8hAgDF_pqV{)u=^
ziZ8&|ibzCZh!rW(o<==U@1|Db`p~Bab@E09luDW2ZGf7M=R8@p!C6zc!GU7Qt<N=(
z{Zaj#kuHL-to{Rl`j}fersn@YitYYW5c?mqSbaW^z5fn)A?$+<AlJMH8glLMGnBwk
z0z(N5B`}o0Py#~<3?(p>z)%812@EALlt6b0Xm5k@`K@L)>iA;`VmQ8R_)if;)^emB
z%Awc!{yHfp^Zz5%*Nd2&7W-fC?=SoRy$R|dYXD|o6z;_Km%aa%;RqarN5hYq>;E|X
z8O*?M!o%U~%=f<;n$Up#@VoE@=KNm{viAS!unE4!JpX&(*)R^@V~+nVAnX2L1U5Vg
zzRx`W_u%tz9qfi2+`=6H`{8x)OgImIfkWdT;l(fkKW1+KX1Eq)FTWBz5$<Mg|1S6}
zyb7KL=fJm_+rJb35nciwOu(bzUzy*36I=;-I2}%dJCWt<;RX=7z7YC<v{J9J&fCHE
z4ztYevcrcys$51*yIo!=HD+a7meGV0YMnJAj>K}O)na$QQp3}GA9*9b^D10rBbOa^
zmRZ<Scl^EZQeId|m7QE<r@m6nn>XGsm5ulwgNcp(f+tQ=p4xax{HuJoW{{A-)k`t>
z71<fgq%PS?=O74Zyjtt5i|i`<yo5;xapD(C2(1`oUhmkYHz6`k*oYJ&St>Rg%~5ft
zrP2^_k&bK%Qmi;mQ~prm)de#lq%?X5A+_<+p<`i=)1~zHatyMQ+8WPn(6+|9u;KVi
z0G1tD7r<K6gc=P}f0=$&m8%C#)vd86s%$!0S@wmH+_g}e0pe6UOvwyvbTiCA{Qk*!
zr3}ypb*#yT*G<_ZHQAI1eF#7Ct-S_pgRwwe=S*@skyLfY!UxSNb}Y5STU?t1u|;+u
z&1jo`s?nV_)m~60rdW#I+idJI(_mwVr8TDVoY8nwfwG%$>@t&Kk!O6@wLzP6EW&%C
zQODxf3NcvIjs<;y>caiC2Z+fw^X97>iP^T!=vumo>3f*DT37T6?!hK-P}Sx@eE&a_
z6Vdu~Zo&p1Zac>;82i2LP?ySIdQ&|UlS!+4YuDG5+@@*k2`R;Mn$gVD+4b+LI&!y}
zZiA8S29vFNwYOT6oUW<CnxqMLVk$}R7kcki(EYUmYDtPViBd%Gz8Ojq+_*)1b&aLk
zmI@|U7`;}9qQfOQts6*YOX%IE2%>J4ItsJ3qe)Ye=-!ItY@+$dA>AZb&}1Z7Mq^Qj
zWe?5BSa!9|MB>uo9;|^#2rZmQv5a)?TlwB!wO_=f)n4p>`5X}&75hJYe;>B~VYm!V
z1z98T5m<za;3Rl2_WcnUhwow2-wp49=fNaA5@fCa=iuFN9qfV4@FVQ|PlDL{Z-iIE
zF?c3C6^d{Nw*JTAWpF8+1NUI(e;RHC+1tMb71#wo!`}Y}yd4h1PIwgjU+n((!Zk1r
zkAeTd=Kpti8@v!+05+Tp+u(HgGWP$M;3lX+6*j~7vHiaXAA}de3t%6d0gr?`@BzFQ
z4#H#M7s&P-@ODsgM%1E-@=<++m?E<IX1Utv@XS)Dy^u6W)WMtz5e*bK<5PjuM)zST
z-`1yd7`b$6?^7ms&X{#MDtb9{(Zo|HOB4HME-~l)j;+7vAzpxpWjtsE@xmNe1Z)n<
zQuw6(6&JHf&c9l>h)`ZC-=%OXj(GV-7spF>)7_!=inctjS}NnPQ9u3eCgyI5hb`&L
zON-+xOjU<mVL~fRC04d})ho3e_EATo111{{vhp%UL$4yi)t9w@#jUu$`bL?`t#0Gt
zxBqmrxrZsWjE*XjKCQT#OK&n7**{0(HGU?YN`B(fU8Vh#Q`2nb%Zmjznzd@-6?>;k
zLBCPDaPRJkU3*G<CU#HCTL|n*&E>r!YAI&MW1Q@y6NZch>2mG9>B*^8B(d^LO-)`*
zal>LvIrFIa-n!EzUzbiyPhYlo>Ov`UDDy7YN*Vf<RZBMbF7gCx&2EpjTXx;^M>4^9
zrp=?5O*c`85=ta1opov7)YBwaM~|_Q@!0s*;s*YWZ9jT!{n2CQu~G@)D-%7vaTv;B
zrBk9e8SdlcUFGPA?l~Ffja=1n*e7`;u2DQ6B{?RU{9@$UX$PmQd5%`njR%t>2}Mji
zU$3s9(7)oE6S>b!?qMVBP<d1v$WR8!apd#BtYq0@{IK1llG%LP=$Sib`8SYwU59!t
zY9X6$n-C%((RI#j-mqaqP75rpFmmQlcwS#8-?Dl)-D}C2ybNk~PR(36F*A`-L!s^m
zJBZ7Du2gQ+=iK>HL+t}!^mw;O$(fsD+J!8dof;%|N~>?m@$$;aFm8PVWr=#SJ>|LI
z6=d)ryRq}!y0?8SE*VmY-V)=fq@?P_0|#Q=xOwB|tvSh(d=fFrX6flNsJvJ4pY!B5
zT|+9{rVGreNck@;Zg){89PLtT>EJ!2l^sC$NZ9kKe1`pDJ~8U<<;gF~PAmRq{%kyA
zq!4*nPZhRT9DQfxL~H}Zl9!G-Qo@zxL51#GA1jHgSZh?AD&mm*M5-tfhm<Ju6O1VR
zc{yK$Wq{mubS+|87gPHkDlFE$9%}o)6Ls2eToj5?nMRC@j4_lLVXf+JnCd9kSbq@C
z<!ZsGDqlrL{Q*wWE?QMStN!#dJ-XQckH#LY2KN6R{+7=#vHAZ2mO$42%lrTS3!DFa
zAU=RA;E5p5{{K7n|7+mY&<5H6?`zop*Fg#7oqnglXR!Nkf>~G(MK}rGhppd$bKq`l
z{J)3)4pZ<X_!_qTE8yi&g-LiK+=FfZW%z5DgQvm6;JeuOpN1F0G8};^cp`igyZ((Z
z54+(ExDA{BR(Kse59C?@li;7R<v#&$gJ*+&ci%g(>F3}v@M+{G^ZYds86JkNAFToM
zt!{w~*UE0I)$rDri*9XRHaS|~t~J+}$JEb+N_JYzXY%e0XEEH^G3AEL=`@s<xpkg)
zWUnLXtDTm1rlkTAF&YtErJl+$DTm#B@FtZ?W2w&jf+{7q#!Ev+S5b^!eP+?m3Y|Cu
zOAlw>NTkq7)n8deaaknel4cYWL-IZnJE>$87j>`91b2d16p;+Xj;p8{Wn}KQ6gm|W
zwIG!p^<gU|$j>s8amxk&fz*O>E}>)v+isN?wruh>%v7|5G1V!asDi3uuh^OG)Z3jR
zdxfp{qg)@D3G3ryjijo`?yDs~u+|noyy$t5i~d`?hGkAimsM=7kXk}Tzk&J1mqi#D
zNI|13jE!qy6xf;RaH7qLKr)Nu$Eb@ma6u%#bX6B6iP5#Fo1GdO&gRF)jfho?et_wz
zs<W89vg+*Mp|~0*r7nf`b+I(NtfoLl=Xq@r4}pdo1<zqMAMUSAQU~%msG?JCZS43H
zpKZ(zmUJxFs2e?K@Hk^)TpZ2cI^Ledg(|b-#`uZF)vd#jvPMD|gGV#z8md*JS&=IS
zi)hS5MA37SihN8wUAHp}TXRO8CRIZz{H3kNk>yL+%d6Vs26ogpC~<)VZ*%f^F;rza
zf6VBWifp#yR(X85_uVj>q7u4^Bug=4O*--rYC89cI!>jNQ$pmw)R|tjqS8T{PJJ<X
z&q_!mj9&%mb;Bw)c=MX9CnVCyjjDe*Ul@_wcU0oX7t<v(GnWQ>aC{L(r5nT))5=(d
z%9Ghg{1trB8?2H<Bu8C`Rz`PJkf=DihKa;`y!sc3DybZiQ!15EJCY|>seEZmeW-Et
zVSP0&H6x%DN4iU6y!ta%T@^@8=dH7tQFP~)cSuDTHO%b7<i&yh<YO`oI<!zR!i=aF
z4Gub4c_QW(6A7-Q{Rd<3WVte{p@$fx?9mQ@UV4R}%-RS=n#^Cnx>f#*9sZQAaeHP}
zu9<sGrXh7J_Wz?WSYIx7EA#)C2gd8?u=j6<5}XNVz}K<$-w1yU2jCR=GPeFr@O(H7
zSHc!J4L*v^|58|lXTn7w>jF-NPhj`|CHx7<{(mRKXR!bO2Cjpna5e0Mb?|lk0I!4V
z!Gle327CuU!0Vw2qmYLO!<X;_yap}@Sp)D@Z2#Nf?eI!?1$ZF)0G<w?!VmCjxEf>+
zzzbjkeg{5<FW{}P2;1RzLHq*$EAR_EgYQXr7<>nxz-{n)Sb}YEHk=IqiH!diMAk18
zd1rr;dKC=xltHLQSe~hHxmM{A8)`H<Ek?xH5xm2am)x`GOvmf0M!9GzrgzXs=aN%~
zaw30jWo&GGvptvaqNg*+zNO+L7%X*(8h^XIP7_zNKMS@aKg&BhGdzpDTEu1W%KK7x
z<eQyY_QFeKMq(3VE{y1^a${*c<02UflMLh6xHvOP<FEf3yfiA-v`~HDj$8AcJMmN{
z+mV#xCzH+Em#%LTmD-JxAJ8o9Qk%OeeNngE(R}<xx60&1Kgx&IUs&U{b(UX7-^;pI
zUQ|10CK_sb2q1B);d5l18_Bh{Xaad9#=FKk$VqACV-D#QcH8rAdAIBf<5qSUg)_p8
zB$p0jE>n%7*MNijLYFp^yft)@cGew%<>os^{2cmai`=QCy!qKpTgu~=ZER<rli&Ii
zDO4asb6%AM$2)^@8n@rd`>SG7Wr7Hlz_sx#wTUMt%#)E`BtOS1<V4Ps!j0uCOQd;S
z=0s<ZcQ%Yw=C*9M6Qb0Yt1)dtis8AIQ{%QcK>rkmP_4<&Y**o%EE!35sRteJsQ$yx
zu9X$V3RPK^ETVU@6uQ-BXaFUuD>WDujZKy*Od$~KAcg9(|Kx<;)pQcCc#8p-ZXsbB
zsv<Hs&{t>T{YR)Hk@p+KV(KOjoou*En%bJW&PY1a^YxHJ>>eym`{km<KyryKufyB~
zvQxV{rlj5@B~{&nNIXnwMG~B~w~cSuG`k`ChKN{9-6R-lRCu-4j;vNL@}{Npu$JpB
zBwD&U7uKX{(z=eq1f*c`(>>%Ct)ax7Q#Ck$f-cw}en7RvNK0DT38nN$WjpfrA{&{@
zyA~p4jg_ODSaqTAOT~<esx<4cIO(vHt*5H0x>Td-+TwK-E4Dec;;M_qBd?bmuqraL
z8;h!2O=Kx}mZVaRSy!PEh`C+pbFmy4p~&eK3+DK`+Q9|csXuVFTHY&=s;EIF9j~m+
zdwA@U6)5MbCE8Crbz#N+e>`@m*rj6smjavgo7nq*4=;uP31v7Beu>R5&jK8SCxiF^
zzJ;y-Pw-awbEv^K_;2u^*!s7?8$sRyAnyY7U?<4?0Pn!=zYz|>S@08V{jb2^!A<Z|
zSb&Z22>1y$|NGzucph8@lW-wC0e+6{e>>a&F93NyK;8`?zJZ_P2lxWK7XBFa!;`^+
zQ$f}Yd<tF*OK={X3HRUw_%{3#ya>cEa6UW=?!g!E5s<Y5*TWKA0lx(w#~<(_=)k$K
z4bFrw;0t&g9DyP{0zQxY-v?3;@0Ggf^U)oJ{$^v{CeH)m;)&c*^vR?mi!)KKmbau3
zVwAw*8;Z_hNzllBFtGvYaTu7gU_LUg%P=jaLoTt`zP+Pop>*LaD{5y(7L12Q=w${n
z(^2I70_LFW&GD+25x=($+fPzf$<YftAu}~qTxz-P?yEBLh*w-3?970ab2{sOSh>MO
zMwPYh_Jur);H`HnrIu%xf;z*1TZ%*}Q75Zg<t?+q37nm?<EY^18ZXIBMh*KC2t}OE
zUsA{tPd=;kMwW6(vfzojD@pxkDSNh}hxyBlCYeRqN=6xRMI_p^UEV(1$8F?0pmfvA
zNde@O5m7(8lN84iI?!^ORlDr?%^^O3(5*dwG)&eUPKwhn4^OWws9qzdvYVY_nfGVm
zQE~`&P}FTKZzi$BRQryy;)$2kz7}qM9+!wJsJ@?tnTph@(n$4&x9ke?*F(isDQ}b0
zgiNKpT6h_&@x!th2}+1Tr5MR@qPGnrHmcQCzAD=%P`)X-W@Kh{dLENzPJZc6n1m7J
zph66T_7~Oh3n?WVRfY5x@|vuQ8?Lv@e0Ws~o8c!8dbwaZU`~rfz9UXKBN&xrwuiFo
zeIyoZEWtz@2o_VsmnepzRnmHymG6x1pQH+;8gJ_9Vbx5-*eP((<tJKUhRa^BbLf&$
zi7HcgU9`$~lIA3sd{mZujv=JJ<=*1YH3qdGal!SCfq>a{xWY){wXu@Gk#Ue;JCVQ<
z*$vuF`;Pr4PLvW2f#)U~t@I)a5pGr!SaN4)tIK*xY)&rIpGa0tWE+U3cz<gIYIRvo
zOF+Kw1AUM5V$tC>^*`bZYXuJo7l-X597_hn7{7gVJclPB)V(5gFVXcaQ%d<=vH;NX
zorjh<k`sNRTpYdyH}o;8cV8x^N90N~t+r0h|6_LNt`VCP`+t96cYYFE|7B2t$HU`5
z*8hv`e?2@4WFLTs!rj>Ucfsv&BRH@Leu{m6EBpz_zJDjf2TAN2kU9S?@NoDhcKw@S
z5w^i8@V~L;Wv+iWJOpmTe!mrDzrPyDGXQ78kFeXn2sgs>K%V=bf~Uh-@I&l(+5hiV
zun5n9C&6#P_p#safPaHq;0R2@nILoipM<x72UBo5d<FacU2rulgUtPJgdbtU-vTn{
zKLIl5e-?ZKx!w$~0+I1^Ao7vw(4YL6JhsV$n;Q~!mbl|3?^-e*7u(2qv72_wuDORB
z{&3eh;*@!ytyaTQO%BLM^Ip5+)|mq4yU}66y&P~_Efg%LGv=S7>AQ{MG|wN@=XdQL
z<?SAewa86EUu?dk{l<>G*QhzAW`ha)=&-u)oe|^QFAF)F=(HOXmCB`lB&Pp2N$S61
zN$VJtG)(8dqNJy%FWFgjG2bIq6N|U+jWMhxNjJHYC|RXV-?G8eUiYwAjbfPWck${@
z>NnX&d@QP@EDB-nUQC_D<BVa5h!3K_Y`o4aD<gSxHVZ}bChkPPw4zyxSfNExyH1+L
zuIxS&aWF3z%EJZ)R*6{oG4<zV;h#pVhqV6hC~Pz$PRh-n5(-{76L|?yPOIH2s_}Rw
z9o|@CGReG~Bv;u>uJaXV5!XSwEd_f-sx}ipn=GXAj_~lVSDI@zY9%7%h3Y)#YPus;
zMqhXd%T{GU9#QppeEp#Op%RR29u&=ozgd^MJlT(#$JwI+x;08kHcX6*j*1xkT+3th
z{20$4%gI%tc)OL7ml2nEYQMaYE%Gpx#6t!W_fYnSGi5(EsO(cBSBj=;$~d>Xnxvs8
zCnuF59}`?HEE6YJ*U+J(fqo_B!iy-p?|VwNnMJ#b7nk^bKbB{lQ<3OQN91QO`hV;O
zr(EheeOe{n(_EE@%}K}gBnRvhESZVi*K|bYJC<5}IdR|ANGMSwgC`{`{Nir7fs+es
zFp^qWXWjaRu(c=ar&M#<QK&wjt#L6ycGVbN7zC1@sZr%h$f<iw*$StEhvn`XI+Q|0
z)#A$Ich#a*$xWt<Nx?jj6t|n$d$pBc3++v5t+Z@<tyJrh<H@cnu2q@!ZN>*mjWh4`
zq1N?x%u2T?%Vt*Kpp}g4VClx~uM?KePR%Gr`;@T7CyFTri4M{u1>5CG5148hiN<TD
znFj90{fV=jltn%&$zHk{aoG(tb*ZMml0BKs|BFfbT=@_C|3F}z-i5vYQFt-5;1XB|
zzYh<BFJkv!2R57ycVh3~0`CWz|KAF~4L`%?|5vyTWIups!Nnl&0C*7O;7j-b-UUtA
z4zdQ|L(qf?I2+D_Pv8f*8D0g~z+rd_$a4Tc!xtd?1biI+5^jd)!aO_yWZwUS@Bw%U
z%)lA&FZcm&hQqKHWS;*{css~`fLq{fkbMCE0{#?akAREeRFFCTw*-EGF~0A@=9jts
zWsv!O@dXs%Zshw>ki7z42}<4~Yie43Om&-qb6j=&`9j%?C9=D}dH7Od;pMT)Jdu@U
zI}%IJ5`+BiYr-XgjD4UYH_~KYW34C*%2SqVk-6*@;@G&lWL7k@FOsaY7kg~qaJ6es
zY2xBZHd)$rVZyFVc2auKKRe$nc}<5&Fxh~LKUyAkS_$V_J>nkeVmdq8boPpS{JdOn
z@uky#Y`gvYf5fPkJcs5j?^0-%?76lit7d=NqhqG>R+HPn{*{R7(ELTClB9g{kzy*t
zDPk|h#pqj1k;_FIm6no6cM~ce2gzr?%^*@+eo;d$=v5ms7Z%ot=uxznmBK0AisaZc
za+K5ZQJ3^l7jsXN5UZ&H(+pL5nRhjb`#x5Jfu!71!jkF!pFE453u-Dzyk7B2=q663
zSGwVi#?{DF$iU5SLYnQp((+;%?bg)d`S+WW3YeO?Jev29ixZ=3TU>oRiscrEL%4Zw
zE?SMgmWs$SnoRV<?7#CtgDTMW3PBf<bPNqv?-|WA%xHE~O6hd0#F@7YrcPSs39h($
zHq}T^t%NXIC#9D8NZx6~gGG^>nVg7tSWSqSf0x})>UHKZyP8!wiAR3|1F!o6Hb&oz
zwLvl=G5%Wk{}bzJlp^#T-D>d{r=};K>MUc4DgR<*-C{z1TKN8V%x5v2eF)}=l+Q_R
z2`d$ly!ZBORAJj;eP*?~ykNH~(w=BeiF!EEjY;a$wA^&+@y0J#XSFVM+<5#=GnW(c
zNwhLKl<e=qIh>rFl~4bB88b&0vAEo+M8=L<wUwD$yqD(ZY;F__PHQ+xXQo2at^r?n
z!uyY|;L=j7!P-K9=d6|Ae|}|6W$<RAn~2!|nB3v>{|Ch;<?~@|{8vI7u7v$y!B4RD
z{~2zDkHMQj-UGM|eu2Gz4}2Bg2}j|4co;kszKN}WGdvsq03HGlhA)zVS3(Qc!Y{D(
z?*Vx~z_sAQ@4+vz^M3)KgAc>A;ptF-H827vgX{_LHn<98Ux58E0rHN(&tvm{4qgvs
z*aK&StP}VFyb~6{g70GY-v+nBb?|g}C_Dt-hwZ-%W3T~!gzYc;1ilrX36BR^=l?J8
zH*f=74o`r4u>b!L$U6Uwy#V~4g+8y<kQe=RQ(Cz!xI}G<#dXZd2S?269DRvfkqd%f
zg$~;znQIPv{ytj0l2SKpaSbrI@?Rau4$t}>gGpO*Jf{oM_df5hnahkdvbtam`}<n{
z4&y57u*LdrHzg>Ub4=dVRPr=OW<30<$7Gk@twgdlXmTo8uI;*$YElH^B#cWzf``r#
z^LoZ)z%Ki1$K(2y6lrARNWx#W+H8!Qdv>T-_qAlr9;k8A_8OE1%Gx!n5se*G@iR0c
zh$W`uS*@;@wyW72?RaDHPt!;|dXyG_#HqV>wd8N)&UUrIZ+|P>P&Bclgvx41ymK^&
zdo&-rKH@%zr#5gre(oaitg$Y0tg>ClH&tUe<?s-Yax!{4|6R%x7AVy-==F2!h|23!
zBkytYfA=r65wVVoFq`5jYWFSKRsPUq>rV(S3@jZWuLr1FUfHSJEw|whJp=90rDvS%
zwtBQtov+oGF_FJMMNZ|~<jvyKM~SqeN})_b{fFFBzA9Wwusx0TMGbb3iVVJ^u~I*5
zG*dB7^flH6cBP9nO%^2Yac{h?P7{;5=3a2eN3$O>yi9lF`z>Rghmms-x>wKP-zQh8
zDUseRn$gJ8G99sqKHf>>L}c<&Wzh|VZL{QEKFM}+RGuVK;}BV~<9AlUx)F&YrFiRP
zDUVIFwwS{1I2NXE#VF&6>I}LS-(=P((XUJpc5A*<!%e7+>ZlY%a7I-+ZMW*lqsq}i
zOP5OR-zN7?@AoS*qeTu>-~M_m$)dTu#`E@U8YOm`%SNhEmyb~|(<Id(a__Vi<b-A}
zFa5o*BrvTq*|Ye)%aa~asx}e?N)`WLeoN#*J5_dHPWN%SCz?$b{i@eT4N~}YIm|i$
zRfwRx8Kp37OVlLNLC5846hd9L`aW9T5uqALB7WmrQmU#3lx8-mUoI0b@Hmbs-DLh)
Oq1jOLqPjk~fBb(>pelm^

literal 0
HcmV?d00001

diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py
index 6a9cbf0..f40098f 100644
--- a/unit_tests/utilities/test_zaza_utilities_openstack.py
+++ b/unit_tests/utilities/test_zaza_utilities_openstack.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import copy
+import datetime
 import io
 import mock
 import tenacity
@@ -854,6 +855,54 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
                 'myvm',
                 'org-hypervisor')
 
+    def test_wait_for_server_update_and_active(self):
+        openstack_utils.wait_for_server_migration.retry.stop = \
+            tenacity.stop_after_attempt(1)
+        novaclient = mock.MagicMock()
+        servermock = mock.MagicMock()
+        servermock.updated = '2019-03-07T13:41:58Z'
+        servermock.status = 'ACTIVE'
+        novaclient.servers.find.return_value = servermock
+        # Implicit assertion that exception is not raised.
+        openstack_utils.wait_for_server_update_and_active(
+            novaclient,
+            'myvm',
+            datetime.datetime.strptime(
+                '2019-03-07T13:40:58Z',
+                '%Y-%m-%dT%H:%M:%SZ'))
+
+    def test_wait_for_server_update_and_active_fail_no_meeta_update(self):
+        openstack_utils.wait_for_server_update_and_active.retry.stop = \
+            tenacity.stop_after_attempt(1)
+        novaclient = mock.MagicMock()
+        servermock = mock.MagicMock()
+        servermock.updated = '2019-03-07T13:41:58Z'
+        servermock.status = 'ACTIVE'
+        novaclient.servers.find.return_value = servermock
+        with self.assertRaises(exceptions.NovaGuestRestartFailed):
+            openstack_utils.wait_for_server_update_and_active(
+                novaclient,
+                'myvm',
+                datetime.datetime.strptime(
+                    '2019-03-07T13:41:58Z',
+                    '%Y-%m-%dT%H:%M:%SZ'))
+
+    def test_wait_for_server_update_and_active_fail_not_active(self):
+        openstack_utils.wait_for_server_update_and_active.retry.stop = \
+            tenacity.stop_after_attempt(1)
+        novaclient = mock.MagicMock()
+        servermock = mock.MagicMock()
+        servermock.updated = '2019-03-07T13:41:58Z'
+        servermock.status = 'NOTACTIVE'
+        novaclient.servers.find.return_value = servermock
+        with self.assertRaises(exceptions.NovaGuestRestartFailed):
+            openstack_utils.wait_for_server_update_and_active(
+                novaclient,
+                'myvm',
+                datetime.datetime.strptime(
+                    '2019-03-07T13:40:58Z',
+                    '%Y-%m-%dT%H:%M:%SZ'))
+
     def test_enable_all_nova_services(self):
         novaclient = mock.MagicMock()
         svc_mock1 = mock.MagicMock()
diff --git a/zaza/charm_tests/masakari/tests.py b/zaza/charm_tests/masakari/tests.py
index f74ad8a..5168bbf 100644
--- a/zaza/charm_tests/masakari/tests.py
+++ b/zaza/charm_tests/masakari/tests.py
@@ -16,6 +16,7 @@
 
 """Encapsulate masakari testing."""
 
+from datetime import datetime
 import logging
 
 import novaclient
@@ -40,27 +41,74 @@ class MasakariTest(test_utils.OpenStackBaseTest):
         cls.nova_client = openstack_utils.get_nova_session_client(
             cls.keystone_session)
 
-    def test_instance_failover(self):
-        """Test masakari managed guest migration."""
-        # Launch guest
-        vm_name = 'zaza-test-instance-failover'
+    @classmethod
+    def tearDown(cls):
+        """Bring hypervisors and services back up."""
+        logging.info('Running teardown')
+        for unit in zaza.model.get_units('nova-compute',
+                                         model_name=cls.model_name):
+            zaza.configure.masakari.simulate_compute_host_recovery(
+                unit.entity_id,
+                model_name=cls.model_name)
+        zaza.utilities.openstack.enable_all_nova_services(cls.nova_client)
+        zaza.configure.masakari.enable_hosts()
+
+    def ensure_guest(self, vm_name):
+        """Return the existing guest or boot a new one.
+
+        :param vm_name: Name of guest to lookup
+        :type vm_name: str
+        """
         try:
-            self.nova_client.servers.find(name=vm_name)
+            guest = self.nova_client.servers.find(name=vm_name)
             logging.info('Found existing guest')
         except novaclient.exceptions.NotFound:
             logging.info('Launching new guest')
-            zaza.configure.guest.launch_instance(
+            guest = zaza.configure.guest.launch_instance(
                 'bionic',
                 use_boot_volume=True,
                 vm_name=vm_name)
+        return guest
 
-        # Locate hypervisor hosting guest and shut it down
+    def get_guests_compute_info(self, vm_name):
+        """Return the hostname & juju unit of compute host hosting vm.
+
+        :param vm_name: Name of guest to lookup
+        :type vm_name: str
+        """
         current_hypervisor = zaza.utilities.openstack.get_hypervisor_for_guest(
             self.nova_client,
             vm_name)
         unit_name = juju_utils.get_unit_name_from_host_name(
             current_hypervisor,
             'nova-compute')
+        return current_hypervisor, unit_name
+
+    def get_guest_qemu_pid(self, compute_unit_name, vm_uuid, model_name=None):
+        """Return the qemu pid of process running guest.
+
+        :param compute_unit_name: Juju unit name of hypervisor running guest
+        :type compute_unit_name: str
+        :param vm_uuid: Guests UUID
+        :type vm_uuid: str
+        :param model_name: Name of model running cloud.
+        :type model_name: str
+        """
+        pid_find_cmd = 'pgrep -u libvirt-qemu -f {}'.format(vm_uuid)
+        out = zaza.model.run_on_unit(
+            compute_unit_name,
+            pid_find_cmd,
+            model_name=self.model_name)
+        return int(out['Stdout'].strip())
+
+    def test_instance_failover(self):
+        """Test masakari managed guest migration."""
+        # Launch guest
+        vm_name = 'zaza-test-instance-failover'
+        self.ensure_guest(vm_name)
+
+        # Locate hypervisor hosting guest and shut it down
+        current_hypervisor, unit_name = self.get_guests_compute_info(vm_name)
         zaza.configure.masakari.simulate_compute_host_failure(
             unit_name,
             model_name=self.model_name)
@@ -80,3 +128,36 @@ class MasakariTest(test_utils.OpenStackBaseTest):
             model_name=self.model_name)
         zaza.utilities.openstack.enable_all_nova_services(self.nova_client)
         zaza.configure.masakari.enable_hosts()
+
+    def test_instance_restart_on_fail(self):
+        """Test singlee guest crash and recovery."""
+        vm_name = 'zaza-test-instance-failover'
+        vm = self.ensure_guest(vm_name)
+        _, unit_name = self.get_guests_compute_info(vm_name)
+        logging.info('{} is running on {}'.format(vm_name, unit_name))
+        guest_pid = self.get_guest_qemu_pid(
+            unit_name,
+            vm.id,
+            model_name=self.model_name)
+        logging.info('{} pid is {}'.format(vm_name, guest_pid))
+        inital_update_time = datetime.strptime(
+            vm.updated,
+            "%Y-%m-%dT%H:%M:%SZ")
+        logging.info('Simulating vm crash of {}'.format(vm_name))
+        zaza.configure.masakari.simulate_vm_crash(
+            guest_pid,
+            unit_name,
+            model_name=self.model_name)
+        logging.info('Waiting for {} to be updated and become active'.format(
+            vm_name))
+        zaza.utilities.openstack.wait_for_server_update_and_active(
+            self.nova_client,
+            vm_name,
+            inital_update_time)
+        new_guest_pid = self.get_guest_qemu_pid(
+            unit_name,
+            vm.id,
+            model_name=self.model_name)
+        logging.info('{} pid is now {}'.format(vm_name, guest_pid))
+        assert new_guest_pid and new_guest_pid != guest_pid, (
+            "Restart failed or never happened")
diff --git a/zaza/configure/masakari.py b/zaza/configure/masakari.py
index f4f84ac..75c6076 100644
--- a/zaza/configure/masakari.py
+++ b/zaza/configure/masakari.py
@@ -167,3 +167,20 @@ def simulate_compute_host_recovery(unit_name, model_name):
         'start',
         ['corosync', 'pacemaker', 'nova-compute'],
         model_name)
+
+
+def simulate_guest_crash(guest_pid, compute_unit_name, model_name):
+    """Simulate a guest crashing.
+
+    :param guest_pid: PID of running qemu provess for guest.
+    :type guest_pid: str
+    :param compute_unit_name: Juju name of hypervisor hosting guest (app/n)
+    :type compute_unit_name: str
+    :param model_name: Name of model unit_name resides in.
+    :type model_name: str
+    """
+    pid_kill_cmd = 'kill -9 {}'
+    zaza.model.run_on_unit(
+        compute_unit_name,
+        pid_kill_cmd.format(guest_pid),
+        model_name=model_name)
diff --git a/zaza/utilities/.openstack.py.swp b/zaza/utilities/.openstack.py.swp
new file mode 100644
index 0000000000000000000000000000000000000000..04d4ce13fa765e596706bd444653543bdee3ef47
GIT binary patch
literal 16384
zcmeHNON<;x8EyzpjKML)oWjACaiC|>PLI95B#SIO9*;L8@9wPTv5qm$sHUf8rm{WV
z?S9O7ylgBGiI7kfDY*ec5DpvwDa1JhP>d2l5jQ|w5aMa&ghU8n6C5YutFG#Kc{U=%
zIYjD~zTN5iANAK)kM6&^8VA;kE94&IfQIWMn)b@#7e4p!8KOOQP}9OjXa}JXy!F+#
zLuZg_P(R>~y|BIRL(*Zk?|2*8hDU9)!~E=ddY;NZVZbdOaFjVNv;BaY8-_dB-d`d~
z7)Tg+I}GgB4lNvvruzQWU1aYawYOuO=q3y#3?vLB3?vLB3?vLB3?vLB47|q~2;7fp
z7tn_Ht9Hcc_WRU*{QpgLwN?4~Dft^p-caR#o|3<=<Y!fRUOA5aU!U^-%#{2!C8x^%
z<0|0T{@RrN=cnZVQSz3uzdR*>b;`bYi&TvL|69pFtL&eel3!Kwrjqaaz@+^ZC3lqE
zo|6Ad$xS8y{gnKlN^U9nksXu%|DohDKi`{@zcS^2_pOume^+u(`FE$}FDv;2O8%EA
z`QMcMw2~jWZPLG@OS?<SzcwXT#mT>ffrNpCfrNpCfrNpCfrNpCfrNpCfrNo~i2;g)
zbuWDH75Ol}|Ht|NpY~|lAAqj|r+_Sw0j}S!Y1e>f02lZK@W<Vnb_sY6cmjAFSOW4u
z8rTC|`>>{627V6w6nFvn7Vu5L1&#op0d4_)@gYt74)8_bJm3K?umao(T;8Q=PXZla
z5x5ta1NH&818;s%)2;w70lx!&30wj$0^bIn03HUu1ayHl;0TZcb^$wq+ki_uHSNoQ
z2V{X)k+WU}t^h9qzX2`+j{p~dvp@y-B(M+I1H6np`32y|z;}VK01p5sfdcR;;C1Bc
zSApLG7lB8B3%~}j1l$Si1wIN~$G*Vxz!q>6*a7?u$@QbaI<O!TZQyzI=TCr3z;nPi
zfX4t1+zp8Sz5|$X5nMW>{=oC8zDPDTQ6Mf|9Ne%R+RWf=MiBa$F7tibW>C`CX=nw+
zW~}Lxz#$DrxQ(+M)Z*t^Q{S4`A`dg12;+w3m>W=&T6$0TifTStDxX*`<&M=W)m*h!
zsTc0g7uKr9(rSeqMqrwDCi1(f+q4^nVKQzxT^2A;pC@`Sa7AhUzFc{wUOlx|sGQcv
zG|t3f%~TwoLCG|!-)T71YwEGZsKQxVvKb>CliSU=2uMnAbltXaLWXVE4kf+O%w}QW
zId-N^1J<X5?d+A>j3&hnsLTHkW2Z+mrqgx9VB64b=*+4&GPRtr2?sXXU|HL<wvCSs
zHt++--nQzYT<}&~TPo*{73!5jdA*n~h%t3q+loe1JTk<9bv)kYcHL(|!0omeucJ<v
zGCi_^F45hMNMkh01Q}h&j>q;$5Djo)^G@p;MKc<xY9`GR#WVVyRy(9p)iu1oSry1>
zGtHlzEjOzUIg6(1P*dw>)ggxJR1MMSy=nf{gr6K}@%$FGj4gn2-IRk|3_f+~n;x^U
z=CiGg&%7S<GNzb_yxY!fQqSyc8ThHsX+m9Z;FupbF@Do}CkWidY&KHP@Gc$(z9CQf
z83-pk^qMt8&trCw)wSBu+G@2{FRtX43iVofSuW-IV!2$Zj7kw_?7ZppZSmAGOzwG(
zFC#N-7U-I?Di@Xuxk}{if3eG=eLmWdRc&tScz=kR_F(v6NT^T5|GGGi=*z*~W;XM<
zDMae$xRjkUnJdmdLc8TTT_TP&8pG3!gm+!X3rKn-jElNXlUaBuONG$0EZ&Ik0%lW_
z2ZPal*KKi&#rOWeAN~v*uIHF|8V^hOx%j6yv~a0MyL`imCC*UYa@ttp+OZ&NexLFn
zF5p9(CY~w*?=np`#_#i1U|JkqOH?b=9jF=;N^QpsXpd8=Vc_{<^cG?>B@+}+s7XsT
znw6y->9OUAeqpRMM}@N-^cLy|V)IeF6O|}4%w45$L-WKWma@A1rzw@m?l(+^&PQb8
zzZz$~1v##Iht!CP7f%`nor4B%f=_gP9HJUI6H2knN3Ub($#7#08IRFB6l<wAAc{$q
zdhgI|%b5gC+v2QmG~VTcsp?Q9<mI41jZ4vcjJ%ina%G>?S9gM^Xo;8is)FHjY}g*h
zwwWpBTs(D4YlT(3CFW1m)oM_wujPv6iX?gs0<qrYOUDYUxs`${uAD3$uTB?^JdJ5b
z?KB*}uGS7ejwG$6QYo$4;Xc7^4lPE+%M0bUPbjg&Zi9KmX^~cFN8$RU<#>2E#v3X{
zf!!v39&|`)c;FdnNGi1=J#DL6iOxoA+J0j3F2~EbSbRt*UNcG1VPu(`I6%z|C{!35
zFN&%cD!$81-ol!O-f4$exJlbV+iVe%%e*d+h7}rNH{s=Ev3g&rRwXBM<#KMdTExpz
zsZ8>v)ni5ZI$k<Xa;vAviQ?+9dBV^nM8eLw9!4h-4eq)a4^0tJlly_k8+bo#-WV|f
zjpGWAjQ~h0S0TkpiX6>Vis-rhq-I;fvB%Da+=HLSfKV4Tn6!ZouxMY@N!uRM5s{5>
zoAf<oBX)b9_)aV6Q!I-6;YVER5!Ubbu_93%NjDs?orN#Kpr2jFo8fAukXgVHd0}$O
z2}zd@h|>_Aj6OCU7k8}*l4$_Vfzq^{fP}v21vE~q;2=gkFhp2b$q=@9&zIw1s`FGo
zjuVxOVvKuuo~+Af89{s`EzDEOC^h%#DBwKe!y9ZTAPWZ$-b3=#7Ef<#k>vpWA@cth
zku$fESBv~V{yzI6a{Z@)$AKPD0CoVcA>aQI@I&BP;8DN=Hh_D9TY)!_@BbS3KF|Q}
z2KECf;IGK<e*ioMJOrEo_5nKq4fqw}_$nacX^EJUD`6mEAYmY3AYmY3AYmY3AYtIW
z$ACyQJQjqWjSVE^Yk`-R`)v41QL1DL_g6EO@_Hs$t(F%%jvuJ4HmQR>EwR_2b{Py=
z=o>BM$!YQ7<8V7<eoz-b<u;bu&M~;%a?<*7=!vXO_>KPKtBBX*zT>G{L;@=UTom6+
zNUs}tS|mu*WO1}_G%8<3JtE<WY8B~~<v5+t9dj`j?P3(VOy;rMW2j&GXm?=T{e(>p
z+aO{K#Kdk>Gx9*Bp%$4S;IbrBBs`+saRSyEwr$v4Dg!mNCPU0DsE>5&NU-ZsGty(#
z&r1Wz=OV@$8N0^xqK-hC=o{FnL*AM8d!`{sMv{qN4Q^9!AUh;>Fwrrof-n<3vbjZ&
z=x%PM3<T7rLF7r=OJBn4+{hJgMK#E;aO5zyN|7^*5Ssd86s_1eKnx<ooEfl%96UWI
zbtij&O#R*Jx~SQVP|K1%E5>s~W7I>E%Q~d>D3(|{-b09d)GHI5>Xlj^Uvesf(hl~L
zu>TNH4}(aLsa3;UvB@l-NgCoV(#z6Bj3_vvE$q|~)59k~(xOP-xAP?IJ<5^ikG39-
zRZQC@F*%jf<B^AS)JJiJaylEbGsikv$kncSJ#^!JSGk=P+p5Nl3Ma`x{I$rkiLU@#
Pq#{1rieGa>Hnjf&Ymyx=

literal 0
HcmV?d00001

diff --git a/zaza/utilities/exceptions.py b/zaza/utilities/exceptions.py
index dc52cca..868a7c6 100644
--- a/zaza/utilities/exceptions.py
+++ b/zaza/utilities/exceptions.py
@@ -166,3 +166,9 @@ class NovaGuestMigrationFailed(Exception):
     """Nova guest migration failed."""
 
     pass
+
+
+class NovaGuestRestartFailed(Exception):
+    """Nova guest restart failed."""
+
+    pass
diff --git a/zaza/utilities/openstack.py b/zaza/utilities/openstack.py
index ad171cb..7dca504 100644
--- a/zaza/utilities/openstack.py
+++ b/zaza/utilities/openstack.py
@@ -42,6 +42,7 @@ from neutronclient.common import exceptions as neutronexceptions
 from octaviaclient.api.v2 import octavia as octaviaclient
 from swiftclient import client as swiftclient
 
+import datetime
 import io
 import juju_wait
 import logging
@@ -1922,6 +1923,37 @@ def wait_for_server_migration(nova_client, vm_name, original_hypervisor):
             current_hypervisor))
 
 
+@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
+                reraise=True, stop=tenacity.stop_after_attempt(80),
+                retry=tenacity.retry_if_exception_type(
+                    exceptions.NovaGuestRestartFailed))
+def wait_for_server_update_and_active(nova_client, vm_name,
+                                      original_updatetime):
+    """Wait for guests metadata to be updated and for status to become active.
+
+    :param nova_client: Authenticated nova client
+    :type nova_client: novaclient.v2.client.Client
+    :param vm_name: Name of guest to monitor
+    :type vm_name: str
+    :param original_updatetime: The time the metadata was previously updated.
+    :type original_updatetime: datetime
+    :raises: exceptions.NovaGuestMigrationFailed
+    """
+    server = nova_client.servers.find(name=vm_name)
+    current_updatetime = datetime.datetime.strptime(
+        server.updated,
+        "%Y-%m-%dT%H:%M:%SZ")
+    if current_updatetime <= original_updatetime or server.status != 'ACTIVE':
+        logging.info('{} Updated: {} Satus: {})'.format(
+            vm_name,
+            current_updatetime,
+            server.status))
+        raise exceptions.NovaGuestRestartFailed(
+            'Restart of {} after crash failed'.format(vm_name))
+    else:
+        logging.info('SUCCESS {} has restarted'.format(vm_name))
+
+
 def enable_all_nova_services(nova_client):
     """Enable all nova services.