#!/usr/bin/perl
###############################################################################
# IBM (C) Copyright 2016 Eclipse Public License
# http://www.eclipse.org/org/documents/epl-v10.html
###############################################################################
# COMPONENT: verifynode
#
# Verify the capabilities of a node.
###############################################################################

package xCAT_plugin::verifynode;
BEGIN
{
    $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat';
}

$XML::Simple::PREFERRED_PARSER='XML::Parser';

use strict;
use warnings;

use Data::Dumper qw(Dumper);
use File::Basename;
use File::Path;
use File::Spec;
use File::Temp;
use Getopt::Long;
use Getopt::Long qw(GetOptionsFromString);
use lib "$::XCATROOT/lib/perl";
use MIME::Base64;
use POSIX qw/strftime/;
use Scalar::Util qw( looks_like_number );
use Sys::Hostname;
use Sys::Syslog qw( :DEFAULT setlogsock );
use Socket;
use Text::Wrap;
use Time::HiRes qw( gettimeofday sleep );
use Unicode::UCD qw(charinfo);

use xCAT::zvmCPUtils;
use xCAT::zvmMsgs;
use xCAT::zvmUtils;

my $version = "1";

my $comments = '';                     # Comments for the IVP run
my $dataFile = '';                     # Input data file
my $decode = '';                       # String specified by the --decode operand.
my $disable = '';                      # Disable scheduled IVP from running.
my $displayHelp = 0;                   # Display help information
my $driver = '';                       # Name of driver script
my $driverLoc = '/var/opt/xcat/ivp';   # Local directory containing the driver script, if it exists
my $encode = '';                       # String specified by the --encode operand.
my $enable = '';                       # Enable scheduled IVP to run.
my $endTime = '';                      # Human readable end time
my %hosts;                             # Information on host nodes known by xCAT MN
my $id = '';                           # Id of the IVP run
my %ignored;                           # List of ignored messages
my $ignoreCnt = 0;                     # Number of times we ignored a message
my $issueCmdOverIUCV = '';             # Issue a command to a node over IUCV
my $issueCmdToNode = '';               # Issue a command to a node using IUCV or SSH
my %localIPs;                          # Hash of the Local IP addresses for the xCAT MN system
my $logDir = '/var/log/xcat/ivp';      # Log location
my $logFile = '';                      # Log file name and location
my $logFileHandle;                     # File handle for the log file.
my @ivpSummary;                        # Summary output lines for an IVP run.
my $locApplSystemRole = '/var/lib/sspmod/appliance_system_role';
my %mnInfo;                            # Managed node environment information
my %msgsToIgnore;                      # Hash of messages to ignore
my $needToFinishLogFile = 0;           # If 1 then finishLogFile() needs to be called
my $nodeName = '';                     # Name of node to be verified
my $notify = 0;                        # Notify a user
my $notifyOnErrorOrWarning = 0;        # Indicates a log file should be sent if the need to notify
                                       #   a user was determined.
my $openstackIP = '';                  # IP address of the OpenStack node
my $openstackUser = 'root';            # User on the OpenStack system that can access the
                                       # OpenStack configuration files
my $orchParms = '';                    # Orchestrator parms to be stored in the zvmivp table
my $prepParms = '';                    # Parameters to pass to the preparation script
my $remove = 0;                        # Remove a scheduled IVP
my $runBasicIVP = 0;                   # Run the basic xCAT IVP
my $runCron = 0;                       # Cron job invocation to run a possible set of IVPs
                                       # on a periodic basis.
my $runFullIVP = 0;                    # Run the full end to end IVP (compute node and xCAT)
my $schedule = '';                     # Hours to schedule an automated IVP
my $scheduledType = '';                # Type of IVP to schedule ('fullivp' or 'basicivp')
my $startTime = '';                    # Human readable start time
my $tgtOS = '';                        # Target Node's distro & version
my $todayDate = '';                    # Today's date in the form 'yyyy-mm-dd'
my $verifyAccess = 0;                  # Verify system can be accessed
my $verifyCaptureReqs = 0;             # Verify systems meets capture requirements
my $verifyCloudInit = 0;               # Verify cloud-init installed
my $verifyXcatconf4z = 0;              # Verify xcatconf4z
my $versionOpt = 0;                    # Show version information flag
my $warnErrCnt = 0;                    # Count of warnings and error messages for a verify set
my $zhcpTarget = '';                   # ZHCP server that is related to this verification
my $zvmHost = '';                      # z/VM host related to this verification
my $zxcatParms = '';                   # Parmeters to pass to zxcatIVP

# Routines to drive based on the selected test.
my %verifySets = (
    '$decode ne \'\'' =>
         [ '',                        # Do not show a function banner.
           'encodeDecodeString()',
         ],
    '$disable and ! $remove and $schedule eq \'\'' =>
         [ 'Disabling an IVP run',
           'enableDisableRun( $id )',
         ],
    '$enable and ! $remove and $schedule eq \'\'' =>
         [ 'Enabling an IVP run',
           'enableDisableRun( $id )',
         ],
    '$encode ne \'\'' =>
         [ 'Encoding a string',
           'encodeDecodeString()',
         ],
    '$issueCmdOverIUCV ne \'\'' =>
         [ 'Issue a command to a node using IUCV',
           'cmdOnVM( \'onlyIUCV\' )',
         ],
    '$issueCmdToNode ne \'\'' =>
         [ 'Issue a command to a node using IUCV or SSH',
           'cmdOnVM( \'either\' )',
         ],
    '$remove' =>
         [ 'Remove an automated periodic IVP',
           'removeIVP( $id )',
         ],
    '$runBasicIVP' =>
         [ 'Verifying the basic general setup of xCAT',
           'addLogHeader( \'basicivp\' )',
           'verifyBasicXCAT()',
         ],
    '$runCron' =>
         [ 'Cron job for automated verification',
           'driveCronList()',
         ],
    '$runFullIVP' =>
         [ 'Verifying the full end-to-end setup of the xCAT and the compute node',
           'addLogHeader( \'fullivp\' )',
           'checkForDefault( \'openstackIP\', \'--openstackIP was not specified\', \'xCAT_master\' )',
           'verifyAccess( $openstackIP ); $rc = 0',
           'getDistro( $openstackIP )',
           'runPrepScript()',
           'runDriverScript()',
         ],
    '$schedule ne \'\'' =>
         [ 'Scheduling an automated periodic IVP',
           'scheduleIVP()',
         ],
    '$verifyAccess' =>
         [ 'Verifying access to the system',
           'verifyPower( $nodeName )',
           'verifyAccess( $nodeName )'
         ],
    '$verifyCaptureReqs' =>
         [ 'Verifying system meets the image requirements',
           'verifyPower( $nodeName )',
           'verifyAccess( $nodeName )',
           'getDistro( $nodeName )',
           'verifyDistro()',
         ],
    '$verifyCloudInit' =>
         [ 'Verifying cloud-init is installed and configured',
           'verifyPower( $nodeName )',
           'verifyAccess( $nodeName )',
           'getDistro( $nodeName )',
           'verifyService($nodeName, "cloud-config,2,3,4,5 cloud-final,2,3,4,5 cloud-init,2,3,4,5 cloud-init-local,2,3,4,5")',
           'showCILevel()',
         ],
    '$verifyXcatconf4z' =>
         [ 'Verifying xcatconf4z is installed and configured',
           'verifyPower( $nodeName )',
           'verifyAccess( $nodeName )',
           'getDistro( $nodeName )',
           'verifyXcatconf4z( $nodeName )',
         ],
    '$test' =>
         [ 'Script test functon.',
           'verifyLogResponse()',
         ],
    );


# set the usage message
my $usage_string = "Usage:\n
    $0 -n <node> <options>\n
    or\n
    $0 <options>\n
    or\n
    $0 -v\n
    or\n
    $0 [ -h | --help ]\n
    The following options are supported:\n
      -a | --access
                          Verify access from xCAT MN to node.
      --basicivp
                          Run the basic xCAT IVP.
      --capturereqs
                          Verify that node meets system capture
                          requirements for OpenStack.
      --cmdoveriucv <cmd>
                          Issue a command to a node using IUCV.
                          Specify a period for <cmd> when combining this
                          option with the --file option.
      --cmdtonode <cmd>   Issue a command to a node using IUCV or SSH
                          if IUCV is not available.
                          Specify a period for <cmd> when combining this
                          option with the --file option.
      -c | --cloudinit
                          Verify that node has cloud-init installed.
      --comments
                          Comments to specify on the IVP run.
      --cron
                          Driven as a cron job to run a set of periodic
                          IVPs.
      --decode <parm>
                          Decode an encoded string.  This is used to
                          view encoded values in the zvmivp table.
      --disable           Indicates that the IVP being scheduled should
                          be disabled from running.
      --enable            Indicates that the IVP being scheduled should
                          be enabled to run.  This is the default when
                          neither --disable or --enable is specified.
      --encode <parm>
                          Encode a string.  This used for debug in order
                          to create a string to compare to an encoded
                          string from the zvmivp table. The <parm> must be
                          an ASCII string.  This function does not support
                          encoding a UTF-8 character string.
      --file              File location containing additional input.
                          When used with the --cmdoveriucv or --cmdtonode
                          operands, this file contains the command to issue.
      --fullivp
                          Run the full end-to-end IVP.  This
                          verification checks the compute node and builds
                          a driver script which is then used to check
                          the xCAT environment based on the values
                          gathered in the driver script.
      -h | --help
                          Display help information.
      -i | --id <idToRun>
                          ID to associate with the IVP run.
      --ignore <val>
                          Blank or comma separated list of message ids or
                          message severities to ignore.  Ignored messages are
                          not counted as failures and do not produce messages.
                          Instead the number of ignored messages and their
                          message numbers are displayed at the end of
                          processing.  Recognized message severities:
                          'bypass', 'info', 'warning', 'error'.
      --notify
                          Notification of IVP failures should be sent
                          to a z/VM userid.
      -n | --node <node>
                          Node name of system to verify.
      --openstackip <ipAddr>
                          IP address of the OpenStack compute node.
      --openstackuser <user>
                          User to use to access the compute node.
                          The user defaults to 'root'.
      --orchparms <parms>
                          Parameters to be stored in the zvmivp table. These
                          parameters are passed to this routine when a
                          scheduled IVP is driven.
      --prepparms <parms>
                          Parameters to pass to the preparation script as
                          part of a full IVP.
      --remove
                          Indicates that an IVP in the zvmivp table should be
                          removed.  The id is specified with the --id
                          operand.
      --schedule <hours>
                          List of hours (in 24 hour format) to periodically
                          run the IVP.  Multiple hours are separated by blanks.
                          For example, \'0 6 12 18 20\'.
      --type < basicivp | fullivp >
                          Type of IVP to be scheduled.
      -v
                          Display the version of this script.
      -x | --xcatconf4z
                          Verify that node has xcatconf4z installed.
      --zxcatparms <parms>
                          Parameters to pass to the zxcatIVP script.\n";


#-------------------------------------------------------

=head3   addLogHeader

    Description : Add a header to the log file.
    Arguments   : None
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = addLogHeader();

=cut

#-------------------------------------------------------
sub addLogHeader {
    my ( $hType ) = @_;

    logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );

    # Generate the header based on the header type.
    if ( $hType eq 'basicivp' ) {
        logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "This log file was created for a basic xCAT IVP. It consists of output from ".
                           "zxcatIVP that validates the xCAT management node (MN) and the ZHCP Agents that are used by ".
                           "the xCAT MN node to manage the z/VM hosts." );
    } elsif ( $hType eq 'fullivp' ) {
        logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "This log file was created for a full xCAT IVP which validates the z/VM OpenStack ".
                           "configuration files, the xCAT management node (MN) and the ZHCP Agents that are used by ".
                           "the xCAT MN node to manage the z/VM host. The log file contains the following output: output ".
                           "from the preparation script that validated the OpenStack properties, the contents of the ".
                           "driver script created by the preparation script to drive the second part of the IVP, output ".
                           "of the zxcatIVP run that is driven by the driver script to complete the full IVP." );
    } else {
        logResponse( 'DF', 'GENERIC_RESPONSE', "VerifyNode log file." );
    }

    logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );

    return 0;
}


#-------------------------------------------------------

=head3   checkForDefault

    Description : Determine if a default is needed for
                  the specified property and set it if
                  it is needed.  Currently, this routine
                  is only used for the master IP address.
    Arguments   : Name of property
                  String to use in a DFLT01 message.
                  Default or function to perform.
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = checkForDefault( 'openStackIP', '--ip was not specified', 'xCAT_master' );

=cut

#-------------------------------------------------------
sub checkForDefault {
    my ( $prop, $reasonString, $default ) = @_;
    my $additionalInfo = '';
    my $rc = 0;
    my $val;

    eval( "\$val = \$$prop" );
    if ( $val eq '' ) {
        if ( $default eq 'xCAT_master' ) {
            $default = '';
            if ( exists $mnInfo{'ip'} ) {
                $default = $mnInfo{'ip'};
                $additionalInfo = 'The value came from the IP address used as the xCAT MN\'s ' .
                    'master IP address that was specified in the site table.';
            } else {
                # Use one of the local IPv4 IPs.  We don't support IPv6.
                if ( keys %localIPs > 0 ) {
                    # Can't find the MN IP so use the first local IPv4 IP.
                    foreach my $ip ( keys %localIPs ) {
                        if ( $ip =~ /:/ ) {
                            next;
                        } else {
                            $default = $ip;
                            $additionalInfo = 'The function could not determine the master IP address for ' .
                                'the xCAT MN so the first local IPv4 address was used.';
                            last;
                        }
                    }
                    if ( $default eq '' ) {
                        $default = '127.0.0.1';
                        $additionalInfo = 'The function could not determine the IP address for the xCAT MN ' .
                            'and could not find any IPv4 addresses on the system so \'127.0.0.1\' will be used ' .
                            'in the hope that it will work.';
                    }
                } else {
                    # Don't know the local IPs.
                    # Default to the IPv4 loopback and let it fail.
                    $default = '127.0.0.1';
                    $additionalInfo = 'The function cound not determine the IP addresses on the ' .
                        'xCAT MN system so \'127.0.0.1\' was used.';
                }
            }
        }

        if ( $rc == 0 ) {
            # Found a default, set it in the property.
            eval( "\$$prop = \'$default\'" );
            logResponse( 'DF', 'DFLT01', $reasonString, $default, $additionalInfo );
        }
    }

    return $rc;
}


#-------------------------------------------------------

=head3   cmdOnVM

    Description : Send a command to a node and show the result.
    Arguments   : Target function, either 'onlyIUCV' or 'either'.
    Returns     : Return code:
                      0 - Normal Linux success
                      255 - Unable to SSH to system
                      non-zero - command error
                  Output from the command or a error string on an SSH failure.
    Example     : $rc = cmdOnVM( 'onlyIUCV' );
                  $rc = cmdOnVM( 'either' );

=cut

#-------------------------------------------------------
sub cmdOnVM {
    my ( $targetFunc ) = @_;
    my $cmd = '';
    my $hcp = '';
    my @lines;
    my $rc = 0;
    my $out = '';
    my $sudo;
    my $user;
    my $userid = '';

    # Get the command to issue.
    if ( $targetFunc eq 'onlyIUCV' ) {
        if ( $issueCmdOverIUCV ne '.' ) {
            $cmd = $issueCmdOverIUCV;
        }
    } else {
        if ( $issueCmdToNode ne '.' ) {
            $cmd = $issueCmdToNode;
        }
    }

    # Use the data file if it was specified.
    if ( $dataFile ne '' ) {
        if ( -e $dataFile ) {
            open my $handle, '<', $dataFile;
            chomp( @lines = <$handle> );
            close $handle;
            if ( $lines[0] ne '' ) {
                $cmd = $lines[0];
            } else {
                logResponse( 'DF', 'GENERIC_RESPONSE', "$dataFile does not have anything in the first line.");
                goto FINISH_cmdUsingIUCV;
            }
        } else {
            logResponse( 'DF', 'GENERIC_RESPONSE', "$dataFile does not exist.");
            goto FINISH_cmdUsingIUCV;
        }
    }

    # Verify that we have a command to issue.
    if ( $cmd eq '' ) {
        logResponse( 'DF', 'GENERIC_RESPONSE', "A command to issue was not specified.");
        goto FINISH_cmdUsingIUCV;
    }

    ($user, $sudo) = xCAT::zvmUtils->getSudoer();

    # Get the required properties (hcp, userid, user) from the node
    $out = `/opt/xcat/bin/lsdef $nodeName -i hcp,userid`;
    $rc = $?;
    if ( $rc eq 0 ) {
        my $fndNode = 0;
        @lines = split( '\n', $out );
        my $host = '';
        foreach my $line ( @lines ) {
            $line =~ s/^\s+|\s+$//g;         # trim blanks from both ends of the string
            if ( $line =~ /Object name:/ ) {
                $fndNode = 1;
                next;
            } elsif ( $line =~ /hcp\=/ ) {
                ($hcp) = $line =~ m/hcp\=(.*)/;
                next;
            } elsif ( $line =~ /userid\=/ ) {
                ($userid) = $line =~ m/userid\=(.*)/;
                next;
            }
        }
        if ( $fndNode != 1 ) {
            logResponse( 'DF', 'GENERIC_RESPONSE', "'$nodeName' is not an xCAT node." );
            goto FINISH_cmdUsingIUCV;
        }
    } else {
        $rc = logResponse( 'DFS', 'GNRL01', "$cmd", $rc, $out );
        goto FINISH_cmdUsingIUCV;
    }

    # Send the command through IUCV
    logResponse( 'DF', '*NONFORMATTED*', "\n**************************\nData for the function call\n**************************" );
    logResponse( 'DF', '*NONFORMATTED*',  "       Node: $nodeName" );
    if ( $targetFunc eq 'onlyIUCV' ) {
        logResponse( 'DF', '*NONFORMATTED*',  "z/VM userid: $userid" );
        logResponse( 'DF', '*NONFORMATTED*',  "        HCP: $hcp" );
        logResponse( 'DF', '*NONFORMATTED*',  "       User: $user" );
        logResponse( 'DF', '*NONFORMATTED*',  "   Function: xCAT::zvmUtils->execcmdthroughIUCV" );
    } else {
        logResponse( 'DF', '*NONFORMATTED*',  "       User: $user" );
        logResponse( 'DF', '*NONFORMATTED*',  "   Function: xCAT::zvmUtils->execcmdonVM" );
    }
    logResponse( 'DF', '*NONFORMATTED*', "        Cmd: $cmd" );

    logResponse( 'DF', '*NONFORMATTED*', "\n*********************\nInvoking the function\n*********************" );
    if ( $targetFunc eq 'onlyIUCV' ) {
        $out = xCAT::zvmUtils->execcmdthroughIUCV( $user, $hcp, $userid, $cmd );
    } else {
        $out = xCAT::zvmUtils->execcmdonVM( $user, $nodeName, $cmd );
    }
    logResponse( 'DF', '*NONFORMATTED*', "\n******\nResult\n******\n$out" );

FINISH_cmdUsingIUCV:
    return $rc;
}


#-------------------------------------------------------

=head3   driveCronList

    Description : Run thru the list of IVPs and run any that
                  are due to be run.
    Arguments   : None.
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = driveCronList();

=cut

#-------------------------------------------------------
sub driveCronList {
    my $attemptedRuns = 0;
    my $disabledRuns = 0;
    my @ivps;
    my $out = '';
    my $rc = 0;
    my $totalRunsInTable = 0;

    # Determine the timestamp and other time related information.
    my ($sec,  $min,  $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
    $year = $year + 1900;
    $todayDate = sprintf( "%04d-%02d-%02d", $year, $mon + 1, $mday );
    $startTime = sprintf( "%02d:%02d:%02d on %04d-%02d-%02d",
                           $hour, $min, $sec, $year, $mon + 1, $mday );
    my $currTime = sprintf( "%04d-%02d-%02d_%02d:%02d:%02d",
                           $year, $mon + 1, $mday, $hour, $min, $sec);
    my $newLastRun = "$year $yday $hour";

    logResponse( 'DFS', '*NONFORMATTED*', "CRON job run started at $startTime." );

    # Read the z/VM IVP table to get the list of known IVP runs.
    $out = `/opt/xcat/sbin/tabdump zvmivp | grep -v "^#"`;
    $rc = $?;
    my @lines = split( '\n', $out );
    $totalRunsInTable = scalar( @lines );
    if ( $totalRunsInTable != 0 ) {
        my $row = 0;
        my $nowTotalHours = ( ( $year ) * 8784 ) + ( $yday   * 24 ) + $hour;
        foreach my $line ( @lines ) {
            $row++;
            chomp( $line );
            my @field = split( ',', $line );
            for ( my $i=0; $i < scalar( @field ); $i++ ) {
                $field[$i] =~ s/^\"|\"$//g;         # trim quotes from both ends of the string
            }

            # Process the fields from the zvmIVP table.  Each row is related to a
            # scheduled run.
            # Field 0 - Unique IVP ID
            #       1 - IP property for the OpenStack system
            #       2 - Schedule, hours to run the IVP
            #       3 - Last time IVP was run, ( year day hour )
            #       4 - Type of run, e.g. basicivp, fullivp
            #       5 - OpenStack user to access the config files
            #       6 - Parameters to pass to the orchestrator, verifynode
            #       7 - Parameters to pass to the preparator script, prep_zxcatIVP
            #       8 - Parameters to pass to the main IVP script, zxcatIVP.pl
            #       9 - Comments related to the IVP run
            #      10 - Disabled, 1 or YES equals disabled
            my %ivpRun;
            if ( defined $field[0] and $field[0] ne '' ) {
                $ivpRun{'id'} = $field[0];
            } else {
                logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row in the ivp table missing a ".
                                   "required property, \'id\'.  The row will be treated as disabled." );
                $disabledRuns++;
                next;
            }

            if ( defined $field[10] and ($field[10] eq '1'  or uc( $field[10] ) eq 'YES') ) {
                # IVP run is disabled
                logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table is ".
                                                       "disabled and will not be processed." );
                $disabledRuns++;
                next;
            }

            if ( defined $field[1] and $field[1] ne '' ) {
                $ivpRun{'ip'} = $field[1];
            } else {
                # Assume this is a local xCAT IVP run
                $ivpRun{'ip'} = 'xcat';
                logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table " .
                                                       "has no value specified for the \'ip\' property. " .
                                                       "The IVP run will be against the system which is " .
                                                       "running the xCAT management node." );
            }

            if ( defined $field[2] and $field[2] ne '' ) {
                $ivpRun{'schedule'} = hexDecode( $field[2] );
            } else {
                # Default run time is 1 AM
                $ivpRun{'schedule'} = '1';
            }

            my $lRYear;
            my $lRYDay;
            my $lRHour;

            if ( defined $field[3] and $field[3] ne '' ) {
                $ivpRun{'last_run'} = $field[3];
                my @parts = split ( '\s', $ivpRun{'last_run'} );
                if ( defined $parts[0] and $parts[0] ne '' ) {
                    $lRYear = $parts[0];
                } else {
                    # Missing year, set default to immediately, 0.
                    $lRYear = 0;
                }
                if ( defined $parts[1] and $parts[1] ne '' ) {
                    $lRYDay = $parts[1];
                } else {
                    # Missing day number, set default to immediately, 0.
                    $lRYDay = 0;
                }
                if ( defined $parts[2] and $parts[2] ne '' ) {
                    $lRHour = $parts[2];
                } else {
                    # Missing hour, set default to immediately, -1.
                    $lRHour = 0;
                }
            } else {
                # Force a schedule run
                $lRYear = 0;
                $lRYDay = 0;
                $lRHour = -1;
            }
            $ivpRun{'last_run'} = "$lRYear $lRYDay $lRHour";

            # Weed out any runs that are not due to be run.
            # Calculate the difference (since 2000) between the last run hour and the current hour,
            # ignoring leap years by saying every year is a leap year.
            my $totalHourDiff = $nowTotalHours - (( ( $lRYear ) * 8784 ) + ( $lRYDay * 24 ) + $lRHour);

            if ( $totalHourDiff == 0 ) {
                # Same hour of the same day of the same year for some reason so ignore this IVP row.
                logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table ".
                                                       "is being ignored because it was already run this hour." );
                next;
            } else {
                # Determine the range of hours that are of interest within a 24 hour time period.  Anything
                # that is more than 24 hours old is automatically overdue and should be run.
                if ( $totalHourDiff < 24) {
                    my $todayStartHour;
                    my $yesterdayStartHour;
                    if ( $yday > $lRYDay ) {
                        # Last run yesterday
                        $todayStartHour = 0;
                        $yesterdayStartHour = $lRHour + 1;
                    } else {
                        # Last run today
                        $todayStartHour = $hour - $totalHourDiff;
                        $yesterdayStartHour = 24;
                    }

                    # Less than a day so we need to worry about which scheduled hours were missed.
                    my $overdue = 0;
                    my @schedHours = split ( '\s', $ivpRun{'schedule'} );
                    foreach my $sHour ( @schedHours ) {
                        if ( $sHour == 0 and $sHour ne '0' ) {
                            # not numeric, ignore
                            logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table ".
                                                                   "contains as schedule hour that is non-numeric: $sHour. ".
                                                                   "The value will be ignored." );
                            next;
                        }
                        if (( $sHour >= $yesterdayStartHour ) or
                             ( $sHour > $todayStartHour and $sHour <= $hour ) ) {
                            logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table ".
                                                                   "is due to be run. Scheduled hour: $sHour, last run: $lRHour." );
                            $overdue = 1;
                            last;
                        }
                    }
                    if ( $overdue == 0 ) {
                        # did not find a scheduled hour that is due
                        logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table is not ".
                                                               "due to be run this hour." );
                        next;
                    }
                } else {
                   logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table ".
                                                          "is due to be run. Last run over 24 hours ago." );
                }
            }

            if ( defined $field[4] and $field[4] ne '' ) {
                $ivpRun{'type_of_run'} = lc( $field[4] );
            } else {
                $ivpRun{'type_of_run'} = 'not specified';
            }
            if ( $ivpRun{'type_of_run'} ne 'basicivp' and $ivpRun{'type_of_run'} ne 'fullivp' ) {
                my $badValue = $ivpRun{'type_of_run'};
                if ( exists $mnInfo{'ip'} and ( $ivpRun{'ip'} eq $mnInfo{'ip'} )) {
                    if (( $mnInfo{'role'} eq 'controller' ) or
                          ( $mnInfo{'role'} eq 'compute' ) or
                          ( $mnInfo{'role'} eq 'compute_mn' )) {
                        $ivpRun{'type_of_run'} = 'fullivp';
                    } else {
                        $ivpRun{'type_of_run'} = 'basicivp';
                    }
                } else {
                    $ivpRun{'type_of_run'} = 'basicivp';
                }
                logResponse( 'DS', 'GENERIC_RESPONSE', "Row $row, ID($ivpRun{'id'}), in the ivp table contains ".
                                                       "an unknown run type value of \'$badValue\'. Instead ".
                                                       "a run type of \'$ivpRun{'type_of_run'}\' will be performed." );
            }

            if ( defined $field[5] and $field[5] ne '' ) {
                $ivpRun{'access_user'} = hexDecode( $field[5] );
            } else {
                $ivpRun{'access_user'} = '';
            }

            if ( defined $field[6] and $field[6] ne '' ) {
                $ivpRun{'orch_parms'} = hexDecode( $field[6] );
            } else {
                $ivpRun{'orch_parms'} = '';
            }

            if ( defined $field[7] and $field[7] ne '' ) {
                $ivpRun{'prep_parms'} = hexDecode( $field[7] );
            } else {
                $ivpRun{'prep_parms'} = '';
            }

            if ( defined $field[8] and $field[8] ne '' ) {
                # Ensure that it is reencoded to pass along to do the actual work.
                $ivpRun{'main_ivp_parms'} = hexEncode( $field[8] );
            } else {
                $ivpRun{'main_ivp_parms'} = '';
            }

            if ( defined $field[9] and $field[9] ne '' ) {
                $ivpRun{'comments'} = hexDecode( $field[9] );
            } else {
                $ivpRun{'comments'} = '';
            }

            push @ivps, \%ivpRun;
        }
    } else {
        # Table is empty.  Create a single run for the z/VM related to the
        # xCAT MN.
        logResponse( 'DS', 'GENERIC_RESPONSE', "The ivp table is empty. Default IVPs will be enabled." );
        if ( exists $mnInfo{'node'} ) {
            # First IVP is a full IVP for an OpenStack related xCAT at 1 am.
            # Otherwise it is a basic IVP.
            my %ivpRun;
            $ivpRun{'id'} = 1;
            if ( exists $mnInfo{'ip'} ) {
                $ivpRun{'ip'} = $mnInfo{'ip'};
            } else {
                $ivpRun{'ip'} = 'xcat';
            }
            $ivpRun{'schedule'} = '1';
            if ( $mnInfo{'role'} eq 'controller' or $mnInfo{'role'} eq 'compute' or $mnInfo{'role'} eq 'compute_mn' ) {
                $ivpRun{'type_of_run'} = 'fullivp';
                $ivpRun{'access_user'} = 'root';
            } else {
                $ivpRun{'type_of_run'} = 'basicivp';
                $ivpRun{'access_user'} = '';
            }
            $ivpRun{'orch_parms'} = '';
            $ivpRun{'prep_parms'} = '';
            $ivpRun{'main_ivp_parms'} = '';
            $ivpRun{'comments'} = 'Default Full IVP run';
            $ivpRun{'defaultRun'} = 'run';
            push @ivps, \%ivpRun;
            $totalRunsInTable++;

            # Second IVP is a basic IVP run each hour except for 1 am when a full IVP is run.
            my %ivpRun2;
            $ivpRun2{'id'} = 2;
            if ( exists $mnInfo{'ip'} ) {
                $ivpRun2{'ip'} = $mnInfo{'ip'};
            } else {
                $ivpRun2{'ip'} = 'xcat';
            }
            $ivpRun2{'schedule'} = '0 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23';
            $ivpRun2{'type_of_run'} = 'basicivp';
            $ivpRun2{'access_user'} = '';
            $ivpRun2{'orch_parms'} = '';
            $ivpRun2{'prep_parms'} = '';
            $ivpRun2{'main_ivp_parms'} = '';
            $ivpRun2{'comments'} = 'Default Basic IVP run';
            $ivpRun2{'defaultRun'} = 'runLater';
            push @ivps, \%ivpRun2;
            $totalRunsInTable++;
        } else {
            # xCAT MN does not exist as a node so we will only do a basic IVP the system.
            # This is a very abnormal case.  We deal in the abnormal so let's run some type
            # of IVP but not define it in the table.
            logResponse( 'DFS', 'MN03' );
            my %ivpRun;
            $ivpRun{'id'} = 1;
            if ( exists $mnInfo{'ip'} ) {
                $ivpRun{'ip'} = $mnInfo{'ip'};
            } else {
                $ivpRun{'ip'} = 'xcat';
            }
            $ivpRun{'schedule'} = '0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23';
            $ivpRun{'type_of_run'} = 'basicivp';
            $ivpRun{'access_user'} = '';
            $ivpRun{'orch_parms'} = '';
            $ivpRun{'prep_parms'} = '';
            $ivpRun{'main_ivp_parms'} = '';
            $ivpRun{'comments'} = 'Default Basic IVP run';
            $ivpRun{'defaultRun'} = 'noTableUpdate';
            push @ivps, \%ivpRun;
            $totalRunsInTable++;
        }
    }

    # Handle pruning, if necessary.
    if ( $hour == 0 ) {
        pruneLogs( $year, $mon + 1, $mday );
    }

    # Go thru the list of IVPs and drive a run for each IVP
    # that is due to be run.
    foreach my $ivp ( @ivps ) {
        my %ivpRun = %$ivp;

        # Build the verify node command.
        my $cmd = '/opt/xcat/bin/verifynode -n xcat --notify';

        if ( $ivpRun{'type_of_run'} eq 'fullivp' ) {
            $cmd = "$cmd --fullivp --openstackip \'$ivpRun{'ip'}\' --openstackuser \'" . hexEncode( $ivpRun{'access_user'} ) . "\'";
        } else {
            $cmd = "$cmd --basicivp";
        }

        if ( $ivpRun{'orch_parms'} ne '' ) {
            $cmd = "$cmd $ivpRun{'orch_parms'}";    # Don't hex encode orchestrator parms
        }

        if ( $ivpRun{'prep_parms'} ne '' ) {
            $cmd = "$cmd --prepparms \'" . hexEncode( $ivpRun{'prep_parms'} ) . "\'";
        }

        if ( $ivpRun{'main_ivp_parms'} ne '' ) {
            $cmd = "$cmd --zxcatparms \'" . hexEncode( $ivpRun{'main_ivp_parms'} ) . "\'";
        }

        # Update the table to indicate the time we start the IVP run.
        if ( ! exists $ivpRun{'defaultRun'} ) {
            my $chtabCmd = "/opt/xcat/sbin/chtab id=\'$ivpRun{'id'}\' zvmivp.last_run=\'$newLastRun\'";
            $out = `$chtabCmd`;
            $rc = $?;
            if ( $rc != 0 ) {
                logResponse( 'DS', '*NONFORMATTED*', "\'$chtabCmd\' failed for IVP ID($ivpRun{'id'}), ip: $ivpRun{'ip'}, rc: $rc, out: $out\n" );
            }
        } elsif ( $ivpRun{'defaultRun'} ne 'noTableUpdate' ) {
            logResponse( 'DS', '*NONFORMATTED*', "Adding IVP ID($ivpRun{'id'}) to the z/VM IVP table: $ivpRun{'comments'}\n" );
            my $chtabCmd = "/opt/xcat/sbin/chtab id=\'$ivpRun{'id'}\' ".
                                                "zvmivp.ip=\'$ivpRun{'ip'}\' ".
                                                "zvmivp.schedule=\'$ivpRun{'schedule'}\' ".
                                                "zvmivp.last_run=\'$newLastRun\' ".
                                                "zvmivp.type_of_run=\'$ivpRun{'type_of_run'}\' ".
                                                "zvmivp.access_user=\'" . hexEncode( $ivpRun{'access_user'} ) . "\' ".
                                                "zvmivp.comments=\'" . hexEncode( $ivpRun{'comments'} ) . "\' ";
            $out = `$chtabCmd`;
            $rc = $?;
            if ( $rc != 0 ) {
                logResponse( 'DS', '*NONFORMATTED*', "\'$chtabCmd\' failed for ID($ivpRun{'id'}), ip: $ivpRun{'ip'}, rc: $rc, out: $out\n" );
            }

            # Skip running this one if it is marked 'runLater'.
            # A basic IVP is not needed if a full IVP is going to be run.
            if ( $ivpRun{'defaultRun'}  eq 'runLater' ) {
                logResponse( 'DS', '*NONFORMATTED*', "Ignoring IVP ID($ivpRun{'id'}) for this hour. It will be run next hour. IVP comment: $ivpRun{'comments'}\n" );
                next;
            }
        }

        # Drive the run.
        $attemptedRuns++;
        my $logComments = '';
        if ( $ivpRun{'comments'} eq '' ) {
            $logComments = "Running IVP ID($ivpRun{'id'})";
        } else {
            $logComments = "Running IVP ID($ivpRun{'id'}) - $ivpRun{'comments'}";
        }
        logResponse( 'DS', '*NONFORMATTED*',  "$logComments" );
        $out = `$cmd`;
        $rc = $?;
        if ( $rc != 0 ) {
            logResponse( 'DS', '*NONFORMATTED*', "The IVP run for ID($ivpRun{'id'}) returned a ".
                                                 "non-zero return code, rc: $rc\n" );
        }
    }

    logResponse( 'DFS', '*NONFORMATTED*', "IVP CRON results: $totalRunsInTable runs considered, $disabledRuns runs disabled, $attemptedRuns runs attempted" );
    $rc = 0;
    return $rc;
}


#-------------------------------------------------------

=head3   enableDisableRun

    Description : Enable or Disable command line function.
    Arguments   : ID of the IVP to be disabled
                  Global input set by command operands are used
                    by this routine:
                    $disable  - Disable a run.
                    $enable   - Enable a run.
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = enableDisableRun();

=cut

#-------------------------------------------------------
sub enableDisableRun {
    my ( $id ) = @_;
    my %ids;
    my $rc = 0;

    if ( $id eq '' ) {
        logResponse( 'DF', 'OPER01', '--id' );
        goto FINISH_enableDisableRun;
    }

    # Get the list of IVP ids in the zvmivp table.
    my $cmd = '/opt/xcat/sbin/tabdump zvmivp | grep -v "^#"';
    my $out = `$cmd`;
    $rc = $?;
    if ( $rc ne 0 ) {
        logResponse( 'DF', 'GNRL04', $cmd, $rc, $out );
        goto FINISH_enableDisableRun;
    }

    my @lines = split( '\n', $out );
    foreach my $line ( @lines ) {
        chomp( $line );
        my ($ivpId, $junk) = split( ',', $line, 2 );
        $ivpId =~ s/^\"|\"$//g;         # trim quotes from both ends of the string
        $ivpId =~ s/^\s+|\s+$//g;       # trim spaces from both ends of the string
        if ( $ivpId ne '' ) {
            $ids{$ivpId} = 1;
        }
    }

    # Validate the specified id.
    if ( ! exists $ids{$id} ) {
        logResponse( 'DF', 'ID03', $id );
    }

    my $action;
    my $disableFlag = '';
    if ( $disable ) {
        $disableFlag = 'YES';
        $action = 'disable';
    } else {
        $action = 'enable';
    }

    # Update the table.
    logResponse( 'DS', '*NONFORMATTED*', "Updating the z/VM IVP table (zvmivp) to $action $id" );
    my $chtabCmd = "/opt/xcat/sbin/chtab id='$id' zvmivp.disable=\'$disableFlag\'";
    $out = `$chtabCmd`;
    $rc = $?;
    if ( $rc != 0 ) {
        logResponse( 'DS', '*NONFORMATTED*', "\'$chtabCmd\' failed for ID($id), rc: $rc, out: $out\n" );
        $rc = 1;
    }

FINISH_enableDisableRun:
    return $rc;
}


#-------------------------------------------------------

=head3   encodeDecodeString

    Description : Encode and decode command line function.
    Arguments   : Global input set by command operands are used
                    by this routine:
                    $decode - String to be decoded if not ''.
                    $encode - String to be encoded if not ''.
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = encodeDecodeString();

=cut

#-------------------------------------------------------
sub encodeDecodeString {
    my $newVal = '';
    if ( $decode ne '' ) {
        binmode(STDOUT, ":utf8");
        print( "Value is: '" . hexDecode( $decode ) . "'\n" );
    }

    if ( $encode ne '' ) {
        print( "Value is: '" . hexEncode( $encode ) . "'\n" );
    }

    return 0;
}


#-------------------------------------------------------

=head3   finishLogFile

    Description : Complete the log file for the run and
                  send it to the notify user if necessary.
    Arguments   : None
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = finishLogFile();

=cut

#-------------------------------------------------------
sub finishLogFile {
    my $rc = 0;
    $needToFinishLogFile = 0;

    # Add the summary information to the log file.
    my ($sec,  $min,  $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
    $endTime = sprintf("%02d:%02d:%02d on %04d-%02d-%02d",
                           $hour, $min, $sec, $year + 1900, $mon + 1, $mday );

    logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );
    logResponse( 'DF', 'GENERIC_RESPONSE', 'Results of the IVP run:\n' );
    logResponse( 'DF', 'GENERIC_RESPONSE', "Run started at $startTime and ended at $endTime." );
    foreach my $line ( @ivpSummary ) {
        logResponse( 'DF', 'GENERIC_RESPONSE', $line );
    }
    if ( $ignoreCnt != 0 ){
        logResponse( 'DFS', 'GENERIC_RESPONSE', "The orchestrator script (verifynode) ignored messages $ignoreCnt times." );
        my @ignoreArray = sort keys %ignored;
        my $ignoreList = join ( ', ', @ignoreArray );
        logResponse( 'DFS', 'GENERIC_RESPONSE', "Message Ids of ignored messages by verifynode: $ignoreList" );
    }
    logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );

    # Notify the z/VM userid, if necessary.
    if ( $notify and $notifyOnErrorOrWarning ) {
        notifyUser();
    }

FINISH_finishLogFile:
    return $rc;
}


#-------------------------------------------------------

=head3   getDistro

    Description : Create the dummy image file.
    Arguments   : Node name or IP address
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = getDistro();

=cut

#-------------------------------------------------------
sub getDistro {
    my ( $nodeOrIP ) = @_;
    my $rc = 0;

    $tgtOS = xCAT::zvmUtils->getOsVersion( $::SUDOER, $nodeOrIP );

FINISH_getDistro:
    return $rc;
}


#-------------------------------------------------------

=head3   getLocalIPs

    Description : Get the IP addresses from ifconfig.
    Arguments   : Node name or IP address
    Returns     : return code (Currently always 0)
                    0 - No error
                    non-zero - Terminating error detected.
                  Hash of local IP addresses
    Example     : $rc = getLocalIPs();

=cut

#-------------------------------------------------------
sub getLocalIPs {
    my $ip;
    my $junk;
    my $rc = 0;

    my $out = `/sbin/ip addr | grep -e '^\\s*inet' -e '^\\s*inet6'`;
    my @lines = split( '\n', $out );
    foreach my $line ( @lines ) {
        my @parts = split( ' ', $line );
        ($ip) = split( '/', $parts[1], 2 );
        $localIPs{$ip} = 1;
    }

FINISH_getLocalIPs:
    return $rc;
}


#-------------------------------------------------------

=head3   getMNInfo

    Description : Get information related to the xCAT MN
                  and the z/VM system where it is running.
    Arguments   : None
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = getMNInfo();

=cut

#-------------------------------------------------------
sub getMNInfo {
    my $cmd;
    my $hostNode;
    my @lines;
    my $masterIP = '';
    my $out = '';
    my @parts;
    my $rc = 0;
    my $xcatMN;

    # Get the list of Local IP addresses
    getLocalIPs();

    # Wait up to 10 minutes for xcatd to come up.
    my $sleepTime = 60;
    my $maxWaits = 10;
    for ( my $i=1; $i<=$maxWaits; $i++ ) {
        # lsdef uses the xCAT daemon.  We use that to see if it is started.
        `/opt/xcat/bin/lsdef`;
        if ( $? == 0 ) {
            last;
        } else {
            logResponse( 'DFS', 'WAIT01', $sleepTime, $i, $maxWaits );
            sleep( $sleepTime );
        }
    }

    # Get information related to all of the host nodes
    $cmd = '/opt/xcat/bin/lsdef -w mgt=zvm -w hosttype=zvm -i hcp';
    $out = `$cmd`;
    $rc = $?;
    if ( $rc == 0 ) {
        @lines = split( '\n', $out );
        my $host = '';
        foreach my $line ( @lines ) {
            $line =~ s/^\s+|\s+$//g;         # trim blanks from both ends of the string
            if ( $line =~ /Object name: / ) {
                ($host) = $line =~ m/Object name: (.*)/;
                next;
            } elsif ( $line =~ /hcp\=/ ) {
                ($hosts{$host}{'hcp'}) = $line  =~ m/hcp\=(.*)/;
                next;
            }
        }
    } else {
        $rc = logResponse( 'DFS', 'GNRL01', "$cmd", $rc, $out );
        $notify = 1;
    }

    # Get key info related to the xCAT MN and the notify targets from the site table.
    $mnInfo{'PruneIVP'} = 7;
    $cmd = "/opt/xcat/sbin/tabdump site";
    $out = `$cmd`;
    $rc = $?;
    if ( $rc == 0 ) {
        @lines = split( '\n', $out );
        @lines = grep( /^\"master\"|^\"zvmnotify\"|^\"zvmpruneivp\"/, @lines );
        foreach my $line ( @lines ) {
            $line =~ s/^\s+|\s+$//g;         # trim blanks from both ends of the string
            @parts = split( ',', $line );
            $parts[1] =~ s/\"//g;
            if ( $parts[0] =~ /^\"master\"/ ) {
                $masterIP = $parts[1];
            } elsif ( $parts[0] =~ /^\"zvmnotify\"/ ) {
                my @notifyInfo = split( ';', $parts[1] );
                my $notifyHost = '';
                my $notifyUser = '';
                foreach my $hostUser ( @notifyInfo ) {
                    ($notifyHost) = $hostUser =~ m/(.*)\(/;
                    ($notifyUser) = $hostUser =~ m/\((.*)\)/;
                    if ( defined $notifyHost and $notifyHost ne '' and defined $notifyUser and $notifyUser ne '' ) {
                        $hosts{$notifyHost}{'notifyUser'} = uc( $notifyUser );
                    } else {
                        $rc = logResponse( 'DFS', 'SITE01', 'zvmnotify', $parts[1], $hostUser );
                        $notify = 1;
                    }
                }
            } elsif ( $parts[0] =~ /^\"zvmpruneivp\"/ ) {
                $mnInfo{'PruneIVP'} = $parts[1];
            }
        }
    } else {
        # Unable to read the site table
        $rc = logResponse( 'DFS', 'GNRL01', "$cmd", $rc, $out );
        $notify = 1;
    }

    # Find a node that has the same address as the master IP and then find
    # the host node that relates to that node.
    if ( $masterIP eq '' ) {
        # Unable to get the master value
        $rc = logResponse( 'DFS', 'SITE02', 'master' );
    } else {
        $cmd = "/opt/xcat/bin/lsdef -w ip=$masterIP -i hcp";
        $out = `$cmd`;
        $rc = $?;
        if ( $rc == 0 ) {
            @lines = split( '\n', $out );
            foreach my $line ( @lines ) {
                $line =~ s/^\s+|\s+$//g;         # trim blanks from both ends of the string
                if ( $line =~ /Object name: / ) {
                    ($xcatMN) = $line =~ m/Object name: (.*)/;
                    $mnInfo{'node'} = $xcatMN;
                    $mnInfo{'ip'} = $masterIP;
                    next;
                } elsif ( $line =~ /hcp\=/ ) {
                    ($mnInfo{'hcp'}) = $line  =~ m/hcp\=(.*)/;
                    next;
                }
            }
        } else {
            $rc = logResponse( 'DFS', 'MN01', $masterIP, $cmd, $rc, $out );
        }

        # Find the host node which uses the hcp.
        foreach $hostNode ( keys %hosts ) {
            if ( exists $hosts{$hostNode}{'hcp'} and $hosts{$hostNode}{'hcp'} eq $mnInfo{'hcp'} ) {
                if ( exists $hosts{$hostNode}{'notify'} ) {
                    $hosts{$hostNode}{'notify'} = $hosts{$hostNode}{'notify'} . 'm';
                } else {
                    $hosts{$hostNode}{'notify'} = 'm';
                }
                $mnInfo{'host'} = $hostNode;
            }
        }
    }

    # The following is useful in a z/VM CMA system only.
    # As a safety measure, in case the site table is lost or does not have zvmnotify property,
    # read the OPNCLOUD application system role file to find the value of the 'notify' property
    # and remember it with the xCAT MN information.
    # Also, determine the system role.
    if ( -e $locApplSystemRole ) {
        $out = `cat /var/lib/sspmod/appliance_system_role | grep -e '^notify\=' -e '^role\='`;
        @lines = split( '\n', $out );
        foreach my $line ( @lines ) {
            $line =~ s/^\s+|\s+$//g;         # trim blanks from both ends of the string
            my ($key, $value) = split( '=', $line, 2 );
            #($mnInfo{'notifyUser'}) = $line  =~ m/^notify\=(.*)/;
            if ( $key eq 'role' ) {
                $mnInfo{'role'} = lc( $value );
            }
            if ( $key eq 'notify' ) {
                $mnInfo{'notifyUser'} = uc( $value );
            }
        }
    } else {
        # Not CMA.  Indicate role is not set.
        $mnInfo{'role'} = '';
    }
}


#-------------------------------------------------------

=head3   getOpenStackLevel

    Description : Get the OpenStack version level as a
                  name.
    Arguments   : IP address of OpenStack compute node.
    Returns     : string - OpenStack version name
                    (e.g. NEWTON), or
                  null string - error
    Example     : $level = getOpenStackLevel();

=cut

#-------------------------------------------------------
sub getOpenStackLevel {
    my ( $nodeOrIP, $openstackUser ) = @_;
    my $cmd;
    my $level = '';
    my $numWords = 0;
    my $out = '';
    my @parts;
    my $rc;
    my %openStackVersion;

    # Get the list of known versions
    $cmd = 'cat /opt/xcat/openstack.versions';
    $out = `$cmd`;
    $rc = $?;
    if ( $rc != 0 ) {
        logResponse( 'DFS', 'GNRL01', $cmd, $rc, $out );
        $level = '';
        goto FINISH_getOpenStackLevel;
    }

    my @versionLines = split ( /\n/, $out );
    foreach my $versionLine ( @versionLines ) {
        if ( $versionLine =~ /^#/ ) { next; }
        @parts = split( ' ', $versionLine );
        $openStackVersion{$parts[0]} = $parts[1];
    }

    # Get the version information from the OpenStack node
    $cmd = "nova-manage --version 2>&1";
    ($rc, $out) = sshToNode( $openstackIP, $openstackUser, $cmd );
    if ( $rc != 0 ) {
        # SSH failed, message already sent.
        goto FINISH_getOpenStackLevel;
    }
    $out =~ s/^\s+|\s+$//g;         # trim blanks from both ends of the string
    ++$numWords while $out =~ /\S+/g;
    if ( $numWords != 1 ) {
        logResponse( 'DFS', 'GOSL01', $cmd, $openstackIP, $rc, $out );
        goto FINISH_getOpenStackLevel;
    }

    @parts = split( '\.', $out );
    if ( !exists $parts[0] or !exists $parts[1] ) {
        logResponse( 'DFS', 'GOSL01', $cmd, $openstackIP, $rc, $out );
        goto FINISH_getOpenStackLevel;
    } elsif ( exists $openStackVersion{"$parts[0]."} ) {
        $level = $openStackVersion{"$parts[0]."}
    } elsif ( exists $openStackVersion{"$parts[0].$parts[1]."} ) {
        $level = $openStackVersion{"$parts[0].$parts[1]."}
    } else {
        logResponse( 'DFS', 'GOSL02', $cmd, $openstackIP, $rc, $out );
        goto FINISH_getOpenStackLevel;
    }

FINISH_getOpenStackLevel:
    return $level;
}


#-------------------------------------------------------

=head3   hexDecode

    Description : Convert a string of printable hex
                  characters (4 hex characters per actual
                  character) into the actual string that
                  it represents.  The string should
                  begin with 'HexEncoded:' which indicates
                  that it is encoded.  The routine tolerates
                  getting called with a string that is not
                  not encoded and will return the string that
                  was passed to it instead of trying to decode
                  it.
    Arguments   : printable hex value
    Returns     : Perl string
    Example     : $rc = hexDecode();

=cut

#-------------------------------------------------------
sub hexDecode {
    my ( $hexVal ) = @_;
    my $result = '';

    if ( $hexVal =~ /^HexEncoded:/ ) {
        ($hexVal) = $hexVal =~ m/HexEncoded:(.*)/;
        my @hexes = unpack( "(a4)*", $hexVal);
        for ( my $i = 0; $i < scalar(@hexes); $i++ ) {
            $result .= chr( hex( $hexes[$i] ) );
        }
    } else {
        $result = $hexVal;
    }

    return $result;
}


#-------------------------------------------------------

=head3   hexEncode

    Description : Convert a string into a string of
                  printable hex characters in which each
                  character in the original string is
                  represented by the 2 byte (4 char) hex
                  code value.  The string is preceded by
                  'HexEncoded:' to indicate that it has
                  been encoded.  The routine will tolerate
                  getting called with a string that has
                  already been encoded and not reencode it.
    Arguments   : ASCII or Unicode string
    Returns     : Hex string
    Example     : $rc = hexEncode();

=cut

#-------------------------------------------------------
sub hexEncode {
    my ( $str ) = @_;
    my $hex;
    my $result = $str;   # All work done within the result variable.

    if ( $result ne '' ) {
        # Encode the string if is not already encoded.  Otherwise, leave it alone.
        if ( $result !~ /^HexEncoded:/ ) {
            $result =~ s/(.)/sprintf("%04x",ord($1))/eg;
            $result = "HexEncoded:$result";
        }
    }

    return $result;
}


#-------------------------------------------------------

=head3   logResponse

    Description : Build and log the response.
    Arguments   : Target destination for the response.  This is
                    string which can contain a charater for each
                    possible destination, e.g 'DSF'. The values are:
                      D - send to STDOUT
                      S - send to syslog
                      F - send to the log file
                    The default is 'DF' so that we send the output
                    to STDOUT and put it in the log file.
                  message ID or special flag:
                    *NONFORMATTED* indicates that the message build should not
                      be invoked but instead the message substitutions are
                      lines for the message to be produced.
                    *RESET* indicates that the message counter should be reset.
                  Array of message substitutions
    Returns     : 0 - No error, general response or info message detected.
                  1 - Non-terminating message detected.
                  2 - Terminating message detected.
    Example     : $rc = logResponse( 'D', 'VX01' );
                  $rc = logResponse( '', 'VX03', $nodeName, $sub2);
                  $rc = logResponse( 'DFS', 'VX03', $nodeName, 'sub2a');

=cut

#-------------------------------------------------------
sub logResponse {
    my ( $dest, $msgId, @msgSubs ) = @_;
    my $rc = 0;
    my $extraInfo = '';
    my $msg = '';
    my @msgLines;
    my $sev;
    my $line;

    if ( $msgId eq '*RESET*' ) {
        $warnErrCnt = 0;
        goto FINISH_logResponse;
    } elsif ( $msgId eq '*NONFORMATTED*' ) {
        $rc = 0;
        if ( @msgSubs ) {
            foreach my $line ( @msgSubs ) {
                $msg = "$msg$line\n";
            }
        } else {
            $msg = '\n';
        }
        $sev = 0;
    } else {
        ( $rc, $sev, $msg, $extraInfo ) = xCAT::zvmMsgs->buildMsg('VERIFYNODE', $msgId, \@msgSubs);
        if ( defined $msgsToIgnore{$msgId} ) {
            # Ignore this message id
            $ignored{$msgId} = 1;
            $ignoreCnt += 1;
            print( "Message $msgId is being ignored but would have occurred here.\n" );
            goto FINISH_logResponse;
        } elsif ( defined $msgsToIgnore{$sev} ) {
            # Ignoring all messages of this severity.
            $ignored{$msgId} = 1;
            $ignoreCnt += 1;
            print( "Message $msgId is being ignored but would have occurred here.\n" );
            goto FINISH_logResponse;
        }
    }

    if ( $sev >= 4 ) {
        $warnErrCnt += 1;
    }

    # Send the message to the requested destination.
    if ( $dest =~ 'D' ) {
        print "$msg";                        # Send message to STDOUT
    }
    if ( $dest =~ 'F' and defined $logFileHandle ) {
        print $logFileHandle $msg;
    }
    if ( $dest =~ 'S' ) {
        my $logMsg = $msg;
        $logMsg =~ s/\t//g;
        $logMsg =~ s/\n/ /g;
        syslog( 'err', $logMsg );
    }

    # Send the extra info to the requested destination (never send it to syslog).
    if ( $extraInfo ne '' ) {
        if ( $dest =~ 'D' ) {
            print "$extraInfo";
        }
        if ( $dest =~ 'F' and defined $logFileHandle ) {
            print $logFileHandle $msg;
        }
    }

FINISH_logResponse:
    return $rc;
}


#-------------------------------------------------------

=head3   notifyUser

    Description : Notify a z/VM user.  Send a message
                  and the log as a SPOOL file.
    Arguments   : Node name or IP address
    Returns     : 0 - Always ok.
    Example     : $rc = notifyUser();

=cut

#-------------------------------------------------------
sub notifyUser {
    my $cmd;
    my $host;
    my @hostOrder;
    my $msg = '';
    my $out = '';
    my $rc = 0;
    my $tempFile;
    my $th;

    # Determine who we should notify based on the host related to the verification.
    # The driver script would have set the host related to an full IVP (if we can trust it).
    # If we cannot get to the host related to the run then we fall back to sending
    # the notification to the user on the host in which the xCAT MN is running.
    # If that fails then "tough luck, Chuck".
    my $mnAdded = 0;
    foreach $host ( keys %hosts ) {
        if ( exists $hosts{$host}{'notify'} and exists $hosts{$host}{'hcp'} and exists $hosts{$host}{'notifyUser'} ) {
            if ( $hosts{$host}{'notify'} =~ /d/ ) {
                # Driver script specified hosts go on the top of the stack.
                push @hostOrder, $host;
                if ( $hosts{$host}{'notify'} =~ /m/ ) {
                    # The host related to the MN is already in the list
                    $mnAdded = 1;
                }
            } elsif ( $hosts{$host}{'notify'} =~ /m/ ) {
                # Management node systems go on the bottom as a fall back plan.
                unshift @hostOrder, $host;
                $mnAdded = 1;
            }
        }
    }
    if ( $mnAdded == 0 and exists $mnInfo{'hcp'} and exists $mnInfo{'notifyUser'} ) {
        # Did not find a host related to this management node but we have the
        # necessary information to send the notification so add a dummy host
        # to the hosts hash and add the dummy host to the end of the hostOrder
        # stack to be used when all other notification attemps fail.
        $host = '*DEFAULT_MN_HOST*';
        $hosts{$host}{'notify'} = 'm';
        $hosts{$host}{'notifyUser'} = $mnInfo{'notifyUser'};
        $hosts{$host}{'hcp'} = $mnInfo{'hcp'};
        unshift @hostOrder, $host;
    }

    # Prepare the message.
    $msg = "The xCAT IVP detected some possible issues. A log file is being sent to your reader.";

    # Prepare the log file by removing tabs and splitting the log file at 80 character lines.
    $tempFile = mktemp( "$logDir/$logFile.XXXXXX" );
    `expand $logDir/$logFile | fold -w 80 1>$tempFile`;

    # Send the message and log file to the first entity in the host order list that lets us
    # successfully do so.
    my $done = 0;
    my $failed = 0;
    foreach $host ( @hostOrder ) {
        # Check if zHCP's punch is online and online it if it is not.
        ($rc, $out) = sshToNode( $hosts{$host}{'hcp'}, '', 'cat /sys/bus/ccw/drivers/vmur/0.0.000d/online' );
        chomp( $out );
        if ($rc != 0 or $out != 1) {
            $cmd = '/sbin/cio_ignore -r 000d;/sbin/chccwdev -e 000d';
            ($rc, $out) = sshToNode( $hosts{$host}{'hcp'}, '', $cmd );
            chomp( $out );
            if ( $rc != 0 or !( $out =~ m/Done$/i ) ) {
                $rc = logResponse( 'DFS', 'GNRL03', $hosts{$host}{'hcp'},  $cmd, $rc, $out );
                next;
            }
            ($rc, $out) = sshToNode( $hosts{$host}{'hcp'}, '', 'which udevadm &> /dev/null && udevadm settle || udevsettle' );
            # Don't worry about the udevadm settle.  If it remains a problem then we will see it on the next command to ZHCP.
        }

        # Send log file.  It will exist temporarily on the ZHCP agent in the /tmp directory
        # under its original log file name.
        $rc = xCAT::zvmUtils->sendFile( 'root', $hosts{$host}{'hcp'}, "$tempFile", "/tmp/$logFile" );
        if ( $rc != 0 ) {
            # An error is not a problem because the zhcp node could be logged off.
            next;
        }

        # Punch the file from the ZHCP agent to the target user.
        $out = xCAT::zvmCPUtils->punch2Reader( 'root', $hosts{$host}{'hcp'}, $hosts{$host}{'notifyUser'}, "/tmp/$logFile", 'XCAT_IVP.RESULTS', '-t', 'A' );
        if ( $out ne 'Done' ) {
            $rc = logResponse( 'DFS', 'GNRL02', "/tmp/$logFile", $hosts{$host}{'hcp'}, $hosts{$host}{'notifyUser'}, $host, "$logDir/$logFile", $out );
            $failed = 1;
        }

        # Clean up the ZHCP node.
        ($rc, $out) = sshToNode( $hosts{$host}{'hcp'}, '', "rm -f /tmp/$logFile" );
        if ( $rc != 0 ) {
            $rc = logResponse( 'DFS', 'GNRL03', $hosts{$host}{'hcp'},  "rm -f /tmp/$logFile", $rc, $out );
            next;
        }

        # Try another host if we failed to send the file.
        if ( $failed ) {
            $failed = 0;
            next;
        }

        # Send a message if the user is logged on the system.
        ($rc, $out) = sshToNode( $hosts{$host}{'hcp'}, '', "vmcp query user $hosts{$host}{'notifyUser'}" );
        if ( $rc != 0 ) {
            # User is either not logged on or we had a problem issuing the command.  Not important, leave.
            last;
        }
        $out = uc( $out );
        if ( $out =~ /^$hosts{$host}{'notifyUser'}/ ) {
            my ($userStatus, $junk) = split '\s', $out, 2;
            chomp( $userStatus );
            $userStatus =~ s/^\s+|\s+$//g;       # trim both ends of the string
            if ( $userStatus eq 'SSI' or $userStatus eq 'DSC' ) {
                logResponse( 'DFS', 'MSG01', $hosts{$host}{'notifyUser'}, $host, $userStatus );
                last;
            }
        }

        # Don't worry if message still does not work. It is only a message.
        ($rc, $out) = sshToNode( $hosts{$host}{'hcp'}, '', "vmcp message $hosts{$host}{'notifyUser'} $msg" );
        last;
    }

    # Remove the temporary log file from the xCAT MN system.
    $out = `rm -f $tempFile`;
    $rc = $?;
    if ( $rc != 0 ) {
        logResponse( 'DFS', 'GNRL01', "rm -f $tempFile", $rc, $out );
    }

    $rc = 0;
    return $rc;
}


#-------------------------------------------------------


=head3   pruneLogs

    Description : Compress and prune log files. The number of days to wait
                  before pruning is specified in the site table with the
                  zvmpruneivp property.  If the property does not exist
                  or Management node information does not exist because of
                  some reason then the default is to prune logs that are
                  7 days or older.
    Arguments   : Current year
                  Current month (numeric)
                  Current day (Julian day, numeric)
    Returns     : None.
    Example     : pruneLogs( $year, $mon + 1, $mday );

=cut

#-------------------------------------------------------
sub pruneLogs{
    my ( $year, $month, $day ) = @_;

    my @lines;
    my $pruneOffset = 0;
    my $out;
    my $pruneDate = '';
    my $rc;
    my $removed = 0;
    my $zipOut;
    my $zipped = 0;
    logResponse( 'DFS', '*NONFORMATTED*', "CRON job is pruning log files." );

    # Determine the prune date value.
    if ( ! exists $mnInfo{'PruneIVP'} or $mnInfo{'PruneIVP'} eq '' ) {
        $pruneOffset = 7;
    } else {
        if ( looks_like_number( $mnInfo{'PruneIVP'} ) ) {
            $pruneOffset = $mnInfo{'PruneIVP'};
        } else {
            $pruneOffset = 7;
            logResponse( 'DFS', 'TP01', 'zvmpruneivp', 'site', 'not numeric', $mnInfo{'PruneIVP'}, $pruneOffset );
        }
    }
    $pruneDate = strftime "%Y-%m-%d", 0, 0, 0, $day - $pruneOffset, $month-1, $year-1900;
    logResponse( 'DFS', '*NONFORMATTED*', "Prune date: $pruneDate." );

    # Zip any log files that are not for today.  Current log files are similar to
    # /var/log/xcat/ivp/IVP_XCAT_2016-09-19_01:01:03.log and will have .gz appended to the name
    # after they have been zipped.
    $out = `ls -1 /var/log/xcat/ivp/*.log`;
    $rc = $?;
    if ( $rc == 0 ) {
        @lines = split( '\n', $out );
        foreach my $fqFile ( @lines ) {
            chomp( $fqFile );
            my ($file) = $fqFile =~ /\/var\/log\/xcat\/ivp\/(.*)/;
            my @parts = split( '_', $file );
            if ( ! defined $parts[0] or $parts[0] ne 'IVP' or
                 ! defined $parts[1] or ! defined $parts[2] or
                 $parts[2] eq $todayDate ) {
                next;
            }
            #logResponse( 'DFS', 'GENERIC_RESPONSE',  "ZIPPING: $file\n" );
            $zipOut = `gzip $fqFile 2>&1`;
            $rc = $?;
            if ( $rc != 0 ) {
                logResponse( 'DF', 'GENERIC_RESPONSE', "Unable to gzip $fqFile, rc: $rc, out: $out" );
            } else {
                $zipped++;
            }
        }
    }

    # Prune any gzipped files that are too old.
    $out = `ls -1 /var/log/xcat/ivp/*.log.gz`;
    $rc = $?;
    if ( $rc == 0 ) {
        @lines = split( '\n', $out );
        foreach my $fqFile ( @lines ) {
            chomp( $fqFile );
            my ($file) = $fqFile =~ /\/var\/log\/xcat\/ivp\/(.*)/;
            my @parts = split( '_', $file );
            if ( ! defined $parts[0] or $parts[0] ne 'IVP' or
                 ! defined $parts[1] or ! defined $parts[2] or
                 $parts[2] ge $todayDate ) {
                next;
            }
            if ( $parts[2] ge $pruneDate ) {
                next;
            }

            #logResponse( 'DFS', 'GENERIC_RESPONSE',  "REMOVING: $file\n" );
            $zipOut = `rm $fqFile 2>&1`;
            $rc = $?;
            if ( $rc != 0 ) {
                logResponse( 'DF', 'GENERIC_RESPONSE', "Unable to remove \'rm $fqFile\', rc: $rc, out: $out" );
            } else {
                $removed++;
            }
        }
    }

    logResponse( 'DFS', '*NONFORMATTED*', "IVP log pruning completed, zipped: $zipped, pruned: $removed." );
    return;
}


#-------------------------------------------------------

=head3   removeIVP

    Description : Remove an automated IVP.
    Arguments   : ID of the IVP to be removed
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = removeIVP();

=cut

#-------------------------------------------------------
sub removeIVP {
    my ( $id ) = @_;
    my $cmd;
    my $out;
    my $rc;
    my $ivpId;
    my %ids;
    my $junk;

    # Get the list of IVP ids in the zvmivp table.
    $cmd = '/opt/xcat/sbin/tabdump zvmivp | grep -v "^#"';
    $out = `$cmd`;
    $rc = $?;
    if ( $rc ne 0 ) {
        logResponse( 'DF', 'GNRL04', $cmd, $rc, $out );
        goto FINISH_removeIVP;
    }
    my @lines = split( '\n', $out );
    foreach my $line ( @lines ) {
        chomp( $line );
        my ($ivpId, $junk) = split( ',', $line, 2 );
        $ivpId =~ s/^\"|\"$//g;         # trim quotes from both ends of the string
        $ivpId =~ s/^\s+|\s+$//g;       # trim spaces from both ends of the string
        if ( $ivpId ne '' ) {
            $ids{$ivpId} = 1;
        }
    }

    # Validate the specified id.
    if ( $id eq '' or ! exists $ids{$id} ) {
        logResponse( 'DF', 'ID03', $id );
    }

    # Update the table.
    logResponse( 'DS', '*NONFORMATTED*', "Updating the z/VM IVP table (zvmivp) to remove $id" );
    my $chtabCmd = "/opt/xcat/sbin/chtab -d id=\'$id\' zvmivp";
    $out = `$chtabCmd`;
    $rc = $?;
    if ( $rc != 0 ) {
        logResponse( 'DS', '*NONFORMATTED*', "\'$chtabCmd\' failed for ID($id), rc: $rc, out: $out\n" );
        $rc = 1;
    }

FINISH_removeIVP:
    return $rc;
}


#-------------------------------------------------------

=head3   runDriverScript

    Description : Run a downloaded driver script.
    Arguments   : This routine is driven from the action
                    table, verifySets.  It takes global
                    input that can be set as command
                    input or by precursor routines.
                    The input variables include:
                    $driver - Driver script name
                    $driverLoc - Location of the driver script
                    $hosts - Infor for defined host nodes
                    $infoCnt - Count of warnings that were generated
                    $warningCnt - Count of warnings that were generated
                    $zxcatParms - Command line parms to
                      pass to zxcatIVP
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = runDriverScript();

=cut

#-------------------------------------------------------
sub runDriverScript{
    my $infoCnt = 0;
    my @lines;
    my $out;
    my $rc = 0;
    my $warningCnt = 0;

    # Analyze the driver script to determine the z/VM host that it intends to support.
    my $hostNodeZhcp = '';
    my $hostNode = `cat $driverLoc/$driver | grep "^export zxcatIVP_hostNode\=" | sed '/^export zxcatIVP_hostNode\=*/!d; s///; s/\"//g;q'`;
    chomp( $hostNode );

    if ( $hostNode ne '' ) {
        if ( exists $hosts{$hostNode}{'hcp'} ) {
            if ( exists $hosts{$hostNode}{'notify'} ) {
                $hosts{$hostNode}{'notify'} = $hosts{$hostNode}{'notify'} . 'd';
            } else {
                $hosts{$hostNode}{'notify'} = 'd';
            }
            $zhcpTarget = $hosts{$hostNode}{'hcp'};
            $zvmHost = $hostNode;
            push @ivpSummary, "The ZHCP related to the host, \'$hostNode\', identified in the driver script has been identified ".
                "as \'$zhcpTarget\'.";
        } else {
            $rc = logResponse( 'DFS', 'DRIV03', $zvmHost );
            $notify = 1;
        }
    } else {
        # Driver script did not specify the host node.
        $rc = logResponse( 'DFS', 'DRIV02', "$driverLoc/$driver", 'zxcatIVP_hostNode' );
        $notify = 1;
    }

    # Analyze the driver script to determine the ZHCP node that it intends to support.
    my $zhcpNode = `cat $driverLoc/$driver | grep "^export zxcatIVP_zhcpNode\=" | sed '/^export zxcatIVP_zhcpNode\=*/!d; s///; s/\"//g;q'`;
    chomp( $zhcpNode );
    my $zhcpHcp = `/opt/xcat/bin/lsdef $zhcpNode | grep "hcp\=" | sed '/hcp\=*/!d; s///; s/\"//g;q'`;
    if ( $zhcpHcp ne '' ) {
        $zhcpHcp =~ s/^\s+|\s+$//g;       # trim both ends of the string
        $zhcpTarget = $zhcpHcp;
        foreach $hostNode ( keys %hosts ) {
            if ( exists $hosts{$hostNode}{'hcp'} and $hosts{$hostNode}{'hcp'} eq $zhcpHcp ) {
                if ( exists $hosts{$hostNode}{'notify'} ) {
                    $hosts{$hostNode}{'notify'} = $hosts{$hostNode}{'notify'} . 'd';
                } else {
                    $hosts{$hostNode}{'notify'} = 'd';
                }
            }
        }
    }

    my $exportCmd = '';
    if ( $zxcatParms ne '' ) {
        $ENV{zxcatIVP_moreCmdOps}=$zxcatParms;
    } else {
        delete $ENV{zxcatIVP_moreCmdOps};
    }

    logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );
    logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "Using the driver script to drive the IVP. The output from the run follows." );
    logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );
    $out = `chmod +x $driverLoc/$driver`;
    $out = `$driverLoc/$driver`;
    $rc = $?;
    if ( $rc != 0 ) {
        $rc = logResponse( 'DFS', 'DRIV01', "$driverLoc/$driver", $rc, $out );
        $notify = 1;
        goto FINISH_runDriverScript;
    }

    # Determine how many warnings and information messages were generated so that we can
    # produce a summary message upon completion of the IVP.
    @lines = split( '\n', $out );
    foreach my $line ( @lines ) {
        if ( $line =~ /^Warning \(/ ) { $warningCnt++; }
        if ( $line =~ /^Info \(/ ) { $infoCnt++; }
    }
    push @ivpSummary, "The driver script generated $warningCnt warnings and $infoCnt information messages.";
    logResponse( 'DF', '*NONFORMATTED*', $out );

FINISH_runDriverScript:
    if ( $warningCnt != 0 ) {
        $notify = 1;
    }
    return $rc;
}


#-------------------------------------------------------

=head3   runPrepScript

    Description : Push the appropriate level of preparation
                  script to a compute node and run it to
                  validate the system and build the driver
                  script.
    Arguments   : Global defaults set as command line parameters provide input
                    to this routine:
                    $driver - Name of driver script
                    $driverLoc - Location of the driver script
                    $infoCnt - Count of warnings that were generated
                    $openstackIP - IP for OpenStack compute node
                    $openstackUser - User for OpenStack compute node
                    $prepParms - Parms for preparation script
                    $warnErrCnt - Count of warnings and errors that were generated
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = runPrepScript();

=cut

#-------------------------------------------------------
sub runPrepScript {
    my $cmd = '';
    my $infoCnt = 0;
    my $line;
    my @lines;
    my $out = '';
    my $rc = 0;
    my $retRC = 0;
    my $tmpDir = '';
    my $warningCnt = 0;

    # Determine the name of the driver script based on the IP address for the compute node.
    $driver = "zxcatIVPDriver_$openstackIP.sh";

    # Create the local IVP directory.
    if ( !-d $driverLoc ) {
        $out = `mkdir -m a=rwx,g=rx,o= -p $driverLoc`;
        $rc = $?;
        if ( $rc != 0 ) {
            $rc = logResponse( 'DFS', 'PREP01', $driverLoc, $rc, $out );
            $notify = 1;
            goto FINISH_runPrepScript;
        }
    }

    # Determine the OpenStack level.
    my $level = getOpenStackLevel( $openstackIP, $openstackUser );
    if ( $level eq '' ) {
        $notify = 1;
        $rc = logResponse( 'DFS', 'PREP06', $openstackIP );
        goto FINISH_runPrepScript;
    }
    my $levelLetter = substr( $level, 0, 1);
    push @ivpSummary, "OpenStack system at $openstackIP is running the Nova $level release.";

    # Determine the appropriate preparation script to send to the compute node.
    my $prepDir = '/opt/xcat/share/xcat/tools/zvm';
    my $prepScript = "prep_zxcatIVP_$level.pl";
    logResponse( 'DF', 'GENERIC_RESPONSE', "Attempting to push $prepScript to the OpenStack system ".
                       "at $openstackIP. It will be used to validate the OpenStack environment and create ".
                       "a driver script to be used for the validation on the xCAT management node." );
    my $runPrepScript = 1;

    # Create a directory on the compute node to hold the preparation script and driver script.
    $cmd = 'mktemp -d /tmp/xCAT.XXXXXXXXXX';
    ($rc, $tmpDir) = sshToNode( $openstackIP, $openstackUser, $cmd );
    if ( $rc != 0 ) {
        # Unable to create the directory.  Let's see if we have an old driver script that we can use.
        $rc = logResponse( 'DFS', 'PREP02', $openstackIP, "$driverLoc/$driver", $cmd, $rc, $out );
        $notify = 1;
        goto FINISH_runPrepScript;
    } else {
        # Push the preparation script to the compute node and run it.
        chomp($tmpDir);
        $cmd = "/usr/bin/scp $prepDir/$prepScript $openstackUser\@$openstackIP:$tmpDir";
        $out = `/usr/bin/scp "$prepDir/$prepScript" "$openstackUser\@$openstackIP:$tmpDir"`;
        $rc = $?;
        if ( $rc != 0 ) {
            # Unable to push the preparation script.  Let's see if we have an old driver script that we can use.
            $rc = logResponse( 'DFS', 'PREP02', $openstackIP, "$driverLoc/$driver", $cmd, $rc, $out );
            $notify = 1;
            goto FINISH_runPrepScript;
        }
    }

    if ( $runPrepScript ) {
        # Run the preparation script.
        ($rc, $out) = sshToNode( $openstackIP, $openstackUser, "chmod +x $tmpDir/$prepScript");
        ($rc, $out) = sshToNode( $openstackIP, $openstackUser, "$tmpDir/$prepScript '--driver' $tmpDir/$driver $prepParms" );
        logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );
        logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "The output from the preparation script that is run on the compute node follows. ".
                     "The preparation script validates the z/VM related OpenStack configuration file properties." );
        logResponse( 'DF', '*NONFORMATTED*', '*******************************************************************************' );
        logResponse( 'DF', '*NONFORMATTED*', $out );

        # Determine how many warnings were generated so that we can produce a summary message upon completion of the IVP.
        if ( $levelLetter gt 'L' ) {
            @lines = split( '\n', $out );
            foreach $line ( @lines ) {
                if ( $line =~ /^Warning \(/ ) { $warningCnt++; }
                if ( $line =~ /^Info \(/ ) { $infoCnt++; }
            }
        } else {
            @lines = split( '\n', $out );
            foreach $line ( @lines ) {
                if ( $line =~ /^Warning:/ ) { $warningCnt++; }
                if ( $line =~ /^Info:/ ) { $infoCnt++; }
            }
        }
        push @ivpSummary, "The preparation script generated $warningCnt warnings and $infoCnt information messages.";

        # Pull back the driver script.
        if ( -e "$driverLoc/$driver" ) {
            $out = `mv -f $driverLoc/$driver $driverLoc/$driver.old`;
            $rc = $?;
            if ( $rc != 0 ) {
                logResponse( 'S', 'GENERIC_RESPONSE', "Unable to move $driverLoc/$driver to $driverLoc/$driver.old, rc: $rc" );
                $notify = 1;
            }
        }

        $cmd = "/usr/bin/scp $openstackUser\@$openstackIP:$tmpDir/$driver $driverLoc/$driver";
        $out = `/usr/bin/scp "$openstackUser\@$openstackIP:$tmpDir/$driver" "$driverLoc/$driver"`;
        $rc = $?;
        if ( $rc != 0 ) {
            $rc = logResponse( 'DFS', 'PREP04', "$driverLoc/$driver", $openstackIP, $cmd, $rc, $out );
            $notify = 1;
            goto FINISH_runPrepScript;
        }

        $cmd = "chmod 660 $driverLoc/$driver 2>&1";
        $out = `$cmd`;
        $rc = $?;
        if ( $rc != 0 ) {
            logResponse( 'FS', 'PREP07', "$driverLoc/$driver", $cmd, $rc, $out );
            $notify = 1;
        }
    }

FINISH_runPrepScript:
    if ( !-s "$driverLoc/$driver"  && -s "$driverLoc/$driver.old" ) {
        # New driver does not exist but old driver exists, use the old one.
        $out = `mv -f $driverLoc/$driver.old $driverLoc/$driver`;
        $rc = $?;
        if ( $rc != 0 ) {
            logResponse( 'S', 'GENERIC_RESPONSE', "Unable to move $driverLoc/$driver.old to $driverLoc/$driver, rc: $rc" );
        }
    }

    if ( -s "$driverLoc/$driver" ) {
        $retRC = 0;
        # Add the driver script to the log file.
        $out = `cat $driverLoc/$driver`;
        logResponse( 'F', '*NONFORMATTED*', '*******************************************************************************' );
        logResponse( 'F', 'GENERIC_RESPONSE_NOINDENT', "The contents of the driver script used for the rest of this IVP follows. ".
                     "The driver script is used to run the rest of the IVP on the xCAT Management Node." );
        logResponse( 'F', 'GENERIC_RESPONSE_NOINDENT', 'Note: Any line which set a password related property has been removed.' );
        logResponse( 'F', '*NONFORMATTED*', '*******************************************************************************' );
        $out = `echo "$out" | grep -v "^export zxcatIVP_xcatUserPw"`;
        logResponse( 'F', '*NONFORMATTED*', $out );
    }

    if ( $warningCnt != 0 ) {
        $notify = 1;
    }

    if ( $tmpDir ne '' ) {
        $cmd = "rm -Rf $tmpDir 2>&1";
        ($rc, $out) = sshToNode( $openstackIP, $openstackUser, "$cmd");
        $rc = $?;
        if ( $rc != 0 ) {
            $rc = logResponse( 'DFS', 'CLNUP01', $openstackIP, $tmpDir, $cmd, $rc, $out );
            $notify = 1;
        }
    }
    return $retRC;
}


#-------------------------------------------------------

=head3   scheduleIVP

    Description : Schedule an automated IVP.
    Arguments   : Global defaults set as command line parameters provide input
                    to this routine:
                    $comments - Comments for IVP
                    $disable - Disable option
                    $id - ID to update or 'NEW'
                    $openstackIP - IP for OpenStack compute node
                    $openstackUser - User for OpenStack compute node
                    $orchParms - Parms for this script when IVP run
                    $prepParms - Parms for preparation script
                    $schedule - Hours to schedule
                    $scheduledType - Type of IVP
                    $zxcatParms - Parms for zxcatIVP script
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = scheduleIVP();

=cut

#-------------------------------------------------------
sub scheduleIVP {
    my $cmd;
    my $out;
    my $rc;
    my $ivpId;
    my %ids;
    my $junk;

    # Get the list of IVP ids in the zvmivp table.
    $cmd = '/opt/xcat/sbin/tabdump zvmivp | grep -v "^#"';
    $out = `$cmd`;
    $rc = $?;
    if ( $rc ne 0 ) {
        logResponse( 'DF', 'GNRL04', $cmd, $rc, $out );
        goto FINISH_scheduleIVP;
    }
    my @lines = split( '\n', $out );
    foreach my $line ( @lines ) {
        chomp( $line );
        my ($ivpId, $junk) = split( ',', $line, 2 );
        $ivpId =~ s/^\"|\"$//g;         # trim quotes from both ends of the string
        $ivpId =~ s/^\s+|\s+$//g;       # trim spaces from both ends of the string
        if ( $ivpId ne '' ) {
            $ids{$ivpId} = 1;
        }
    }

    # Validate the specified id.  Either generate a new ID if this is a new
    # request (specified as 'new' or omitted or use the id that was passed.
    if ( $id eq '' or $id eq 'NEW' ) {
        # Generate the new Id
        my $i;
        for ( $i = 10; $i < 10010; $i++ ) {
            if ( ! exists $ids{$i} ) {
                last;
            }
        }
        if ( $i <= 10010 ) {
            $id = $i;
        } else {
            # Could not find an available number in the first 10000 IDs.
            logResponse( 'DF', 'ID01' );
            goto FINISH_scheduleIVP;
        }
    } else {
        # Validate the Id
        if ( ! exists $ids{$id} ) {
            logResponse( 'DF', 'ID02', $id );
        }
    }

    if ( $scheduledType eq '' ) {
        logResponse( 'DF', 'OPER01', '--type' );
        goto FINISH_scheduleIVP;
    }
    $scheduledType = lc( $scheduledType );
    if ( $scheduledType ne 'basicivp' and $scheduledType ne 'fullivp' ) {
        logResponse( 'DF', 'OPER02', '--type', $scheduledType, '\'basicivp\' or \'fullivp\'' );
        goto FINISH_scheduleIVP;
    }
    if ( $scheduledType eq 'basicivp' ) {
        # Ignore any FULLIVP parms
        $openstackIP = '';
        $openstackUser = '';
        $prepParms = '';
    }

    # Normalize the schedule.
    my %hours;
    my @parts = split( ' ', $schedule );
    foreach my $hour ( @parts ) {
        $hour =~ s/^\s+|\s+$//g;         # trim spaces from both ends of the string
        $hours{$hour} = 1;
    }
    $schedule = '';
    for ( my $hour = 0; $hour <= 23; $hour++ ) {
        if ( exists $hours{$hour} ) {
            $schedule = "$schedule $hour";
            delete $hours{$hour};
        }
    }
    $schedule =~ s/^\s+|\s+$//g;         # trim spaces from both ends of the string
    my (@leftOver) = keys %hours;
    if ( scalar @leftOver != 0 ) {
        my $badValues = join( '\' and \'', @leftOver );
        logResponse( 'DF', 'OPER02', '--schedule', $badValues, '0-23' );
        goto FINISH_scheduleIVP;
    }

    if ( $comments =~ /,/ ) {
        $comments =~ s/,//g;         # trim commas from the string
    }

    my $disableFlag = '';
    if ( $disable == 1 ) {
        $disableFlag = 'YES';
    }

    # Update the table.
    logResponse( 'DS', '*NONFORMATTED*', "Updating the z/VM IVP table (zvmivp) for $id" );
    my $chtabCmd = "/opt/xcat/sbin/chtab id=\'$id\' ".
                   "zvmivp.ip=\'$openstackIP\' ".
                   "zvmivp.schedule=\'$schedule\' ".
                   "zvmivp.last_run=\'\' ".
                   "zvmivp.type_of_run=\'$scheduledType\' ".
                   "zvmivp.access_user=\'" . hexEncode( $openstackUser ) . "\' ".
                   "zvmivp.orch_parms=\'" . hexEncode( $orchParms ) . "\' ".
                   "zvmivp.prep_parms=\'" . hexEncode( $prepParms ) . "\' ".
                   "zvmivp.main_ivp_parms=\'" . hexEncode( $zxcatParms ) . "\' ".
                   "zvmivp.comments=\'" . hexEncode( $comments ) . "\' ".
                   "zvmivp.disable=\'$disableFlag\'";
    $out = `$chtabCmd`;
    $rc = $?;
    if ( $rc != 0 ) {
        logResponse( 'DS', '*NONFORMATTED*', "\'$chtabCmd\' failed for ID($id), rc: $rc, out: $out\n" );
        $rc = 1;
    }

FINISH_scheduleIVP:
    return $rc;
}

#-------------------------------------------------------

=head3   setupLogFile

    Description : Set up the log file for this run.
    Arguments   : None
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = setupLogFile();

=cut

#-------------------------------------------------------
sub setupLogFile {
    my $out;
    my $rc = 0;
    my $ipString = '';

    # Indicate that the log file will need to be finished.
    # This is a safety in case something goes wrong during
    # the IVP run and causes the program to break.
    $needToFinishLogFile = 1;

    # Determine the IP address to associate with the log file.
    if ( $openstackIP eq '' ) {
        # Must be doing either a generic zxcatIVP or a full
        # IVP for the compute node sharing the system with this
        # xCAT MN.  Use the string "XCAT".
        $ipString = 'XCAT';
    } else {
        if ( exists $localIPs{$openstackIP} ) {
            $ipString = 'XCAT';
        } else {
            $ipString = $openstackIP;
        }
    }

    # Determine the timestamp to use for the log.
    my ($sec,  $min,  $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
    $startTime = sprintf("%02d:%02d:%02d on %04d-%02d-%02d",
                           $hour, $min, $sec, $year + 1900, $mon + 1, $mday );
    my $currTime = sprintf("%04d-%02d-%02d_%02d:%02d:%02d",
                           $year + 1900, $mon + 1, $mday, $hour, $min, $sec);

    # If the log directory does not exist then create it.
    if ( !-d $logDir ) {
        # Directory did not exist.  Create it.
        $out = `mkdir -p $logDir`;
        $rc = $?;
        if ( $rc != 0 ) {
            logResponse( 'DFS', 'GENERIC_RESPONSE', 'Failed to set up the log directory.  IVP will run without a log file.' );
            $rc = 0;     # Continue processing.
            goto FINISH_setupLogFile;
        }
    }

    # Create the log file.
    $logFile = "IVP\_$ipString\_$currTime\.log";
    open $logFileHandle, ">", "$logDir/$logFile";
    $rc = $?;
    if ( $rc != 0 ) {
        logResponse( 'DFS', 'GENERIC_RESPONSE', "Failed to open the log file: $logFile.  IVP will run without a log file. $!" );
        $rc = 0;     # Continue processing.
        goto FINISH_setupLogFile;
    }
    $out = `chmod 644 "$logDir/$logFile"`;

    push @ivpSummary, "The following logfile was created for this run: $logFile";

FINISH_setupLogFile:
    return $rc;
}


#-------------------------------------------------------

=head3   showCILevel

    Description : Show the cloud-init version installed
                  in the node.
    Arguments   : None
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = showCILevel();

=cut

#-------------------------------------------------------
sub showCILevel {
    my $ciLevel = '';
    my $out = '';
    my $rc = 0;

    # Attempted to obtain the version using the routine
    # accessible through the $PATH variable.
    ($rc, $out) = sshToNode( $nodeName, '', 'cloud-init --version 2>&1');
    if ( $rc == 255 ) {
        logResponse( 'DF', 'GENERIC_RESPONSE', $out );
        goto FINISH_showCILevel;
    } elsif ( $rc == 0 ) {
        if ( $out =~ /^cloud-init / ) {
            ($ciLevel) = $out =~ /^cloud-init (.*)/;
            my $msgText = "Version of cloud-init: $ciLevel";
            logResponse( 'DF', 'GENERIC_RESPONSE', $msgText );
            # <todo> Add code to compare the version to what is available in xcat.
            goto FINISH_showCILevel;
        }
    }

    # Attempt to locate the version in the various cloud-init scripts
    # in the target node.
    # <todo> Add support to look for versions on the node if the invocation
    # using the path did not show the version.

FINISH_showCILevel:
    return $rc;
}


#-------------------------------------------------------

=head3   showHelp

    Description : Show the help inforamtion.
    Arguments   : None.
    Returns     : None.
    Example     : showHelp();

=cut

#-------------------------------------------------------
sub showHelp{
    print "$0 verify the capabilities of a node.\n\n";
    print $usage_string;
    return;
}


#-------------------------------------------------------

=head3   sshToNode

    Description : SSH to a node and issue a command.
                  Check for SSH errors.
    Arguments   : Node name or IP address/DNS name
                  user if we need to specify SUDO
                  Command to issue
                  Options (char string):
                    q - Quiet, Don't generate an error message on failure
    Returns     : Return code:
                      0 - Normal Linux success
                      255 - Unable to SSH to system
                      non-zero - command error
                  Output from the command or a error string on an SSH failure.
    Example     : ($rc, $out) = sshToNode( $nodeName, $user, $command );

=cut

#-------------------------------------------------------
sub sshToNode{
    my ( $nodeName, $user, $cmd, $options ) = @_;
    my $rc = 0;
    my $out = '';

    if ( ! defined $options ) {
        $options = '';
    }

    if ( $user eq '' ) {
        $out = `ssh $nodeName -qoBatchMode=yes '$cmd'`;
        $rc = $? >> 8;
    } else {
        $out = `ssh $user\@$nodeName -qoBatchMode=yes '$cmd'`;
        $rc = $? >> 8;
    }

    if ( $rc == 255 and $options !~ /q/ ) {
        logResponse( 'DFS', 'VSTN01', $nodeName );  # Keep $rc = 255.
        $out = "Unable to SSH to $nodeName";
    }

    return ($rc, $out);
}


#-------------------------------------------------------

=head3   verifyAccess

    Description : Verify the xCAT MN can access a system.
    Arguments   : Node name or IP address
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = verifyAccess();

=cut

#-------------------------------------------------------
sub verifyAccess{
    my ( $nodeOrIP ) = @_;

    my $rc = 0;
    my $out = '';

    ($rc, $out) = sshToNode( $nodeOrIP, '', 'pwd 2>/dev/null' );
    goto FINISH_verifyAccess if ( $rc == 255 );    # Already generated a message
    if ( $rc != 0 ) {
        $rc = logResponse( 'DFS', 'VA01' );
    }

FINISH_verifyAccess:
    return $rc;
}


#-------------------------------------------------------

=head3   verifyBasicXCAT

    Description : Verify the basic setup of xCAT.
    Arguments   : None
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = verifyBasicXCAT();

=cut

#-------------------------------------------------------
sub verifyBasicXCAT{
    my $infoCnt = 0;
    my @lines;
    my $rc = 0;
    my $warningCnt = 0;

    $rc = logResponse( 'DF', 'GENERIC_RESPONSE', 'The IVP will be invoked with BYPASS messages suppressed. '.
        'This is because a basic IVP run will produce a number of BYPASS messages due to the fact that '.
        'it is not driven with a full set of configuration operands causing it to avoid some tests. '.
        'Instead you will see an information line indicating that the message was ignored.' );
    $ENV{'zxcatIVP_bypassMsg'} = 0;   # No bypass messages
    $ENV{'zxcatIVP_moreCmdOps'} = $zxcatParms;
    my $out = `/opt/xcat/bin/zxcatIVP.pl`;

    $rc = logResponse( 'DF', '*NONFORMATTED*', $out );
    # Determine how many warnings were generated so that we can produce a summary message upon completion of the IVP.
    @lines = split( '\n', $out );
    foreach my $line ( @lines ) {
        if ( $line =~ /^Warning/ ) { $warningCnt++; }
        if ( $line =~ /^Info/ ) { $infoCnt++; }
    }
    push @ivpSummary, "The zxcatIVP.pl script generated $warningCnt warnings and $infoCnt information messages.";

FINISH_verifyBasicXCAT:
    if ( $warningCnt != 0 ) {
        $notify = 1;
    }
    return $rc;
}


#-------------------------------------------------------

=head3   verifyDistro

    Description : Verify that the distro is supported.
    Arguments   : None
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = verifyDistro();

=cut

#-------------------------------------------------------
sub verifyDistro{
    my $rc = 0;

    my $supported = xCAT::zvmUtils->isOSVerSupported( $tgtOS );
    if ( !$supported ) {
        $rc = logResponse( 'DF', 'VD01', ($tgtOS) );
    }

    return $rc;
}


#-------------------------------------------------------

=head3   verifyLogResponse

    Description : Script test function to test functioning
                  of the logResponse() routine.
    Arguments   : None
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = verifyLogResponse();

=cut

#-------------------------------------------------------
sub verifyLogResponse{
    my $rc = 0;
    my $sub2 = 'sub2';

    $rc = logResponse( 'DF', 'VX01' );
    print "rc: $rc\n";
    logResponse( 'DF', 'VPF01', ($nodeName, 0, 'result message') );
    $rc = logResponse( 'DF', 'VX03', $nodeName, $sub2);
    print "rc: $rc\n";
    $rc = logResponse( 'DF', 'VX03', $nodeName, 'sub2a');
    print "rc: $rc\n";

    return 0;
}


#-------------------------------------------------------

=head3   verifyPower

    Description : Verify the node is powered on.
    Arguments   : None
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = verifyPower();

=cut

#-------------------------------------------------------
sub verifyPower {
    my ( $nodeName ) = @_;
    my $rc = 0;
    my $powerState;

    my $out = `/opt/xcat/bin/rpower $nodeName stat 2>&1`;
    $rc = $?;
    if ( $rc != 0 ) {
        $rc = logResponse( 'DF', 'VPF01', ($nodeName, $rc, $out) );
        goto FINISH_verifyPower if ( $rc > 0 );
    }

    $out = lc( $out );
    if ( $out =~ /$nodeName: / ) {
        chomp $out;
        ($powerState) = $out =~ /$nodeName: (.*)/;
    }

    if ( $powerState ne 'on' ) {
        $rc = logResponse( 'DFS', 'VP02', ($nodeName) );
    }

FINISH_verifyPower:
    return $rc;
}


#-------------------------------------------------------

=head3   verifyService

    Description : Verify that specified services are running.
    Arguments   : Node name or IP address
                  Blank delimitted list of service names
                    service name begins with the name and
                    has a comma separated list of run levels
                    which are to be verified as 'on'
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = verifyService();

=cut

#-------------------------------------------------------
sub verifyService {
    my ( $nodeOrIP, $namesAndLevels ) = @_;
    my @serviceList = split ( / /, $namesAndLevels );

    my $rc = 0;
    my $serviceOut = '';

    # Get the list of service names and run levels into an array.

    # Get the list of configured services.  The output looks similar to:
    # service_name    0:off   1:off   2:off   3:off   4:off   5:off   6:off
    ($rc, $serviceOut) = sshToNode( $nodeOrIP, '', "chkconfig --list 2>&1" );
    goto FINISH_verifyService if ( $rc == 255 );

    # Look for missing services
    my @missingService;
    my @foundService;
    my @nonrunningService;
    foreach my $serviceInfo ( @serviceList ) {
        my @offLevels;
        my $levels;
        my @serviceRunLevels = split( /,/, $serviceInfo );
        my $service = shift @serviceRunLevels;
        ($levels) = $serviceOut =~ /^$service(.*)\s(.*)\n/m;
        $levels =~ s/^\s+|\s+$//g if ( defined $levels );
        if (( defined $levels ) and ( $levels ne '' )) {
            # Verify the run levels are enabled.
            foreach my $level ( @serviceRunLevels ) {
                if ( $levels !~ /$level:on/ ) {
                    push @offLevels, $level;
                }
            }
            if ( @offLevels ) {
                $levels = join(", ", @offLevels);
                $rc = logResponse( 'DF', 'VS06', $service, $levels );
                goto FINISH_verifyService if ( $rc > 0 );
            } else {
                push @foundService, $service;
            }
        } else {
            push @nonrunningService, $service;
        }
    }

    if ( @foundService ) {
        my $list = join(", ", @foundService);
        $rc = logResponse( 'DF', 'GENERIC_RESPONSE', "The following services are configured to start: $list" );
        goto FINISH_verifyService if ( $rc > 0 );
    }
    if ( @nonrunningService ) {
        my $list = join(", ", @nonrunningService);
        $rc = logResponse( 'DF', 'VS05', $list );
        goto FINISH_verifyService if ( $rc > 0 );
    }

FINISH_verifyService:
    return $rc;
}


#-------------------------------------------------------

=head3   verifyXcatconf4z

    Description : Verify xcatconf4z is properly installed
                  and is the correct version.
    Arguments   : Node name or IP address
    Returns     : 0 - No error
                  non-zero - Terminating error detected.
    Example     : $rc = verifyXcatconf4z();

=cut

#-------------------------------------------------------
sub verifyXcatconf4z{
    my ( $nodeOrIP ) = @_;
    my $rc = 0;
    my $out;
    my ($mnVersion, $tgtVersion);

    # Verify service is installed
    $rc = verifyService( $nodeOrIP, 'xcatconf4z,2,3,5' );
    goto FINISH_verifyXcatconf4z if ( $rc == 255 );

    # Get the xCAT MN's xcatconf4z version level.
    $out = `/opt/xcat/share/xcat/scripts/xcatconf4z version`;
    if ( $out =~ /xcatconf4z version: / ) {
        chomp $out;
        ($mnVersion) = $out =~ /xcatconf4z version: (.*)/;
    } else {
        $rc = logResponse( 'DFS', 'VX01' );
        goto FINISH_verifyXcatconf4z if ( $rc > 0 );
    }

    # <todo> Verify that the node contains xcatconf4z in the correct location and is executable.
    #$out = `ssh $nodeOrIP 'ls -al /opt/xcatconf4z 2>&1'`;
    ($rc, $out) = sshToNode( $nodeOrIP, '', 'ls -al /opt/xcatconf4z 2>&1');
    goto FINISH_verifyXcatconf4z if ( $rc == 255 );
    if ( $out =~ /No such file or directory/ ) {
        $rc = logResponse( 'DF', 'VX06' );
        goto FINISH_verifyXcatconf4z if ( $rc > 0 );
    } else {
        # <todo> Verify that it is executable
    }

    # Get the node's xcatconf4z version level.
    #$out = `ssh $nodeOrIP '/opt/xcatconf4z version'`;
    ($rc, $out) = sshToNode( $nodeOrIP, '', '/opt/xcatconf4z version');
    goto FINISH_verifyXcatconf4z if ( $rc == 255 );
    if ( $out =~ /xcatconf4z version: / ) {
        chomp $out;
        ($tgtVersion) = $out =~ /xcatconf4z version: (.*)/;
    } else {
        $rc = logResponse( 'DF', 'VX02' );
        goto FINISH_verifyXcatconf4z if ( $rc > 0 );
    }

    # Verify the version is up to date.
    if ( defined $tgtVersion ) {
        if ( $mnVersion > $tgtVersion ) {
            $rc = logResponse( 'DF', 'VX03', ($tgtVersion, $mnVersion) );
            goto FINISH_verifyXcatconf4z if ( $rc > 0 );
        }
    }

    # Verify that authorized senders is setup.
    #$out = `ssh $nodeOrIP '/opt/xcatconf4z status'`;
    ($rc, $out) = sshToNode( $nodeOrIP, '', '/opt/xcatconf4z status');
    goto verifyXcatconf4z if ( $rc == 255 );
    if ( $out =~ /xcatconf4z is disabled / ) {
        $rc = logResponse( 'DF', 'VX04' );
        goto FINISH_verifyXcatconf4z if ( $rc > 0 );
    } else {
        # Get list of authorized users and put out an info message.
    }

    # Verify that mkisofs is available
    ($rc, $out) = sshToNode( $nodeOrIP, '', 'stat /opt/bin/mkisofs');
    goto verifyXcatconf4z if ( $rc == 255 );
    if ( $rc != 0 ) {
        $rc = logResponse( 'DF', 'VX07' );
        goto FINISH_verifyXcatconf4z if ( $rc > 0 );
    }

FINISH_verifyXcatconf4z:
    return $rc;
}


#*****************************************************************************
# Main routine
#*****************************************************************************
my $ignoreOpt;
my $rc = 0;
my $thisScript = $0;
my $out;

# Parse the arguments
$Getopt::Long::ignorecase = 0;
Getopt::Long::Configure( "bundling" );
if (!GetOptions(
         'a|access'        => \$verifyAccess,
         'basicivp'        => \$runBasicIVP,
         'capturereqs'     => \$verifyCaptureReqs,
         'c|cloudinit'     => \$verifyCloudInit,
         'comments=s'      => \$comments,
         'cron'            => \$runCron,
         'decode=s'        => \$decode,
         'disable'         => \$disable,
         'enable'          => \$enable,
         'encode=s'        => \$encode,
         'file=s'          => \$dataFile,
         'fullivp'         => \$runFullIVP,
         'h|help'          => \$displayHelp,
         'i|id=s'          => \$id,
         'ignore=s'        => \$ignoreOpt,
         'cmdoveriucv=s'   => \$issueCmdOverIUCV,
         'cmdtonode=s'     => \$issueCmdToNode,
         'n|node=s'        => \$nodeName,
         'notify'          => \$notifyOnErrorOrWarning,
         'openstackuser=s' => \$openstackUser,
         'openstackip=s'   => \$openstackIP,
         'orchparms=s'     => \$orchParms,
         'prepparms=s'     => \$prepParms,
         'remove'          => \$remove,
         'schedule=s'      => \$schedule,
         'type=s'          => \$scheduledType,
         'v|version'       => \$versionOpt,
         'x|xcatconf4z'    => \$verifyXcatconf4z,
         'zxcatparms=s'    => \$zxcatParms,
      )) {
    print $usage_string;
}

if ( $versionOpt ) {
    logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "Version: $version\n" );
}

if ( $displayHelp ) {
    showHelp();
}

if ( $displayHelp or $versionOpt ) {
    goto FINISH_main;
}

# Convert any encoded operand values back to their unencoded value.
my @convertibleParms = ( 'comments', 'nodeName', 'openstackUser', 'orchParms', 'prepParms', 'zxcatParms' );
foreach my $parmName ( @convertibleParms ) {
    eval( "\$$parmName = hexDecode( \$$parmName )" );
}

if ( $schedule eq '' and $orchParms ne '' ) {
    # If not scheduling an IVP run and orchestrator parms are present then use them.
    my $unrecognizedOps = '';
    $rc = GetOptionsFromString(
        $orchParms,
        'ignore=s'        => \$ignoreOpt,
        );
    if ( $rc == 0 ) {
        print "Unrecognized option in --orchParms. ".
              "Only --ignore is allowed as an orchestrator option for an immediate run.\n";
        $rc = 2;
        goto FINISH_main;
    }
}

# Handle messages to ignore.
if ( defined( $ignoreOpt ) ) {
    # Make hash from the specified ignore operands
    $ignoreOpt = uc( $ignoreOpt );
    my @ingoreList;
    if ( $ignoreOpt =~ ',' ) {
        @ingoreList = split( ',', $ignoreOpt );
    } else {
        @ingoreList = split( ' ', $ignoreOpt );
    }

    %msgsToIgnore = map { $_ => 1 } @ingoreList;

    # Convert general severity type operands to their numeric value.
    if ( $msgsToIgnore{'BYPASS'} ) {
        delete $msgsToIgnore{'BYPASS'};
        $msgsToIgnore{'2'} = 1;
    }
    if ( $msgsToIgnore{'INFO'} ) {
        delete $msgsToIgnore{'INFO'};
        $msgsToIgnore{'3'} = 1;
    }
    if ( $msgsToIgnore{'WARNING'} ) {
        delete $msgsToIgnore{'WARNING'};
        $msgsToIgnore{'4'} = 1;
    }
    if ( $msgsToIgnore{'ERROR'} ) {
        delete $msgsToIgnore{'ERROR'};
        $msgsToIgnore{'5'} = 1;
    }
}

if ( $openstackIP ne '' ) {
    $openstackIP = uc( $openstackIP );
}

# Handle the combinations of disable and enable options.
if ( $disable eq '' ) {
    $disable = 0;
}
if ( $enable eq '' ) {
    $enable = 0;
}
if ( $disable == 1 and $enable == 1 ) {
    $rc = logResponse( 'DFS', 'OPER03', '--disable and --enable' );
    goto FINISH_main;
}

if ( $runFullIVP or $runBasicIVP ) {
    # Set up a log file for an IVP run.
    setupLogFile();
}

# Determine the information about the xCAT managed node and the z/VM
# host environment where it is running (if running on z/VM).
getMNInfo();

if ( $decode ne '' or
     $disable or
     $enable or
     $encode ne '' or
     $runFullIVP or
     $runCron or
     $runBasicIVP or
     $remove or
     $schedule ne '' ) {
    # IVP runs do not need the node.
} elsif ( $nodeName eq '' ) {
    $rc = logResponse( 'DFS', 'OPER01', '-n or --node' );
    goto FINISH_main;
}

($::SUDOER, $::SUDO) = xCAT::zvmUtils->getSudoer();

# Setup for a test run of logResponse.  Normally, commented out.
#($verifyAccess, $verifyCaptureReqs, $verifyCloudInit, $verifyXcatconf4z) = 0;
#my $test = 1;

# Run the selected tests
foreach my $verify (keys %verifySets) {
    if ( eval $verify ) {
        if ( $verifySets{$verify}[0] ne '' ) {
            logResponse( 'D', 'GENERIC_RESPONSE_NOINDENT', "\n********************************************************************\n" );
            logResponse( 'D', 'GENERIC_RESPONSE_NOINDENT', "$verifySets{$verify}[0]\n" );
            logResponse( 'D', 'GENERIC_RESPONSE_NOINDENT', "********************************************************************\n" );
        }
        my $size = @{$verifySets{$verify}};
        logResponse( 'D', '*RESET*' );
        for (my $i = 1; $i < $size; $i++) {
            ($rc) = eval $verifySets{$verify}[$i];
            if ( ! defined $rc ) {
                logResponse( 'DFS', 'PERL01', $thisScript, $verifySets{$verify}[$i], $@ );
                $rc = 666;
                last;
            }

            if ( $rc > 0 ) {
                last;
            }
        }

        if ( $warnErrCnt == 0 ) {
            logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "\nProcessing completed." );
        } else {
            logResponse( 'DF', 'GENERIC_RESPONSE_NOINDENT', "\nProcessing completed with warning or error messages. " .
                "See previous messages for information.\n" );
        }
    }
}

FINISH_main:
if ( $needToFinishLogFile ) {
    # A log file was started.  We need to complete it by adding the summary
    # section.  If necessary, we will send the log file to the notify user.
    finishLogFile();
}

exit $rc;
