# IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html

package xCAT::PPC;
use strict;
use lib "/opt/xcat/lib/perl";
use xCAT::Table;
use xCAT::Utils;
use xCAT::TableUtils;
use xCAT::ServiceNodeUtils;
use xCAT::SvrUtils;
use xCAT::FSPUtils;
use xCAT::Usage;
use POSIX "WNOHANG";
use Storable qw(freeze thaw);
use Time::HiRes qw(gettimeofday sleep);
use IO::Select;
use Socket;
use xCAT::PPCcli;
use xCAT::GlobalDef;
use xCAT::DBobjUtils;
use xCAT_monitoring::monitorctrl;
use Thread qw(yield);
use xCAT::PPCdb;

#use Data::Dumper;

##########################################
# Globals
##########################################
my %modules = (
    rinv => { hmc => "xCAT::PPCinv",
        ivm   => "xCAT::PPCinv",
        fsp   => "xCAT::FSPinv",
        bpa   => "xCAT::FSPinv",
        cec   => "xCAT::FSPinv",
        frame => "xCAT::FSPinv",
    },
    rpower => { hmc => "xCAT::PPCpower",
        ivm   => "xCAT::PPCpower",
        fsp   => "xCAT::FSPpower",
        bpa   => "xCAT::FSPpower",
        cec   => "xCAT::FSPpower",
        frame => "xCAT::FSPpower",
    },
    rvitals => { hmc => "xCAT::PPCvitals",
        ivm   => "xCAT::PPCvitals",
        fsp   => "xCAT::FSPvitals",
        bpa   => "xCAT::FSPvitals",
        cec   => "xCAT::FSPvitals",
        frame => "xCAT::FSPvitals",
    },
    rscan => { hmc => "xCAT::PPCscan",
        fsp => "xCAT::FSPscan",
        cec => "xCAT::FSPscan",
        ivm => "xCAT::PPCscan",
    },
    mkvm => { hmc => "xCAT::PPCvm",
        fsp => "xCAT::FSPvm",
        cec => "xCAT::FSPvm",
    },
    rmvm => { hmc => "xCAT::PPCvm",
        fsp => "xCAT::FSPvm",
    },
    lsvm => { hmc => "xCAT::PPCvm",
        ivm => "xCAT::PPCvm",
        fsp => "xCAT::FSPvm",
        cec => "xCAT::FSPvm",
    },
    chvm => { hmc => "xCAT::PPCvm",
        fsp => "xCAT::FSPvm",
        cec => "xCAT::FSPvm",
    },
    rnetboot => { hmc => "xCAT::PPCboot",
        ivm => "xCAT::PPCboot",
        fsp => "xCAT::FSPboot",
        cec => "xCAT::FSPboot",
    },
    getmacs => { hmc => "xCAT::PPCmac",
        ivm => "xCAT::PPCmac",
        fsp => "xCAT::FSPmac",
        cec => "xCAT::FSPmac",
    },
    reventlog => { hmc => "xCAT::PPClog",
    },
    rspconfig => { hmc => "xCAT::PPCcfg",
        fsp   => "xCAT::FSPcfg",
        bpa   => "xCAT::FSPcfg",
        cec   => "xCAT::FSPcfg",
        blade => "xCAT::FSPcfg",
        frame => "xCAT::FSPcfg",
    },
    rflash => { hmc => "xCAT::PPCrflash",
        fsp   => "xCAT::FSPflash",
        bpa   => "xCAT::FSPflash",
        cec   => "xCAT::FSPflash",
        frame => "xCAT::FSPflash",
    },
    mkhwconn => { hmc => "xCAT::PPCconn",
        fsp   => "xCAT::FSPconn",
        cec   => "xCAT::FSPconn",
        bpa   => "xCAT::FSPconn",
        frame => "xCAT::FSPconn",
    },
    rmhwconn => { hmc => "xCAT::PPCconn",
        fsp   => "xCAT::FSPconn",
        cec   => "xCAT::FSPconn",
        bpa   => "xCAT::FSPconn",
        frame => "xCAT::FSPconn",
    },
    lshwconn => { hmc => "xCAT::PPCconn",
        fsp   => "xCAT::FSPconn",
        cec   => "xCAT::FSPconn",
        bpa   => "xCAT::FSPconn",
        frame => "xCAT::FSPconn",
    },
    renergy => { hmc => "xCAT::PPCenergy",
        fsp => "xCAT::PPCenergy",
        cec => "xCAT::PPCenergy",
    },
    rbootseq => { fsp => "xCAT::FSPbootseq",
        cec => "xCAT::FSPbootseq",
    },
);


##########################################
# Database errors
##########################################
my %errmsg = (
    NODE_UNDEF => "Node not defined in '%s' database",
    NO_ATTR    => "'%s' not defined in '%s' database",
    NO_UNDEF   => "'%s' not defined in '%s' database for '%s'",
    DB_UNDEF   => "'%s' database not defined"
);

##########################################################################
# Invokes the callback with the specified message
##########################################################################
sub send_msg {

    my $request = shift;
    my $ecode   = shift;
    my %output;

    #################################################
    # Called from child process - send to parent
    #################################################
    if (exists($request->{pipe})) {
        my $out = $request->{pipe};

        $output{errorcode} = $ecode;
        $output{data}      = \@_;
        print $out freeze([ \%output ]);
        print $out "\nENDOFFREEZE6sK4ci\n";
    }
    #################################################
    # Called from parent - invoke callback directly
    #################################################
    elsif (exists($request->{callback})) {
        my $callback = $request->{callback};

        $output{errorcode} = $ecode;
        $output{data}      = \@_;
        $callback->(\%output);
    }
}


##########################################################################
# Fork child to execute remote commands
##########################################################################
sub process_command {

    my $request      = shift;
    my $hcps_will    = shift;
    my $failed_nodes = shift;
    my $failed_msg   = shift;
    my %nodes        = ();
    my $callback     = $request->{callback};

    #my $sitetab  = xCAT::Table->new( 'site' );
    my @site = qw(ppcmaxp ppctimeout maxssh ppcretry fsptimeout powerinterval syspowerinterval);
    my $start;
    my $verbose = $request->{verbose};

    #######################################
    # Default site table attributes
    #######################################
    $request->{ppcmaxp}          = 64;
    $request->{ppctimeout}       = 0;
    $request->{fsptimeout}       = 0;
    $request->{ppcretry}         = 3;
    $request->{maxssh}           = 8;
    $request->{powerinterval}    = 0;
    $request->{syspowerinterval} = 0;

    #######################################
    # Get site table attributes
    #######################################
    foreach (@site) {
        my @val = xCAT::TableUtils->get_site_attribute($_);
        if (defined($val[0])) {
            $request->{$_} = $val[0];
        }
    }
    if (exists($request->{verbose})) {
        $start = Time::HiRes::gettimeofday();
    }

    #######################################
    # Group nodes based on command
    #######################################
    my $nodes;
    unless ($request->{command} =~ /^rspconfig$/ && exists($request->{opt}->{resetnet})) {

        $nodes = preprocess_nodes($request, $hcps_will);
        if (!defined($nodes)) {
            return (1);
        }
    }

    #get new node status
    my %oldnodestatus = ();    #saves the old node status
    my @allerrornodes = ();
    my $check         = 0;
    my $global_check  = 1;
    my @val = xCAT::TableUtils->get_site_attribute("nodestatus");
    if (defined($val[0])) {
        if ($val[0] =~ /0|n|N/) { $global_check = 0; }
    }

    my $command = $request->{command};
    if (($command eq 'rpower') || ($command eq 'rnetboot')) {
        my $subcommand = "temp";
        if ($command eq 'rpower') { $subcommand = $request->{op}; }

        #pdu commands will be handled in the pdu plugin
        if(($subcommand eq 'pduoff') || ($subcommand eq 'pduon') || ($subcommand eq 'pdustat') || ($subcommand eq 'pdureset')){
             return 0;
        }

        if (($global_check) && ($subcommand ne 'stat') && ($subcommand ne 'status') && ($subcommand ne 'state')) {
            $check = 1;
            my $noderange = $request->{node};
            my @allnodes  = @$noderange;

            #save the old status
            my $nodelisttab = xCAT::Table->new('nodelist');
            if ($nodelisttab) {
                my $tabdata = $nodelisttab->getNodesAttribs(\@allnodes, [ 'node', 'status' ]);
                foreach my $node (@allnodes)
                {
                    my $tmp1 = $tabdata->{$node}->[0];
                    if ($tmp1) {
                        if ($tmp1->{status}) { $oldnodestatus{$node} = $tmp1->{status}; }
                        else                 { $oldnodestatus{$node} = ""; }
                    }
                }
            }

            #print "oldstatus:" . Dumper(\%oldnodestatus);
        }
    }



    #######################################
    # Fork process
    #######################################
    my $children = 0;
    my $fds      = new IO::Select;

    # For the commands getmacs and rnetboot, each time
    # to fork process, pick out the HMC that has the
    # least process number created to connect to the HMC.
    # After the process by preprocess_nodes, the $nodes
    # variable has following structure:
    # $nodes
    #   |hcp
    #       |[[hcp,node1_attr], [hcp,node2_attr] ...]
    #       |count    //node number managed by the hcp
    #       |runprocess    //the process number connect to the hcp
    #       |index    //the index of node will be forked of the hcp
    if (($request->{command} =~ /^(getmacs)$/ && exists($request->{opt}->{D})) || ($request->{command} =~ /^(rnetboot)$/) || ($request->{command} =~ /^(rbootseq)$/)) {
        my %pid_owner = ();

        $request->{maxssh} = int($request->{maxssh} / 2);

        # Use the CHID signal to control the
        #connection number of certain hcp
        $SIG{CHLD} = sub { my $pid = 0; while (($pid = waitpid(-1, WNOHANG)) > 0)
            { $nodes->{ $pid_owner{$pid} }{'runprocess'}--; delete $pid_owner{$pid}; $children--; } };

        $SIG{INT} = $SIG{TERM} = sub { #prepare to process job termination and propogate it down
            foreach (keys %pid_owner) {
                kill 9, $_;
            }
            exit 0;
        };

        my $hasnode = 1;
        while ($hasnode) {
            while ($children >= $request->{ppcmaxp}) {
                my $handlednodes = {};
                child_response($callback, $fds, $handlednodes, $failed_nodes, $failed_msg, $verbose);

                #update the node status to the nodelist.status table
                if ($check) {
                    updateNodeStatus($handlednodes, \@allerrornodes);
                }

                Time::HiRes::sleep(0.1);
            }

            # Pick out the hcp which has least processes
            my $least_processes = $request->{maxssh};
            my $least_hcp;
            my $got_one = 0;
            while (!$got_one) {
                $hasnode = 0;
                foreach my $hcp (keys %$nodes) {
                    if ($nodes->{$hcp}{'index'} < $nodes->{$hcp}{'count'}) {
                        $hasnode = 1;
                        if ($nodes->{$hcp}{'runprocess'} < $least_processes) {
                            $least_processes = $nodes->{$hcp}{'runprocess'};
                            $least_hcp       = $hcp;
                        }
                    }
                }

                if (!$hasnode) {

                    # There are no node in the $nodes
                    goto ENDOFFORK;
                }

                if ($least_processes < $request->{maxssh}) {
                    $got_one = 1;
                } else {
                    my $handlednodes = {};
                    child_response($callback, $fds, $handlednodes, $failed_nodes, $failed_msg, $verbose);

                    #update the node status to the nodelist.status table
                    if ($check) {
                        updateNodeStatus($handlednodes, \@allerrornodes);
                    }
                    Time::HiRes::sleep(0.1);
                }
            }

            Time::HiRes::sleep(0.1);
            my ($pipe, $pid) = fork_cmd($nodes->{$least_hcp}{'nodegroup'}->[ $nodes->{$least_hcp}{'index'} ]->[0],
                $nodes->{$least_hcp}{'nodegroup'}->[ $nodes->{$least_hcp}{'index'} ]->[1], $request);

            if ($pid) {
                $pid_owner{$pid} = $least_hcp;
                $nodes->{$least_hcp}{'index'}++;
                $nodes->{$least_hcp}{'runprocess'}++;
            }

            if ($pipe) {
                $fds->add($pipe);
                $children++;
            }
        }
    } elsif ($request->{command} =~ /^(getmacs)$/ && exists($request->{opt}->{arp})) {
        my $display = "";
        if (defined($request->{opt}->{d})) {
            $display = "yes";
        }
        my $output = xCAT::SvrUtils->get_mac_by_arp($nodes, $display);

        my $rsp = ();
        foreach my $node (keys %{$output}) {
            push @{ $rsp->{node} }, { name => [$node], data => [ $output->{$node} ] };
        }
        $rsp->{errorcode} = 0;
        $callback->($rsp);
    } elsif ($request->{command} =~ /^rpower$/) {
        my $hw;
        my $sessions;
        my $pid_owner;
        my $remain_node = $nodes;
        my $num         = 0;

        while (scalar($remain_node)) {
            $remain_node = ();
            foreach my $hash (@$nodes) {
                $SIG{CHLD} = sub { my $pid = 0; while (($pid = waitpid(-1, WNOHANG)) > 0) { $hw->{ $pid_owner->{$pid} }--; $children--; } };
                $SIG{INT} = $SIG{TERM} = $SIG{KILL} = sub { #prepare to process job termination and propogate it down
                    foreach my $pid (keys %{$pid_owner}) {
                        &kill_children_by_pid($pid);
                    }
                    exit 0;
                };

                while ($children >= $request->{ppcmaxp}) {
                    my $handlednodes = {};
                    child_response($callback, $fds, $handlednodes, $failed_nodes, $failed_msg, $verbose);

                    #update the node status to the nodelist.status table
                    if ($check) {
                        updateNodeStatus($handlednodes, \@allerrornodes);
                    }

                    Time::HiRes::sleep(0.1);
                }
                if ($hw->{ @$hash[0] } >= $request->{maxssh}) {
                    my $handlednodes = {};
                    child_response($callback, $fds, $handlednodes, $failed_nodes, $failed_msg, $verbose);

                    #update the node status to the nodelist.status table
                    if ($check) {
                        updateNodeStatus($handlednodes, \@allerrornodes);
                    }

                    Time::HiRes::sleep(0.1);
                    push(@$remain_node, [ @$hash[0], @$hash[1] ]);
                    next;
                }

                if ($num > 0 && $request->{op} =~ /^on/ && $request->{fsp_api} == 1) {
                    my $t_hash           = @$hash[1];
                    my $one_key_in_thash = (keys %$t_hash)[0];
                    my $one_d            = $t_hash->{$one_key_in_thash};

                    #print Dumper($one_d);
                    if ($$one_d[4] =~ /^fsp$/ || $$one_d[4] =~ /^cec$/) {
                        if ($request->{syspowerinterval} > 0) {
                            no_interrupt_sleep($request->{syspowerinterval});
                        }
                    }
                }

                my ($pipe, $pid) = fork_cmd(@$hash[0], @$hash[1], $request);

                $num++;

                if ($pid) {
                    $pid_owner->{$pid} = @$hash[0];
                    $hw->{ @$hash[0] }++;
                }

                if ($pipe) {
                    $fds->add($pipe);
                    $children++;
                }
            }

            $nodes = $remain_node;
        }
    } elsif ($request->{command} =~ /^rspconfig$/ && exists($request->{opt}->{resetnet})) {
        runcmd($request);
    } else {
        my %pid_owner = ();
        $SIG{CHLD} = sub { while (waitpid(-1, WNOHANG) > 0) { $children--; } };
        $SIG{INT} = $SIG{TERM} = $SIG{KILL} = sub { #prepare to process job termination and propogate it down
            foreach my $pid (keys %pid_owner) {
                &kill_children_by_pid($pid);
            }
            exit 0;
        };
        my $hw;
        my $sessions;

        foreach (@$nodes) {
            while ($children >= $request->{ppcmaxp}) {
                my $handlednodes = {};
                child_response($callback, $fds, $handlednodes, $failed_nodes, $failed_msg, $verbose);

                #update the node status to the nodelist.status table
                if ($check) {
                    updateNodeStatus($handlednodes, \@allerrornodes);
                }

                Time::HiRes::sleep(0.1);
            }
            ###################################
            # sleep between connects to same
            # HMC/IVM so as not to overwelm it
            ###################################
            if ($hw ne @$_[0]) {
                $sessions = 1;
            } elsif ($sessions++ >= $request->{maxssh}) {
                sleep(1);
                $sessions = 1;
            }
            $hw = @$_[0];

            my ($pipe, $pid) = fork_cmd(@$_[0], @$_[1], $request);
            if ($pipe) {
                $pid_owner{$pid} = $pid;
                $fds->add($pipe);
                $children++;
            }
        }
    }

  ENDOFFORK:
    #######################################
    # Process responses from children
    #######################################
    while ($fds->count > 0 or $children > 0) {
        my $handlednodes = {};
        child_response($callback, $fds, $handlednodes, $failed_nodes, $failed_msg, $verbose);

        #update the node status to the nodelist.status table
        if ($check) {
            updateNodeStatus($handlednodes, \@allerrornodes);
        }

        Time::HiRes::sleep(0.1);
    }

    #drain one more time
    my $rc = 1;
    while ($rc > 0) {
        my $handlednodes = {};
        $rc = child_response($callback, $fds, $handlednodes, $failed_nodes, $failed_msg, $verbose);

        #update the node status to the nodelist.status table
        if ($check) {
            updateNodeStatus($handlednodes, \@allerrornodes);
        }
    }

    if (exists($request->{verbose})) {
        my $elapsed = Time::HiRes::gettimeofday() - $start;
        my $msg = sprintf("Total Elapsed Time: %.3f sec\n", $elapsed);
        trace($request, $msg);
    }

    if ($check) {

        #print "allerrornodes=@allerrornodes\n";
        #revert the status back for there is no-op for the nodes
        my %old = ();
        foreach my $node (@allerrornodes) {
            my $stat = $oldnodestatus{$node};
            if (exists($old{$stat})) {
                my $pa = $old{$stat};
                push(@$pa, $node);
            }
            else {
                $old{$stat} = [$node];
            }
        }
        xCAT_monitoring::monitorctrl::setNodeStatusAttributes(\%old, 1);
    }


    return (0);
}

sub no_interrupt_sleep {
    my $sleep_time = shift;
    my $sleep_end  = time + $sleep_time;
    while (1) {
        my $sleep_duration = $sleep_end - time;
        last if $sleep_duration <= 0;
        Time::HiRes::sleep($sleep_duration);
    }
}

sub kill_children_by_pid {
    my $pid  = shift;
    my @pids = `ps -o pid,ppid -e`;
    for my $a_pid (@pids) {
        if ($a_pid =~ /\s*(\d*)\s*(\d*)/) {
            my $tmp_pid  = $1;
            my $tmp_ppid = $2;
            if ($tmp_ppid == $pid) {
                kill 9, $tmp_pid;
            }
        }
    }
}

##########################################################################
# updateNodeStatus
##########################################################################
sub updateNodeStatus {
    my $handlednodes  = shift;
    my $allerrornodes = shift;
    foreach my $node (keys(%$handlednodes)) {
        if ($handlednodes->{$node} == -1) { push(@$allerrornodes, $node); }
    }
}

##########################################################################
# Verbose mode (-V)
##########################################################################
sub trace {

    my $request = shift;
    my $msg     = shift;

    my ($sec, $min, $hour, $mday, $mon, $yr, $wday, $yday, $dst) = localtime(time);
    my $formatted = sprintf "%02d:%02d:%02d %5d %s", $hour, $min, $sec, $$, $msg;

    my $callback = $request->{callback};
    $callback->({ data => [$formatted] });
}


##########################################################################
# Send response from child process back to xCAT client
##########################################################################
sub child_response {

    my $callback     = shift;
    my $fds          = shift;
    my $errornodes   = shift;
    my $failed_nodes = shift;
    my $failed_msg   = shift;
    my $verbose      = shift;
    my @ready_fds    = $fds->can_read(1);
    my $rc           = @ready_fds;
    my $mkvm_cec;

    foreach my $rfh (@ready_fds) {
        my $data = <$rfh>;

        #################################
        # Read from child process
        #################################
        if (defined($data)) {
            while ($data !~ /ENDOFFREEZE6sK4ci/) {
                $data .= <$rfh>;
            }
            my $responses = thaw($data);
            my @nodes;
            foreach (@$responses) {
                my $node = $_->{node}->[0]->{name}->[0];
                if (!grep /^$node$/, @nodes) {
                    push(@nodes, $node);
                }
            }

            foreach (sort @nodes) {
                my $nodename = $_;
                foreach (@$responses) {
                    if ($nodename eq $_->{node}->[0]->{name}->[0]) {

                        # One special case for mkvm to output some messages
                        if ((exists($_->{errorcode})) && ($_->{errorcode} eq "mkvm")) {
                            $mkvm_cec = $nodename;
                            next;
                        }

                        #save the nodes that has errors for node status monitoring
                        if ((exists($_->{errorcode})) && ($_->{errorcode} != 0)) {

                            #if($$failed_nodes[0] !~ /^noloop$/) {
                            #if (!grep /^$nodename$/, @$failed_nodes) {
                            #     push(@$failed_nodes, $nodename);
                            #}
                            #if( defined( $failed_msg->{$nodename} )) {
                            #    my $f = $failed_msg->{$nodename};
                            #    my $c = scalar(@$f);
                            #    $failed_msg->{$nodename}->[$c] = $_;
                            #} else {
                            #    $failed_msg->{$nodename}->[0] = $_;
                            #}
                            #} else {
                            #$callback->( $_ );
                            #}

                            if ($errornodes) { $errornodes->{ $_->{node}->[0]->{name}->[0] } = -1; }

                            #if verbose, print all the message;
                            #if not, print successful mesg for success, or all the failed mesg for failed.
                            #if ( $verbose ) {
                            #    $callback->( $_ );
                            #    }


                        } else {
                            if ($errornodes) { $errornodes->{ $_->{node}->[0]->{name}->[0] } = 1; }

                            #$callback->( $_ );
                        }
                        $callback->($_);
                    }
                }
            }
            if (defined($mkvm_cec)) {
                my $r;
                $r->{errorcode} = 0;
                $r->{node}->[0]->{name}->[0] = $mkvm_cec;
                $r->{node}->[0]->{data}->[0]->{contents}->[0] = "Please reboot the CEC $mkvm_cec firstly, and then use chvm to assign the I/O slots to the LPARs";
                $callback->($r);
            }

            next;
        }
        #################################
        # Done - close handle
        #################################
        $fds->remove($rfh);
        close($rfh);
    }
    yield;    #Try to avoid useless iterations as much as possible
    return $rc;
}


##########################################################################
# Finds attributes for given node is various databases
##########################################################################
sub resolve_hcp {
    my $request   = shift;
    my $noderange = shift;
    my @nodegroup = ();
    my $tab = ($request->{hwtype} eq "fsp" or $request->{hwtype} eq "bpa") ? "ppcdirect" : "ppchcp";
    my $db = xCAT::Table->new($tab);

    ####################################
    # Database not defined
    ####################################
    if (!defined($db)) {
        send_msg($request, 1, sprintf($errmsg{DB_UNDEF}, $tab));
        return undef;
    }
    ####################################
    # Process each node
    ####################################
    foreach my $hcp (@$noderange) {

        #        my ($ent) = $db->getAttribs( {hcp=>$hcp},"hcp" );
        #        my ($ent) = $db->getNodeAttribs( $hcp, ["hcp"]);

        #        if ( !defined( $ent )) {
        #            my $msg = sprintf( "$hcp: $errmsg{NODE_UNDEF}", $tab );
        #            send_msg( $request, 1, $msg );
        #            next;
        #        }
        ################################
        # Get userid and password
        ################################
        my @cred = xCAT::PPCdb::credentials($hcp, $request->{hwtype});
        $request->{$hcp}{cred} = \@cred;

        ################################
        # Save values
        ################################
        push @nodegroup, [$hcp];
    }
    return (\@nodegroup);

}


##########################################################################
# Group nodes depending on command
##########################################################################
sub preprocess_nodes {

    my $request   = shift;
    my $hcps_will = shift;
    my $noderange = $request->{node};
    my $method    = $request->{method};
    my %nodehash  = ();
    my @nodegroup = ();
    my %hcpgroup  = ();
    my %tabs      = ();
    my $netwk;

    ########################################
    # Special cases
    #   rscan - Nodes are hardware control pts
    #   FSPpower, FSPinv and FSPrflash
    ########################################
    if ((!$request->{hcp} && ($request->{hcp} ne "hmc"))
        and ($request->{command} !~ /^renergy$/)
        and (($request->{command} =~ /^(rscan|rspconfig)$/)
            or ($request->{hwtype} eq "fsp" or $request->{hwtype} eq "bpa")
            or ($request->{command} eq 'lshwconn' and $request->{nodetype} eq 'hmc')) and ($request->{fsp_api} != 1)
      ) {
        my $result = resolve_hcp($request, $noderange);
        return ($result);
    }
    ##########################################
    # Special processing - rnetboot
    ##########################################
    if ($request->{command} eq "rnetboot" || $request->{command} eq "rbootseq") {
        $netwk = resolve_netwk($request, $noderange);
        if (!(%$netwk)) {
            return undef;
        }
    }
    ##########################################
    # Open databases needed
    ##########################################
    foreach (qw(ppc vpd nodetype)) {
        $tabs{$_} = xCAT::Table->new($_);

        if (!exists($tabs{$_})) {
            send_msg($request, 1, sprintf($errmsg{DB_UNDEF}, $_));
            return undef;
        }
    }

    ##########################################################
    # For DFM actions, all the IPs of the FSPs/BPAs corresponding CECs/Frames will be used to do the
    # comminucation with the hdwr_svr. So use the gethcpAttribs() to collect the CECs' FSPs, and Frames'
    # BPA here for process fork .
    # In the HMC hardware control environment, the mkhwconn/rmhwconn commands will use the FSPs' IPs or
    # the BPAs' IPs. So it also need to collect the CECs' FSPs, and Frames' BPA here.
    ##########################################################
    if ($request->{fsp_api} == 1 || $request->{command} =~ /^(mkhwconn|rmhwconn|lshwconn)$/) {
        xCAT::FSPUtils::getHcpAttribs($request, \%tabs);
    }


    ##########################################
    # Group nodes. The first key is hcp, and the second kery is mtms.
    # All the attributs collected here will be used for the different commands Grcup Nodes later.
    ##########################################
    foreach my $node (@$noderange) {
        my $d = resolve($request, $node, \%tabs);

        ######################################
        # Error locating node attributes
        ######################################
        if (ref($d) ne 'ARRAY') {
            send_msg($request, 1, "$node: $d");
            next;
        }
        ######################################
        # Get data values
        ######################################
        my $hcp = @$d[3];

        #my $hcp  = $hcps_will->{$node};
        #@$d[3] = $hcp;
        my $mtms = @$d[2];

        ######################################
        # Special case for mkhwconn with -p. The hcp in the command is specified by the users,
        # not in the DB, and the hcp value is stored in  $request->{opt}->{p}.
        # So set the  $request->{opt}->{p} as the key the %nodehash directly.
        ######################################
        if ($request->{command} eq "mkhwconn" and
            exists $request->{opt}->{p})
        {
            $nodehash{ $request->{opt}->{p} }{$mtms}{$node} = $d;
        }
        ######################################
        #The common case
        ######################################
        else
        {
            $nodehash{$hcp}{$mtms}{$node} = $d;
        }
    }

    ##########################################
    # Get userid and password
    ##########################################
    while (my ($hcp, $hash) = each(%nodehash)) {
        my @cred;
        if ($request->{fsp_api} != 1) {
            if ($request->{hcp} && ($request->{hcp} eq "hmc")) {
                @cred = xCAT::PPCdb::credentials($hcp, $request->{hcp});
            } else {
                @cred = xCAT::PPCdb::credentials($hcp, $request->{hwtype});
            }
            $request->{$hcp}{cred} = \@cred;
        }

        ######################################################
        #In DFM suppport, only the mkhwconn command need the userid/password to
        #create the connection between hdwr_svr and FSPs/BPAs
        ######################################################
        if ($request->{fsp_api} == 1 && $request->{command} eq "mkhwconn") {
            my $user;
            while (my ($mtms, $h) = each(%$hash)) {
                while (my ($node, $tmp_d) = each(%$h)) {
                    my $type = $$tmp_d[4];
                    ###############
                    #For NGP, get the passwd of the cmm's username USERID
                    ###############
                    if ($type && $type =~ /^blade$/) {
                        $user = "USERID";
                        my $cmm = $$tmp_d[5];
                        @cred = xCAT::PPCdb::credentials($cmm, $type, $user);
                        $request->{$cmm}{cred} = \@cred;
                    }
                    ############################
                    #For P7 IH DFM, get the password of the CEC's/Frame's username HMC
                    ###########################
                    if ($type && $type =~ /^(fsp|bpa|cec|frame)$/) {
                        $user = "HMC";
                        @cred = xCAT::PPCdb::credentials($hcp, $type, $user);
                        $request->{$hcp}{cred} = \@cred;
                    }
                }
            }

        }

    }
    ##########################################
    # Group the nodes - we will fork one
    # process per nodegroup array element.
    ##########################################

    ##########################################
    # These commands are grouped on an
    # LPAR-by-LPAR basis - fork one process
    # per LPAR.
    ##########################################
    if (($method =~ /^(getmacs)$/ && exists($request->{opt}->{D})) || ($method =~ /^(rnetboot)$/) || $method =~ /^(rbootseq)$/) {
        while (my ($hcp, $hash) = each(%nodehash)) {
            @nodegroup = ();
            while (my ($mtms, $h) = each(%$hash)) {
                while (my ($lpar, $d) = each(%$h)) {
                    push @$d, $lpar;

                    ##########################
                    # Save network info
                    ##########################
                    if ($method =~ /^rnetboot$/ || $method =~ /^(rbootseq)$/) {
                        push @$d, $netwk->{$lpar};
                    }
                    push @nodegroup, [ $hcp, $d ];
                }
            }
            $hcpgroup{$hcp}{'nodegroup'}  = [@nodegroup];
            $hcpgroup{$hcp}{'count'}      = $#nodegroup + 1;
            $hcpgroup{$hcp}{'runprocess'} = 0;
            $hcpgroup{$hcp}{'index'}      = 0;
        }
        return (\%hcpgroup);
    }

    elsif ($method =~ /^(getmacs)$/ && exists($request->{opt}->{arp})) {
        return ($noderange);
    }

    ##########################################
    # Power control commands are grouped
    # by CEC which is the smallest entity
    # that commands can be sent to in parallel.
    # If commands are sent in parallel to a
    # single CEC, the CEC itself will serialize
    # them - fork one process per CEC.
    ##########################################
    elsif ($method =~ /^powercmd/ || $method =~ /^renergy/) {
        while (my ($hcp, $hash) = each(%nodehash)) {
            while (my ($mtms, $h) = each(%$hash)) {
                push @nodegroup, [ $hcp, $h ];
            }
        }
        return (\@nodegroup);
    }
    ##########################################
    # All other commands are grouped by
    # hardware control point - fork one
    # process per hardware control point.
    ##########################################
    while (my ($hcp, $hash) = each(%nodehash)) {
        push @nodegroup, [ $hcp, $hash ];
    }
    return (\@nodegroup);
}


##########################################################################
# Finds attributes for given node is various databases
##########################################################################
sub resolve_netwk {

    my $request   = shift;
    my $noderange = shift;
    my %nethash   = xCAT::DBobjUtils->getNetwkInfo($noderange);
    my $tab       = xCAT::Table->new('mac');
    my %result    = ();
    my $ip;

    #####################################
    # Network attributes undefined
    #####################################
    if (!%nethash) {
        send_msg($request, 1, sprintf($errmsg{NODE_UNDEF}, "networks"));
        return undef;
    }
    #####################################
    # mac database undefined
    #####################################
    if (!defined($tab)) {
        send_msg($request, 1, sprintf($errmsg{DB_UNDEF}, "mac"));
        return undef;
    }

    foreach (@$noderange) {
        #################################
        # Get gateway (-G)
        #################################
        if (!exists($nethash{$_})) {
            my $msg = sprintf("$_: $errmsg{NODE_UNDEF}", "networks");
            send_msg($request, 1, $msg);
            next;
        }
        my $gateway = $nethash{$_}{gateway};
        my $gateway_ip;
        if (defined($gateway) && $gateway) {
            $ip = xCAT::NetworkUtils::toIP($gateway);
            if (@$ip[0] != 0) {
                send_msg($request, 1, "$_: Cannot resolve '$gateway'");
                next;
            }
            $gateway_ip = @$ip[1];
        }

        # Without gateway, rnetboot should still work

        my $netmask = $nethash{$_}{mask};
        if (!defined($netmask)) {
            my $msg = sprintf("$_: $errmsg{NO_ATTR}", "mask", "networks");
            send_msg($request, 1, $msg);
            next;
        }

        #################################
        # Get server (-S)
        #################################
        my $server = xCAT::TableUtils->GetMasterNodeName($_);
        if ($server == 1) {
            send_msg($request, 1, "$_: Unable to identify master");
            next;
        }
        $ip = xCAT::NetworkUtils::toIP($server);
        if (@$ip[0] != 0) {
            send_msg($request, 1, "$_: Cannot resolve '$server'");
            next;
        }
        my $server_ip = @$ip[1];

        #################################
        # Get client (-C)
        #################################
        $ip = xCAT::NetworkUtils::toIP($_);
        if (@$ip[0] != 0) {
            send_msg($request, 1, "$_: Cannot resolve '$_'");
            next;
        }
        my $client_ip = @$ip[1];

        #################################
        # Get mac-address (-m)
        #################################
        my ($ent) = $tab->getNodeAttribs($_, ['mac']);
        if (!defined($ent)) {
            my $msg = sprintf("$_: $errmsg{NO_ATTR}", "mac", "mac");
            send_msg($request, 1, $msg);
            next;
        }
        #################################
        # Save results
        #################################
        $result{$_}{gateway} = $gateway_ip;
        $result{$_}{server}  = $server_ip;
        $result{$_}{client}  = $client_ip;
        $result{$_}{mac}     = $ent->{mac};
        $result{$_}{netmask} = $netmask;
    }
    return (\%result);
}


##########################################################################
# Finds attributes for given node is various databases
##########################################################################
sub resolve {

    my $request = shift;
    my $node    = shift;
    my $tabs    = shift;
    my @attribs = qw(id pprofile parent hcp);
    my @values  = ();

    #################################
    # Get node type
    #################################
    #my $ent = $tabs->{nodetype}->getNodeAttribs($node,[qw(nodetype node)]);
    #if ( !defined( $ent )) {
    #    return( sprintf( $errmsg{NODE_UNDEF}, "nodetype" ));
    #}
    #################################
    # Check for type
    #################################
    #if ( !exists( $ent->{nodetype} )) {
    #    return( sprintf( $errmsg{NO_ATTR}, "nodetype","nodetype" ));
    #}
    #################################
    # Check for valid "type"
    #################################
    my $ttype = xCAT::DBobjUtils->getnodetype($node, "ppc");
    my ($type) = grep(
/^$::NODETYPE_LPAR|$::NODETYPE_OSI|$::NODETYPE_BPA|$::NODETYPE_FSP|$::NODETYPE_CEC|$::NODETYPE_FRAME|$::NODETYPE_BLADE$/,

        #split /,/, $ent->{nodetype} );
        split /,/, $ttype);

    if (!defined($type)) {

        #return( "Invalid node type: $ent->{nodetype}" );
        return ("Invalid node type: $ttype");
    }
    #################################
    # Get attributes
    #################################
    my ($att) = $tabs->{ppc}->getNodeAttribs($node, \@attribs);

    if (!defined($att)) {
        return (sprintf($errmsg{NODE_UNDEF}, "ppc"));
    }
    #################################
    # Special lpar processing
    #################################
    if ($type =~ /^$::NODETYPE_OSI|$::NODETYPE_LPAR$/) {
        $att->{bpa}  = 0;
        $att->{type} = "lpar";
        $att->{node} = $att->{parent};

        if (!exists($att->{parent})) {
            return (sprintf($errmsg{NO_ATTR}, "parent", "ppc"));
        }
        #############################
        # Get BPA (if any)
        #DFM doesn't support rvitals with temp, so add the if  fsp_api != 1
        #It will improve the performance of rvitals with all
        #############################
        if (($request->{command} eq "rvitals") &&
            ($request->{method} =~ /^all|temp$/) && $request->{fsp_api} != 1) {
            my ($ent) = $tabs->{ppc}->getNodeAttribs($att->{parent}, ['parent']);

            #############################
            # Find MTMS in vpd database
            #############################
            if ((defined($ent)) && exists($ent->{parent})) {
                my @attrs = qw(mtm serial);
                my ($vpd) = $tabs->{vpd}->getNodeAttribs($ent->{parent}, \@attrs);

                ########################
                # Verify attributes
                ########################
                foreach (@attrs) {
                    if (!defined($vpd) || !exists($vpd->{$_})) {
                        return (sprintf($errmsg{NO_UNDEF}, $_, "vpd", $ent->{parent}));
                    }
                }
                $att->{bpa} = "$vpd->{mtm}*$vpd->{serial}";
            }
        }
    }
    #################################
    # Optional and N/A fields
    #################################
    elsif ($type =~ /^$::NODETYPE_FSP$/) {
        $att->{pprofile} = 0;
        $att->{id}       = 0;
        $att->{fsp}      = 0;
        $att->{node}     = $node;
        $att->{type}     = $type;
        $att->{parent}   = exists($att->{parent}) ? $att->{parent} : 0;
        $att->{bpa}      = $att->{parent};

        my $ntype;
        if (exists($att->{parent})) {
            $ntype = xCAT::DBobjUtils->getnodetype($att->{parent}, "ppc");
        }
        if (($request->{command} eq "rvitals") &&
            ($request->{method} =~ /^all|temp$/ && $ntype =~ /^cec$/) && $request->{fsp_api} != 1) {
            my ($ent) = $tabs->{ppc}->getNodeAttribs($att->{parent}, ['parent']);

            #############################
            # Find MTMS in vpd database
            #############################
            if ((defined($ent)) && exists($ent->{parent})) {
                my @attrs = qw(mtm serial);
                my ($vpd) = $tabs->{vpd}->getNodeAttribs($ent->{parent}, \@attrs);

                ########################
                # Verify attributes
                ########################
                foreach (@attrs) {
                    if (!defined($vpd) || !exists($vpd->{$_})) {
                        return (sprintf($errmsg{NO_UNDEF}, $_, "vpd", $ent->{parent}));
                    }
                }
                $att->{bpa} = "$vpd->{mtm}*$vpd->{serial}";
            }
        } elsif (($request->{command} eq "rvitals") &&
            ($request->{method} =~ /^all|temp$/ && $ntype =~ /^bpa$/) && $request->{fsp_api} != 1) {
            my @attrs = qw(mtm serial);
            my ($vpd) = $tabs->{vpd}->getNodeAttribs($att->{parent}, \@attrs);
            ########################
            # Verify attributes
            ########################
            foreach my $attr (@attrs) {
                if (!defined($vpd) || !exists($vpd->{$attr})) {
                    return (sprintf($errmsg{NO_UNDEF}, $attr, "vpd", $att->{parent}));
                }
            }
            $att->{bpa} = "$vpd->{mtm}*$vpd->{serial}";
        }
    }
    elsif ($type =~ /^$::NODETYPE_BPA$/) {
        $att->{pprofile} = 0;
        $att->{id}       = 0;
        $att->{bpa}      = 0;
        $att->{parent}   = 0;
        $att->{fsp}      = 0;
        $att->{node}     = $node;
        $att->{type}     = $type;
    }
    elsif ($type =~ /^$::NODETYPE_CEC|$::NODETYPE_BLADE$/) {
        $att->{pprofile} = 0;
        if ($type =~ /^$::NODETYPE_CEC$/) {
            $att->{id} = 0;
        }
        $att->{fsp}    = 0;
        $att->{node}   = $node;
        $att->{type}   = $type;
        $att->{parent} = exists($att->{parent}) ? $att->{parent} : 0;
        $att->{bpa}    = $att->{parent};

        if (($request->{command} eq "rvitals") &&
            ($request->{method} =~ /^all|temp$/) && $request->{fsp_api} != 1) {

            #############################
            # Find MTMS in vpd database
            #############################
            if ($att->{parent}) {
                my @attrs = qw(mtm serial);
                my ($vpd) = $tabs->{vpd}->getNodeAttribs($att->{parent}, \@attrs);

                ########################
                # Verify attributes
                ########################
                foreach (@attrs) {
                    if (!defined($vpd) || !exists($vpd->{$_})) {
                        return (sprintf($errmsg{NO_UNDEF}, $_, "vpd", $att->{parent}));
                    }
                }
                $att->{bpa} = "$vpd->{mtm}*$vpd->{serial}";
            }
        }
    }
    elsif ($type =~ /^$::NODETYPE_FRAME$/) {
        $att->{pprofile} = 0;
        $att->{id}       = 0;
        $att->{bpa}      = 0;
        $att->{parent}   = 0;
        $att->{fsp}      = 0;
        $att->{node}     = $node;
        $att->{type}     = $type;
    }
    #################################
    # Find MTMS in vpd database
    #################################
    my @attrs = qw(mtm serial);
    my ($vpd) = $tabs->{vpd}->getNodeAttribs($att->{node}, \@attrs);

    if (!defined($vpd)) {
        return (sprintf($errmsg{NODE_UNDEF}, "vpd: ($att->{node})"));
    }
    ################################
    # Verify both vpd attributes
    ################################
    foreach (@attrs) {
        if (!exists($vpd->{$_})) {
            return (sprintf($errmsg{NO_ATTR}, $_, "vpd: ($att->{node})"));
        }
    }
    $att->{fsp} = "$vpd->{mtm}*$vpd->{serial}";

    #################################
    # Verify required attributes
    #################################
    foreach my $at (@attribs) {
        if (!exists($att->{$at})) {
            if (!($request->{fsp_api} == 1 && !exists($att->{pprofile}))) { #for p7 ih, there is no pprofile attribute
                return (sprintf($errmsg{NO_ATTR}, $at, "ppc"));
            }
        }
    }

    ############################
    #Specail case for mkhwconn/lshwconn/rmhwconn with -s . The sfp value( always the hmc) should
    #be set as the hcp.
    ############################
    if ($request->{command} =~ /^(mkhwconn|lshwconn|rmhwconn)$/ && grep (/^-s$/, @{ $request->{arg} })) {
        my $sfp;
        unless ($request->{sfp}) {
            my $ent = $tabs->{ppc}->getNodeAttribs($node, [qw(sfp)]);
            if ($request->{command} =~ /^(lshwconn|rmhwconn)$/ && !defined($ent)) {
                return ("$node: No sfp defined in the ppc table.");

            }
            $sfp = $ent->{sfp};

        } else {
            $sfp = $request->{sfp};
        }
        if ($request->{command} =~ /^(mkhwconn)$/ && !defined($sfp)) {
            return ("$node: Please specify the sfp in the commands or in the ppc table.");
        }
        $request->{fsp_api} = 0;
        $request->{hwtype}  = "hmc";
        $att->{hcp}         = $sfp;
    }


    #################################
    # Build array of data
    #################################
    foreach (qw(id pprofile fsp hcp type bpa)) {
        push @values, $att->{$_};
    }
    return (\@values);
}



##########################################################################
# Forks a process to run the ssh command
##########################################################################
sub fork_cmd {

    my $host    = shift;
    my $nodes   = shift;
    my $request = shift;

    #######################################
    # Pipe childs output back to parent
    #######################################
    my $parent;
    my $child;
    pipe $parent, $child;
    my $pid = xCAT::Utils->xfork;

    if (!defined($pid)) {
        ###################################
        # Fork error
        ###################################
        send_msg($request, 1, "Fork error: $!");
        return undef;
    }
    elsif ($pid == 0) {
        ###################################
        # Child process
        ###################################
        close($parent);
        $request->{pipe} = $child;

        invoke_cmd($host, $nodes, $request);
        exit(0);
    }
    else {
        ###################################
        # Parent process
        ###################################
        close($child);
        return ($parent, $pid);
    }
    return (0);
}

sub handle_cmd {
    my $server  = shift;
    my $host    = shift;
    my $request = shift;
    my $verbose = $request->{verbose};
    eval { require xCAT::PPCfsp };
    if ($@) {
        send_msg($request, 1, $@);
        return;
    }
    my @exp = xCAT::PPCfsp::connect($request, $server);

####################################
    # Error connecting
####################################
    if (ref($exp[0]) ne "LWP::UserAgent") {

        #send_msg( $request, 1, $exp[0] );
        my @output_array;
        my $methods = $request->{method};
        foreach (keys %$methods) {
            my %output;
            $output{node}->[0]->{name}->[0] = "$host";
            $output{node}->[0]->{data}->[0]->{contents}->[0] = "$server: $exp[0]";
            $output{node}->[0]->{cmd}->[0] = $_;
            $output{errorcode} = 128;
            push @output_array, \%output;
        }
        return \@output_array;
    }
    $request->{host} = $host;
    my $result = xCAT::PPCfsp::handler($server, $request, \@exp);

####################################
    # Output verbose Perl::LWP
####################################
    if ($verbose) {
        my $verbose_log = $exp[3];
        my $output      = shift @$result;
        $output->{data} = [$$verbose_log];
        unshift @$result, \%$output;
    }
    return $result;
}

sub handle_find_hw_children {
    my $host       = shift;
    my $child_type = shift;
    my @children   = ();
    my $vpdtab     = xCAT::Table->new("vpd");
    if (!defined($vpdtab) or !defined($child_type)) {
        return undef;
    }
    my $mtms = $vpdtab->getNodeAttribs($host, qw(serial mtm));
    if (!defined($mtms)) {
        return undef;
    }
    my @nodearray = $vpdtab->getAttribs({ serial => $mtms->{serial}, mtm => $mtms->{mtm} }, qw(node side)); # need regx
    if (!(@nodearray)) {
        return undef;
    }
    my @tempnodes;
    foreach (@nodearray) {
        push @tempnodes, $_->{node};
    }
    my $typehash = xCAT::DBobjUtils->getnodetype(\@tempnodes, "ppc");
    foreach my $node (@nodearray) {
        my $n_type = $$typehash{ $node->{node} };
        if ($n_type !~ /^$child_type$/ or !defined($node->{side})) {
            next;
        }
        push @children, $node;
    }
    if (scalar(@children) eq '0') {
        return undef;
    }
    return \@children;
}

sub print_res {
    my $request   = shift;
    my $host      = shift;
    my $res_array = shift;
    if (!defined($res_array) or ref($res_array) ne 'ARRAY') {
        return;
    }
    my $out = $request->{pipe};
    print $out freeze($res_array);
    print $out "\nENDOFFREEZE6sK4ci\n";
    return;
}

sub get_dirindex_from_side {
    my $side = shift;
    my $dir  = undef;
    if ($side =~ /(a+)-\d*/i) {
        $dir = '0';
    } elsif ($side =~ /(b+)-\d*/i) {
        $dir = '1';
    } else {
        $dir = '2';
    }
    return $dir;
}

sub reorgnize_res_for_handle_cmd {
    my $request   = shift;
    my $host      = shift;
    my $output    = shift;
    my $methods   = $request->{method};
    my %re_output = ();
    my %succ_side = ();
    my $dir       = undef;
    foreach my $side (keys %$output) {
        my $res = $output->{$side};
        foreach my $index (@$$res) {
            $dir = &get_dirindex_from_side($side);
            my $cmd = $index->{node}->[0]->{cmd}->[0];
            if (($index->{node}->[0]->{data}->[0]->{contents}->[0]) =~ /feature is not available/) {
                $dir = '0';
            }
            if (!defined($succ_side{$cmd}{$dir}{done})) {
                if (defined($re_output{$cmd}{$dir})) {
                    my $array = $re_output{$cmd}{$dir};
                    my $n     = scalar(@$array);
                    $array->[$n] = $index;
                } else {
                    $re_output{$cmd}{$dir}[0] = $index;
                }
            } else {
                next;
            }
            if ($index->{errorcode} eq '0') {
                @{ $re_output{$cmd}{$dir} } = ();
                $re_output{$cmd}{$dir}[0] = $index;
                $succ_side{$cmd}{$dir}{done}++;
            }
        }
    }
    foreach my $method (keys %re_output) {
        my $value = $re_output{$method};
        foreach my $side (keys %$value) {
            my $res_array = $value->{$side};
            &print_res($request, $host, $res_array);
        }
    }
    return undef;
}

sub process_children {
    my $request    = shift;
    my $host       = shift;
    my $node_array = shift;
    my %output     = ();
    my %conn_flag  = ();
    foreach my $index (@$node_array) {
        my $side = $index->{side};
        my $dir  = &get_dirindex_from_side($side);
        if (defined($conn_flag{$dir})) {
            next;
        }
        $request->{ $index->{node} }{cred} = $request->{$host}{cred};
        my $res = &handle_cmd($index->{node}, $host, $request);
        $output{$side} = \$res;
        if ($res->[0]->{errorcode} ne '128') {
            $conn_flag{$dir} = 1;
        }
    }
    &reorgnize_res_for_handle_cmd($request, $host, \%output);
    return undef;
}


my %children_type = (
    cec   => "fsp",
    frame => "bpa"
);

sub handle_redundance_fsps {
    my $host    = shift;
    my $hwtype  = shift;
    my $request = shift;
    my $res     = undef;
    if ($hwtype =~ /^(bpa|fsp)$/) {
        $res = &handle_cmd($host, $host, $request);
        &print_res($request, $host, $res);
    } elsif ($hwtype =~ /^(cec|frame)$/) {
        my $child_type = $children_type{$hwtype};
        my $children = &handle_find_hw_children($host, $child_type);
        if (!defined($children)) {
            send_msg($request, 1, "Not found any $child_type for $host");
        } else {
            &process_children($request, $host, $children);
        }
    } else {
        send_msg($request, 1, "$hwtype not support!");
    }
    return undef;
}

sub check_node_info {
    my $hash    = shift;
    my $if_lpar = undef;
    while (my ($mtms, $h) = each(%$hash)) {
        while (my ($name, $d) = each(%$h)) {
            my $node_type = @$d[4];
            if ($node_type =~ /^lpar$/) {
                $if_lpar = $name;
                last;
            }
        }
    }
    return $if_lpar;
}


##########################################################################
# Run the command, process the response, and send to parent
##########################################################################
sub invoke_cmd {

    my $host    = shift;
    my $nodes   = shift;
    my $request = shift;
    my $hwtype  = $request->{hwtype};
    my $verbose = $request->{verbose};
    my $cmd     = $request->{command};
    my $power   = $request->{hcp};
    my @exp;
    my $verbose_log;
    my @outhash;

    ########################################
    # If the request command is renergy, just
    # uses the xCAT CIM Client to handle it
    ########################################
    if ($request->{command} eq "renergy") {
        my $result = &runcmd($request, $host, $nodes);

        ########################################
        # Format and send back to parent
        ########################################
        foreach my $line (@$result) {
            my %output;
            $output{node}->[0]->{name}->[0]                  = @$line[0];
            $output{node}->[0]->{data}->[0]->{contents}->[0] = @$line[1];
            $output{errorcode}                               = @$line[2];
            push @outhash, \%output;
        }
        my $out = $request->{pipe};
        print $out freeze([@outhash]);
        print $out "\nENDOFFREEZE6sK4ci\n";

        return;
    }

    ########################################
    # Direct-attached FSP handler
    ########################################
    if (($power ne "hmc") && ($hwtype =~ /^(fsp|bpa|cec|frame|blade)$/) && $request->{fsp_api} == 0) {

        if ($request->{command} eq 'rpower') {
            my $check = &check_node_info($nodes);
            if (defined($check)) {
                my %output;
                $output{node}->[0]->{name}->[0] = $check;
                $output{node}->[0]->{data}->[0]->{contents}->[0] = "Error: $request->{command} not support on lpar in ASMI mode";
                $output{node}->[0]->{cmd}->[0] = $request->{method};
                $output{errorcode} = 1;
                my $out = $request->{pipe};
                print $out freeze([ \%output ]);
                print $out "\nENDOFFREEZE6sK4ci\n";
                return;
            }

            if (ref($request->{method}) ne 'HASH') {
                my %method_hash = ();
                my $method      = $request->{method};
                $method_hash{$method} = undef;
                $request->{method} = \%method_hash;
            }
        }
        &handle_redundance_fsps($host, $hwtype, $request);
        return;
    }

    ########################################
    # HMC and IVM-managed handler
    # Connect to list of remote servers
    ########################################
    if ($request->{fsp_api} == 0) {
        foreach (split /,/, $host) {
            if ($power ne "hmc") {
                @exp = xCAT::PPCcli::connect($request, $hwtype, $_);
            } else {
                @exp = xCAT::PPCcli::connect($request, $power, $_);
            }
            ####################################
            # Successfully connected
            ####################################
            if (ref($exp[0]) eq "Expect") {
                last;
            }
        }

        ########################################
        # Error connecting
        ########################################
        if (ref($exp[0]) ne "Expect") {
            send_msg($request, 1, $exp[0]);
            return;
        }
    }

    ########################################
    # Process specific command
    ########################################
    my $result = runcmd($request, $nodes, \@exp);

    ########################################
    # Close connection to remote server
    ########################################
    if ($request->{fsp_api} == 0) {
        xCAT::PPCcli::disconnect(\@exp);
    }

    ########################################
    # Get verbose Expect output
    ########################################
    if ($verbose) {
        $verbose_log = $exp[6];
    }
    ########################################
    # Return error
    ########################################
    if (ref($result) ne 'ARRAY') {
        send_msg($request, 1, $$verbose_log . $result);
        return;
    }
    ########################################
    # Prepend verbose output
    ########################################
    if (defined($verbose_log)) {
        my %output;
        $output{data} = [$$verbose_log];
        push @outhash, \%output;
    }
    ########################################
    # Send result back to parent process
    ########################################
    if (@$result[0] eq "FORMATDATA6sK4ci") {
        my $out = $request->{pipe};

        push @outhash, @$result[1];
        print $out freeze([@outhash]);
        print $out "\nENDOFFREEZE6sK4ci\n";
        return;
    }
    ########################################
    # Format and send back to parent
    ########################################
    foreach (@$result) {
        my %output;
        $output{node}->[0]->{name}->[0] = @$_[0];

        #$output{node}->[0]->{data}->[0]->{contents}->[0] = @$_[1];
        $output{errorcode} = @$_[2];
        if ($output{errorcode} != 0) {
            if ($request->{fsp_api} == 1) {

                #$output{node}->[0]->{data}->[0]->{contents}->[0] = "(trying fsp-api)@$_[1]";
                $output{node}->[0]->{data}->[0]->{contents}->[0] = "@$_[1]";
            } else {

                #$output{node}->[0]->{data}->[0]->{contents}->[0] = "(trying HMC    )@$_[1]";
                $output{node}->[0]->{data}->[0]->{contents}->[0] = "@$_[1]";
            }
        } else {
            $output{node}->[0]->{data}->[0]->{contents}->[0] = @$_[1];
        }
        push @outhash, \%output;
    }
    my $out = $request->{pipe};
    print $out freeze([@outhash]);
    print $out "\nENDOFFREEZE6sK4ci\n";
}


##########################################################################
# Run the command method specified
##########################################################################
sub runcmd {

    my $request = shift;
    my $cmd     = $request->{command};
    my $method  = $request->{method};
    my $hwtype  = $request->{hwtype};

    #my $modname = $modules{$cmd};
    my $modname = $modules{$cmd}{$hwtype};

    ######################################
    # Command not supported
    ######################################
    if (!defined($modname)) {
        return (["$cmd not a supported command by $hwtype method"]);
    }
    ######################################
    # Load specific module
    ######################################
    eval "require $modname";
    if ($@) {
        return ([$@]);
    }
    ######################################
    # Invoke method
    ######################################
    no strict 'refs';
    my $result = ${ $modname . "::" }{$method}->($request, @_);
    use strict;

    return ($result);

}

##########################################################################
# Pre-process request from xCat daemon. Send the request to the the service
# nodes of the HCPs.
##########################################################################
sub preprocess_request {

    my $package = shift;
    my $req     = shift;

    #if ($req->{_xcatdest}) { return [$req]; }    #exit if preprocessed
    if ($req->{_xcatpreprocessed}->[0] == 1) { return [$req]; }
    my $callback = shift;
    my $subreq   = shift;
    my @requests;

    #####################################
    # Parse arguments
    #####################################
    my $opt = parse_args($package, $req, $callback);
    if (ref($opt) eq 'ARRAY')
    {
        send_msg($req, 1, @$opt);
        delete($req->{callback}); # if not, it will cause an error --  "Can't encode a value of type: CODE" in hierairchy.
        return (1);
    }
    delete($req->{callback}); # remove 'callback' => sub { "DUMMY" } in hierairchy.
    $req->{opt} = $opt;

    if (exists($req->{opt}->{V})) {
        $req->{verbose} = 1;
    }

    ####################################
    # Get hwtype
    ####################################
    $package =~ s/xCAT_plugin:://;

    my $deps;
    my $nodeseq;
    if (($req->{command}->[0] eq 'rpower') && (!grep(/^--nodeps$/, @{ $req->{arg} }))
        && (($req->{op}->[0] eq 'on') || ($req->{op}->[0] eq 'off')
            || ($req->{op}->[0] eq 'softoff') || ($req->{op}->[0] eq 'reset'))) {

        $deps = xCAT::SvrUtils->build_deps($req->{node}, $req->{op}->[0]);

        # no dependencies at all
        if (!defined($deps)) {
            foreach my $node (@{ $req->{node} }) {
                $nodeseq->[0]->{$node} = 1;
            }
        } else {
            $nodeseq = xCAT::SvrUtils->handle_deps($deps, $req->{node}, $callback);
        }
    }

    if ($nodeseq == 1) {
        return undef;
    }

    # no dependency defined in deps table,
    # generate the $nodeseq hash
    if (!$nodeseq) {
        foreach my $node (@{ $req->{node} }) {
            $nodeseq->[0]->{$node} = 1;
        }
    }

    my $i = 0;
    for ($i = 0 ; $i < scalar(@{$nodeseq}) ; $i++) {

        #reset the @requests for this loop
        @requests = ();
        ####################################
        # Prompt for usage if needed and on MN
        ####################################
        my @dnodes = keys(%{ $nodeseq->[$i] });

        if (scalar(@dnodes) == 0) {
            next;
        }
        if (scalar(@{$nodeseq}) > 1) {
            my %output;
            my $cnodes = join(',', @dnodes);
            $output{data} = ["Performing action against the following nodes: $cnodes"];
            $callback->(\%output);
        }
        my $noderange = \@dnodes;
        $req->{node} = \@dnodes;    #Should be arrayref
             #$req->{noderange} = \@dnodes; #Should be arrayref
        my $command  = $req->{command}->[0];
        my $extrargs = $req->{arg};
        my @exargs   = ($req->{arg});
        if (ref($extrargs)) {
            @exargs = @$extrargs;
        }
        if ($ENV{'XCATBYPASS'}) {
            my $usage_string = xCAT::Usage->parseCommand($command, @exargs);
            if ($usage_string) {
                $callback->({ data => [$usage_string] });
                $req = {};
                return;
            }
            if (!$noderange) {
                $usage_string = "Missing noderange";
                $callback->({ data => [$usage_string] });
                $req = {};
                return;
            }
        }


        ##################################################################
        # get the HCPs for the LPARs in order to figure out which service
        # nodes to send the requests to
        ###################################################################
        #my $hcptab_name = ($package eq "fsp" or $package eq "bpa") ? "ppcdirect" : "ppchcp";
        #my $hcptab  = xCAT::Table->new( $hcptab_name );
        #unless ($hcptab ) {
        #    $callback->({data=>["Cannot open $hcptab_name table"]});
        #    $req = {};
        #    return;
        #}
        # Check if each node is hcp
        my %hcp_hash    = ();
        my @missednodes = ();
        my $support_hcp_type;

        # in the DFM model, cec/fsp/Frame/bpa can be hcp.
        if ($package eq "fsp" or $package eq "bpa") {
            $support_hcp_type = "(fsp|cec|bpa|frame|blade)";

            # in the HMC model, only hmc can be hcp.
        } elsif ($package eq "hmc") {
            $support_hcp_type = "hmc";

            # package equal 'ivm', only ivm can be hcp.
        } else {
            $support_hcp_type = "ivm";
        }
        my $typehash = xCAT::DBobjUtils->getnodetype(\@$noderange);
        foreach (@$noderange) {
            my $nodetype = $$typehash{$_};
            if ($nodetype and $nodetype =~ /$support_hcp_type/) {
                push @{ $hcp_hash{$_}{nodes} }, $_;
            } else {
                push @missednodes, $_;
            }
        }

        #foreach ( @$noderange ) {
        #    my ($ent) = $hcptab->getNodeAttribs( $_,"hcp" );
        #    if ( !defined( $ent )) {
        #       push @missednodes, $_;
        #    next;
        #    }
        #    push @{$hcp_hash{$_}{nodes}}, $_;
        #}

        #check if the left-over nodes are lpars
        if (@missednodes > 0) {
            my $ppctab = xCAT::Table->new("ppc");
            unless ($ppctab) {
                $callback->({ data => ["Cannot open ppc table"] });
                $req = {};
                return;
            }
            foreach my $node (@missednodes) {

                my $ent = $ppctab->getNodeAttribs($node, ['hcp']);

                #if (defined($ent->{hcp})) { push @{$hcp_hash{$ent->{hcp}}{nodes}}, $node;}
                if (defined($ent->{hcp})) {

                    #for multiple hardware control points, the hcps should be split to nodes
                    my @h = split(",", $ent->{hcp});
                    foreach my $hcp (@h) {
                        push @{ $hcp_hash{$hcp}{nodes} }, $node;
                    }
                } else {
                    $callback->({ data => ["The node $node is neither a hcp nor an lpar"] });
                    $req = {};
                    return;
                }
            }
        }

        my @masters = xCAT::TableUtils->get_site_attribute("master");

        #When run mkhwconn/lshwconn/rmhwconn with -T fnm for CNM, it will send the command to CEC/Frame direclty,
        #not through the service node if specified.
        if ($req->{command}->[0] =~ /^(mkhwconn|lshwconn|rmhwconn|rpower)$/
            && ($req->{opt}->{T} == 1)) {

            #for fnm
            my $reqcopy = {%$req};

            #my @masters = xCAT::TableUtils->get_site_attribute("master");
            if ($masters[0]) {
                $reqcopy->{'_xcatdest'} = $masters[0];
                push @requests, $reqcopy;
            } else {
                $callback->({ data => ["The value of the attribute master in the site table is NOT set"] });
                $req = {};
                return;
            }
        } else {

            # find service nodes for the HCPs
            # build an individual request for each service node
            my $service = "xcat";
            my @hcps    = keys(%hcp_hash);
            my $sn;
            my @dfmdispatch = xCAT::TableUtils->get_site_attribute("hwctrldispatch");
            if (defined($dfmdispatch[0]) and ($dfmdispatch[0] =~ /0|n/i)) {
                if ($masters[0]) {
                    push @{ $sn->{ $masters[0] } }, @hcps;
                } else {
                    $callback->({ data => ["The value of the attribute master in the site table is NOT set"] });
                    $req = {};
                    return;
                }
            } else {
                $sn = xCAT::ServiceNodeUtils->get_ServiceNode(\@hcps, $service, "MN");
            }

            # build each request for each service node
            foreach my $snkey (keys %$sn)
            {
                #$callback->({data=>["The service node $snkey "]});
                my $reqcopy = {%$req};
                $reqcopy->{'_xcatdest'} = $snkey;
                $reqcopy->{_xcatpreprocessed}->[0] = 1;
                my $hcps1 = $sn->{$snkey};
                my @nodes = ();
                foreach (@$hcps1) {
                    push @nodes, @{ $hcp_hash{$_}{nodes} };
                }
                @nodes = sort @nodes;
                my %hash = map { $_ => 1 } @nodes; #remove the repeated node for multiple hardware control points
                @nodes = keys %hash;
                $reqcopy->{node} = \@nodes;

                #print "nodes=@nodes\n";
                push @requests, $reqcopy;

                if (($req->{command}->[0] eq "rflash") && (exists($req->{opt}->{activate}))) {
                    my $linuxrequired = 0;
                    if (xCAT::Utils->isLinux()) {
                        my @installloc = xCAT::TableUtils->get_site_attribute("installloc");
                        if (!($installloc[0])) {
                            $linuxrequired = 1;
                        }
                    }

                    if (($linuxrequired || xCAT::Utils->isAIX()) && ($masters[0] ne $snkey)) {
                        my $install_dir = xCAT::TableUtils->getInstallDir();
                        my $cmd = "$::XCATROOT/bin/xdcp $snkey -R  $install_dir/packages_fw $install_dir/";
                        my $result = xCAT::Utils->runcmd("$cmd", -1);
                        if ($::RUNCMD_RC != 0) {
                            $callback->({ data => ["$result. Could not copy rpms in the $install_dir/packages_fw  to $snkey.\n"] });
                            $req = {};
                            return;
                        }
                    }

                }
            }
        }

        # No dependency, use the original logic
        if (scalar(@{$nodeseq}) == 1) {
            return \@requests;
        }

        # do all the new request entries in this loop
        my $j = 0;
        for ($j = 0 ; $j < scalar(@requests) ; $j++) {
            $subreq->(\%{ $requests[$j] }, $callback);
        }

        # We can not afford waiting 'msdelay' for each node,
        # for performance considerations,
        # use the maximum msdelay for all nodes
        my $delay = 0;

        # do not need to calculate msdelay for the last loop
        if ($i < scalar(@{$nodeseq})) {
            foreach my $reqnode (@{ $req->{node} }) {
                foreach my $depnode (keys %{$deps}) {
                    foreach my $depent (@{ $deps->{$depnode} }) {

                        # search if the 'nodedep' includes the $reqnode
                        # do not use grep, performance problem!
                        foreach my $depentnode (split(/,/, $depent->{'nodedep'})) {
                            if ($depentnode eq $reqnode) {
                                if ($depent->{'msdelay'} > $delay) {
                                    $delay = $depent->{'msdelay'};
                                }
                            }
                        }
                    }
                }
            }
        }

        if ($ENV{'XCATDEBUG'}) {
            my %output;
            $output{data} = ["delay = $delay"];
            $callback->(\%output);
        }

        #convert from millisecond to second
        $delay /= 1000.0;
        if ($delay && ($i < scalar(@{$nodeseq}))) {
            my %output;
            $output{data} = ["Waiting $delay seconds for node dependencies\n"];
            $callback->(\%output);
            if ($ENV{'XCATDEBUG'}) {
                $output{data} = ["Before sleep $delay seconds"];
                $callback->(\%output);
            }
            Time::HiRes::sleep($delay);
            if ($ENV{'XCATDEBUG'}) {
                $output{data} = ["Wake up!"];
                $callback->(\%output);
            }
        }
    }
    return undef;
}
####################################
# Parse arguments
####################################
sub parse_args
{
    my $package  = shift;
    my $req      = shift;
    my $callback = shift;
    $package =~ s/xCAT_plugin:://;

    #    if ( exists $req->{opt})
    #    {
    #        return $req->{opt};
    #    }

    #################################
    # To match the old logic
    ##################################
    my $command = $req->{command}->[0];
    my $stdin   = $req->{stdin}->[0];
    $req->{command}  = $command;
    $req->{stdin}    = $stdin;
    $req->{hwtype}   = $package;
    $req->{callback} = $callback;
    $req->{method}   = "parse_args";

    my $opt = runcmd($req);

    $req->{command} = [$command];
    $req->{stdin}   = [$stdin];
    $req->{method}  = [ $req->{method} ];
    $req->{op}      = [ $req->{op} ];
    return $opt;
}




##########################################################################
# Process request from xCat daemon
##########################################################################
sub process_request {

    my $package  = shift;
    my $req      = shift;
    my $callback = shift;
    my $subreq   = shift;

    ####################################
    # Get hwtype
    ####################################
    $package =~ s/xCAT_plugin:://;
    ####################################
    # Build hash to pass around
    ####################################
    my $request = {%$req};
    $request->{command}    = $req->{command}->[0];
    $request->{stdin}      = $req->{stdin}->[0];
    $request->{method}     = $req->{method}->[0];
    $request->{op}         = $req->{op}->[0];
    $request->{enableASMI} = $req->{enableASMI};

    #support more options in hierachy
    if (ref($req->{opt}) eq 'ARRAY') {
        my $h = $req->{opt}->[0];
        my %t = ();
        foreach my $k (keys %$h) {
            $t{$k} = $h->{$k}->[0];
        }
        $request->{opt} = \%t;
    }
    if (ref($req->{hwtype}) eq 'ARRAY') {
        $request->{hwtype} = $req->{hwtype}->[0];
    }

    #   $request->{hwtype}  = $package;
    $request->{callback} = $callback;
    $request->{subreq}   = $subreq;
    #########################
    #This is a special case for rspconfig and mkhwconn,
    #we shouldn't set hwtype as$package. and reserved for other commands.
    #probably for all HW ctrl commands it still true?
    #########################
    if ($request->{command} ne "rspconfig" and
        $request->{command} ne "mkhwconn") {
        $request->{hwtype} = $package;
    }

    ####################################
    # Option -V for verbose output
    ####################################
    if (exists($request->{opt}->{V})) {
        $request->{verbose} = 1;
    }

    #if( $request->{hwtype} ne 'hmc' ) {
    if ($request->{hwtype} !~ /hmc|ivm/) {
        $request->{fsp_api} = 1;

        #For using rspconfig options through ASMI
        if ($request->{command} eq "rspconfig" and ref($request->{method}) eq 'HASH') {
            $request->{fsp_api} = 0;
        }
    } else {
        $request->{fsp_api} = 0;
    }

    #print Dumper($request);
    process_command($request);
    return;


    ####################################
    # Process remote command
    ####################################
    #process_command( $request );

    #The following code will not be used in xCAT 2.7

    #The following code supports for Multiple hardware control points.
    #use @failed_nodes to store the nodes which need to be run.
    #print "before process_command\n";
    #print Dumper($request);
    my %failed_msg   = ();                 # store the error msgs
    my $t            = $request->{node};
    my @failed_nodes = @$t;

    #print "-------failed nodes-------\n";
    #print Dumper(\@failed_nodes);
    my $hcps = getHCPsOfNodes(\@failed_nodes, $callback, $request);
    if (!defined($hcps)) {

        #Not found the hcp for one node
        $request = {};
        return;
    }
    #####################
    #print Dumper($hcps);
    #$VAR1 = {
    #          'lpar01' => {
    #                        'num' => 2,
    #                        'hcp' => [
    #                                   'Server-9110-51A-SN1075ECF',
    #                                   'c76v2hmc02'
    #                                 ]
    #                       }
    #         };
    ######################
    while (1) {
        my $lasthcp_type;
        my %hcps_will = ();
        my @next      = ();
        my $count;    #to count the nodes who doesn't have hcp in the $hcps
        if (@failed_nodes == 0) {

            #all nodes succeeded --- no node in @$failed_nodes;
            return;
        }

        foreach my $node (@failed_nodes) {

            #for multiple, get the first hcp in the $hcps{$node}.
            my $hcp_list = $hcps->{$node}->{hcp};

            #print Dumper($hcp_list);
            my $thishcp = shift(@$hcp_list);
            if (!defined($thishcp)) {

                #if no hcp, count++;
                $count++;
                if ($count == @failed_nodes) {

                    # all the hcps of the nodes are tried. But still failed. so output the error msg and exit.
                    #print Dumper(\%failed_msg);
                    #prompt all the error msg.
                    foreach my $failed_node (@failed_nodes) {
                        my $msg = $failed_msg{$failed_node};
                        foreach my $item (@$msg) {
                            if ($item) {

                                #print Dumper($item);
                                $callback->($item);
                            }    # end of if
                        }    # end of foreach
                    }    #end of foreach
                         #output all the msgs of the failed nodes, so return
                    return;
                }    #end of if
                     #if $count != @failed_nodes, let's check next node
                next;
            }    #end of if
                 #print "thishcp:$thishcp\n";
                 #get the nodetype of hcp:
             #my $thishcp_type = xCAT::FSPUtils->getTypeOfNode($thishcp,$callback);
            my $thishcp_type = xCAT::DBobjUtils->getnodetype($thishcp, "ppc");
            if (!defined($thishcp_type)) {
                my %output = ();
                $output{node}->[0]->{name} = [$node];
                $output{node}->[0]->{data} = ["the type of $node\'s hcp is not defined in the 'ppc' table."];
                $output{errorcode} = '1';
                $callback->(\%output);
                next;
            }

            #print "lasthcp_type:$lasthcp_type ;thishcp_type:$thishcp_type\n";
            if (defined($lasthcp_type)) {
                if (($lasthcp_type =~ /^(hmc)$/ && $thishcp_type =~ /^(fsp|bpa|cec)$/) or (($lasthcp_type =~ /^(fsp|bpa|cec)$/) && ($thishcp_type =~ /^(hmc)$/))) {
                    $callback->({ data => ["the $node\'s hcp type is different from the other's in the specified noderange in the 'ppc' table."] });
                    return;
                }
            }

            $lasthcp_type = $thishcp_type;
            $hcps_will{$node} = $thishcp;
            push(@next, $node);

        }    #end of foreach
        my $request_new;
        %$request_new           = %$request;
        $request_new->{node}    = \@next;
        $request_new->{fsp_api} = 0;
        if ($lasthcp_type =~ /^(fsp|bpa|cec|frame)$/) {

            #my $fsp_api = check_fsp_api($request);
            #if($fsp_api == 0 ) {
            $request_new->{fsp_api} = 1;

            # }
        }

        #For mkhwconn ....
        if ($request->{hwtype} ne 'hmc') {
            $request_new->{hwtype} = $lasthcp_type;
        } else {
            $request_new->{fsp_api} = 0;
        }

        #print Dumper($request_new);
        @failed_nodes = ();
        if ($hcps->{maxnum} == 1 && $request_new->{command} !~ /^(rspconfig|rpower|reventlog)$/) {
            push @failed_nodes, "noloop"; # if each node only has one hcp, it will return immediately.
        }

        #For rspconfig to use ASMI
        if (defined($request->{enableASMI}) && ($request->{enableASMI} eq "1")) {
            $request_new->{fsp_api} = 0;
        }
        process_command($request_new, \%hcps_will, \@failed_nodes, \%failed_msg);

        #print "after result:\n";
        #print Dumper(\@failed_nodes);
        if ($lasthcp_type =~ /^(fsp|bpa|cec|frame)$/ &&
            $request->{hwtype} ne 'hmc' &&
            $request_new->{fsp_api} ne '0') {
            if ($request_new->{command} =~ /^(rspconfig|rpower|reventlog)$/) {
                my @enableASMI = xCAT::TableUtils->get_site_attribute("enableASMI");
                if (defined($enableASMI[0])) {
                    if (($request_new->{command} !~ /^rspconfig$/) ||
                        (ref($request_new->{method} eq 'HASH'))) {
                        $enableASMI[0] =~ tr/a-z/A-Z/;    # convert to upper
                        if (($enableASMI[0] eq "1") || ($enableASMI[0] eq "YES")) {

                            #through asmi ......
                            $request_new->{fsp_api} = 0;
                            if (@failed_nodes != 0) {
                                my @temp = @failed_nodes;
                                @failed_nodes = ();
                                $request_new->{node} = \@temp;
                                process_command($request_new, \%hcps_will, \@failed_nodes, \%failed_msg);
                            }                             #end of if
                        }    #end of if
                    }    # end of if
                }    #end of if
            }
        }    #end of if
    }    #end of while(1)
}

##########################################################################
# connect hmc via ssh and execute remote command
##########################################################################
sub sshcmds_on_hmc
{
    my $ip       = shift;
    my $user     = shift;
    my $password = shift;
    my @cmds     = @_;

    my %handled;
    my @data;
    my @exp;
    for my $cmd (@cmds)
    {
        if ($cmd =~ /(.+?)=(.*)/)
        {
            my ($command, $value) = ($1, $2);
            $handled{$command} = $value;
        }
    }
    my %request = (
        ppcretry   => 1,
        verbose    => 0,
        ppcmaxp    => 64,
        ppctimeout => 0,
        fsptimeout => 0,
        ppcretry   => 3,
        maxssh     => 8

    );

    my $valid_ip;
    foreach my $individual_ip (split /,/, $ip) {
        ################################
        # Get userid and password
        ################################
        my @cred = ($user, $password);
        $request{$individual_ip}{cred} = \@cred;

        @exp = xCAT::PPCcli::connect(\%request, 'hmc', $individual_ip);
        ####################################
        # Successfully connected
        ####################################
        if (ref($exp[0]) eq "Expect") {
            $valid_ip = $individual_ip;
            last;
        }
    }

    ########################################
    # Error connecting
    ########################################
    if (ref($exp[0]) ne "Expect") {
        return ([ 1, @cmds ]);
    }
    ########################################
    # Process specific command
    ########################################
    for my $cmd (keys %handled)
    {
        my $result;
        if ($cmd eq 'network_reset')
        {
            $result = xCAT::PPCcli::network_reset(\@exp, $valid_ip, $handled{$cmd});
            my $RC = shift(@$result);
        }
        push @data, @$result[0];
    }
    ########################################
    # Close connection to remote server
    ########################################
    xCAT::PPCcli::disconnect(\@exp);

    return ([ 0, undef, \@data ]);
}


sub check_fsp_api
{
    my $request = shift;

    #    my $fsp_api = "/opt/xcat/sbin/fsp-api";
    my $fsp_api = ($::XCATROOT) ? "$::XCATROOT/sbin/fsp-api" : "/opt/xcat/sbin/fsp-api";

    #my $libfsp  = "/usr/lib/libfsp.a";
    my $libfsp_aix = ($::XCATROOT) ? "$::XCATROOT/lib/libfsp.so" : "/opt/xcat/lib/libfsp.so";
    my $libfsp_linux = ($::XCATROOT) ? "$::XCATROOT/lib/libfsp.a" : "/opt/xcat/lib/libfsp.a";

    #    my $libfsp  = "/opt/xcat/lib/libfsp.a";
    #    my $libfsp    = ($::XCATROOT) ? "$::XCATROOT/lib/libfsp.a" : "/opt/xcat/lib/libfsp.a";
    #    my $hw_svr  = "/opt/csm/csmbin/hdwr_svr";

    my $msg = ();

    #    if((-e $fsp_api) && (-x $fsp_api)&& (-e $libfsp) && (-e $hw_svr)) {
    if ((-e $fsp_api) && (-x $fsp_api) && ((-e $libfsp_aix) || (-e $libfsp_linux))) {
        return 0;
    }

    return -1;
}

sub getHCPsOfNodes
{
    my $nodes    = shift;
    my $callback = shift;
    my $request  = shift;
    my %hcps;

    if ($request->{command} eq "mkhwconn" or $request->{command} eq "lshwconn" or $request->{command} eq "rmhwconn") {
        if (grep (/^-s$/, @{ $request->{arg} })) {

            my $ppctab = xCAT::Table->new('ppc');
            my %newhcp;
            if ($ppctab) {
                my $typeref = xCAT::DBobjUtils->getnodetype($nodes, "ppc");
                my $i = 0;
                unless ($request->{sfp}) {
                    for my $n (@$nodes) {
                        if ($$typeref{$n} =~ /^fsp|bpa$/) {
                            my $np = $ppctab->getNodeAttribs($n, [qw(parent)]);
                            if ($np) { # use parent(frame/cec)'s sfp attributes first,for high end machine with 2.5/2.6+ database
                                my $psfp = $ppctab->getNodeAttribs($np->{parent}, [qw(sfp)]);
                                $newhcp{$n}{hcp} = [ $psfp->{sfp} ] if ($psfp);
                            } else { # if the node don't have a parent,for low end machine with 2.5 database
                                my $psfp = $ppctab->getNodeAttribs($n, [qw(sfp)]);
                                $newhcp{$n}{hcp} = [ $psfp->{sfp} ] if ($psfp);
                            }
                        } else {
                            my $psfp = $ppctab->getNodeAttribs($n, [qw(sfp)]);
                            $newhcp{$n}{hcp} = [ $psfp->{sfp} ] if ($psfp);
                        }
                        $newhcp{$n}{num} = 1;
                    }
                    return \%newhcp;
                } else {
                    my $sfp = $request->{sfp};
                    my %sfphash;
                    for my $n (@$nodes) {

                        # record hcp
                        $newhcp{$n}{hcp} = [$sfp];
                        $newhcp{$n}{num} = 1;

                        # set the sfp attribute to the database
                        if ($$typeref{$n} =~ /^fsp|bpa$/) {
                            my $np = $ppctab->getNodeAttribs($n, [qw(parent)]);
                            $sfphash{$np}{sfp} = $sfp if ($np);
                        }
                        $sfphash{$n}{sfp} = $sfp;
                    }
                    $ppctab->setNodesAttribs(\%sfphash);
                    return \%newhcp;
                }
            } else {
                $callback->({ data => ["Could not open the ppc table"] });
                return undef;
            }
        }
    }


    $hcps{maxnum} = 0;

    #get hcp from ppc.
    foreach my $node (@$nodes) {

        #my $thishcp_type = xCAT::FSPUtils->getTypeOfNode($node, $callback);
        my $thishcp_type = xCAT::DBobjUtils->getnodetype($node, "ppc");
        if ($thishcp_type eq "hmc") {
            $hcps{$node}{hcp} = [$node];
            $hcps{$node}{num} = 1;
        } else {
            my $ppctab = xCAT::Table->new('ppc');
            unless ($ppctab) {
                $callback->({ data => ["Cannot open ppc table"] });
                return undef;
            }

            #xCAT::MsgUtils->message('E', "Failed to open table 'ppc'.") if ( ! $ppctab);
            my $hcp_hash = $ppctab->getNodeAttribs($node, [qw(hcp)]);
            my $hcp = $hcp_hash->{hcp};
            if (!$hcp) {

                #xCAT::MsgUtils->message('E', "Not found the hcp of $node");
                $callback->({ data => ["Not found the hcp of $node"] });
                return undef;
            }

            #print "hcp:\n";
            #print Dumper($hcp);
            my @h = split(",", $hcp);
            $hcps{$node}{hcp} = \@h;
            $hcps{$node}{num} = @h;
        }
        if ($hcps{maxnum} < $hcps{$node}{num}) {
            $hcps{maxnum} = $hcps{$node}{num};
        }
    }

    #print "in getHCPsOfNodes\n";
    #print Dumper(\%hcps);
    return \%hcps;
}



1;

