#!/usr/bin/perl
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
#  
#  
# Licensed Materials - Property of IBM 
#  
# (C) COPYRIGHT International Business Machines Corp. 2000,2006 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
# @(#)24	1.50  src/avs/fs/mmfs/ts/admin/mmdsh, mmfs, avs_rgpfs24, rgpfs240610b 2/12/06 12:34:29
#############################################################################
#
# Module:  mmdsh
#
#CPRY
# 5765-296 (C) Copyright IBM Corporation 1994, 2000
# Licensed Materials - Property of IBM
# All rights reserved.
# US Government Users Restricted Rights -
# Use, duplication or disclosure restricted by
# GSA ADP Schedule Contract with IBM Corp.
#CPRY
#
#############################################################################
#
# Description:  command for running commands on multiple nodes or
#               network connected hosts in parallel
#
#   mmdsh [-isv] [-f num] [-F hostlistfile] [-L listofhosts] [-I file] [-k]
#         [-N nodespecification] [-R reportfile] [-r] [command]
#
# Flags/parms:
#
# [-f num]    Specifies a fanout value used for concurrent execution.
#
# [-F hostlistfile]
#             Runs the command on the hosts in the specified hostlistfile.
#             The hostlistfile argument is the fully qualified name
#             of a file that lists the names of the desired hosts,
#             one host per line.
#
# [-i]        Displays the set of nodes on which the command will run
#             before command execution.
#
# [-I file]   Name of the file to be copied on the remote node prior
#             to executing the command.  The file must be in /var/mmfs/tmp.
#             This option can be specified twice.
#
# [-k]        Remove files specified with the -I flag before exiting.
#
# [-L listofhosts]
#             Runs the command on the specified list of hosts.
#             The listofhosts argument is a comma-separated list of hosts.
#
# [-N nodespecification]
#             Runs the command on the nodes in the given node specification.
#             The nodespecification argument can be a comma-separated list of
#             nodes, a node file, or a node class.  The nodes in the list or
#             the file can be specified as long or short admin or daemon
#             node names, node numbers, node number ranges, or IP addresses.
#
# [-R reportfile]
#             Reports the list of hosts removed from the working collective
#             when host verification (host ping) fails.  The report is written
#             to the specified file with one host per line.  The report is
#             generated only when combined with the '-v' flag.
#
# [-r]        Reports the list of hosts removed from the working collective
#             when host verification (host ping) fails.  The report is written
#             to standard error.  The report is generated only when combined
#             with the '-v' flag.
#
# [-s]        Suppresses the prepending of the hostname string to each line
#             of output generated by running the command on the remote host.
#
# [-v]        Verify that each host is reachable before adding it to the
#             set of nodes on which the command will run.
#
# [command]   command to be run on the remote hosts.
#             If the command is the reserved word "_SELECT_FROM_FILE_",
#             then the commands to be run on the different hosts are
#             expected to be in the file pointed to by the environment
#             variable mmdshCommandsFile.  Each line of this file consists
#             of hostname followed by a command string.
#
# Outputs:
#   The command is executed on the specified host(s).
#
# Examples:
#   mmdsh -F /nodes uname -a
#   mmdsh -L host1,host2,host3 ls
#   mmdsh -N quorumnodes date
#
#############################################################################

# Obtain valid "waitpid" flags.
use POSIX ":sys_wait_h";

$MMCOMMON="/usr/lpp/mmfs/bin/mmcommon";
$REL_HOSTNAME_Field=8;


#----------------------------------------------------------------------------
#
# Function:  Correct the return value from system() and etc.
#
#----------------------------------------------------------------------------
sub normalizeRC
{
  local ($rc);

  $rc = $_[0] + 0;
  if ($rc >= 256) { $rc = $rc / 256; }

  return $rc;

}  #---- end of function normalizeRC -----------------------------------


#----------------------------------------------------------------------------
#
# Function:  Print a message using the message catalog facilities.
#
# Input:     $_[0] - message number in catalog
#            $_[1] - message text formatting string
#            ...   - optional message parameters (up to 5)
#
#----------------------------------------------------------------------------

sub prtMsg
{
  local ($msgNum) = shift(@_);
  local ($p1)     = shift(@_);
  local ($p2)     = shift(@_);
  local ($p3)     = shift(@_);
  local ($p4)     = shift(@_);
  local ($p5)     = shift(@_);

  local ($msgTxt);
  local ($prog);

  chop($prog = `/bin/basename $0`);
	
  for ($msgNum) {

    if    (/^13$/)
    {
      $msgTxt="%s:  Incorrect option: %s";
    }
    elsif (/^36$/)
    {
      $msgTxt="%s:  %s flag specified twice.";
    }
    elsif (/^38$/)
    {
      $msgTxt="%s:  Invalid extra argument: %s";
    }
    elsif (/^40$/)
    {
      $msgTxt="%s:  Invalid integer for %s: %s";
    }
    elsif (/^153$/)
    {
      $msgTxt="%s:  Invalid value for %s flag";
    }
    elsif (/^168$/)
    {
      $msgTxt="%s:  The device name %s starts with a slash, but not /dev/.";
    }
    elsif (/^171$/)
    {
      $msgTxt="%s:  Unexpected error from %s. Return code: %s";
    }
    elsif (/^204$/)
    {
      $msgTxt="%s:  Missing argument after %s flag";
    }
    elsif (/^206$/)
    {
      $msgTxt="Command %s failed with return code %s.";
    }
    elsif (/^362$/)
    {
      $msgTxt="%s:  Nodes on which the command will be run:";
    }
    elsif (/^363$/)
    {
      $msgTxt="%s:  WCOLL (working collective) environment variable not set.";
    }
    elsif (/^364$/)
    {
      $msgTxt="%s:  Cannot open file %s.  Error string was:  %s";
    }
    elsif (/^365$/)
    {
      $msgTxt="%s:  %s remote shell process had return code %s.";
    }
    elsif (/^366$/)
    {
      $msgTxt="%s:  Caught SIG %s - terminating the child processes.";
    }
    elsif (/^367$/)
    {
      $msgTxt="%s:  There are no available nodes on which to run the command.";
    }
    elsif (/^368$/)
    {
      $msgTxt="%s:  Unable to pipe.  Error string was:  %s";
    }
    elsif (/^369$/)
    {
      $msgTxt="%s:  Unable to redirect %s.  Error string was:  %s";
    }
    elsif (/^422$/)
    {
      $msgTxt="%s:  Invalid or missing remote shell command:  %s";
    }
    elsif (/^423$/)
    {
      $msgTxt="%s:  Invalid or missing remote file copy command:  %s";
    }
    elsif (/^573$/)
    {
      $msgTxt="%s:  Node %s is not available to run the command.";
    }
    else            # should never happen
    {
      $msgTxt="\n%s:  An unknown error number %s was passed to prtMsg.";
      printf STDERR  "$msgTxt\n\n", $prog, $msgNum;
      return;
    }
  }

  if ($osName eq "AIX")
  {
    $msg=`$DSPMSG $msgNum \"$msgTxt\" \"$prog\" \"$p1\" \"$p2\" \"$p3\" \"$p4\" \"$p5\"`;
    printf STDERR  "$msg\n";
  }
  else
  {
    printf STDERR  "$msgTxt\n", $prog, $p1, $p2, $p3, $p4, $p5;
  }

}  #---- end of function prtMsg ----------------------------------------


#----------------------------------------------------------------------------
#
# read_pipes_and_wait
#
# Read STDOUT and STDERR on specified child rsh process pipes.
# Display to parent's STDOUT or STDERR.  If the '-s' flag is not
# specified, then host output is preceded by "hostname:  ".
#
# Output from STDERR is first written to an array, since there is
# a limit of 32k from STDERR before it ends up hanging.
#
# Reap completed children to allow additional child processes to be
# spawned for fanout maintenance.
#
# Parameters:
#   final_read -- true to "read/wait" on all specified children, false
#                 to only "read" currently available data for specified
#                 children and reap any completed children for fanout
#                 maintenance.
#   hosts      -- Array of hostnames with STDOUT/STDERR pipes to query.
#
#----------------------------------------------------------------------------

sub read_pipes_and_wait {

  my $final_read = shift;

  local(@hosts) = @_;

  local($fno,$fh,$rin,$rout,$num_fh,$host,$err_ctr,$array_updated,$offset);

  # Initialize the pipes read select vector.
  $rin = '';
  $num_fh = 0;
  foreach $host (@hosts) {
    if (($fno = fileno($r_out{$host})) ne '') {
      vec($rin, $fno, 1) = 1;
      $num_fh++;
    }
    if (($fno = fileno($r_err{$host})) ne '') {
      vec($rin, $fno, 1) = 1;
      $num_fh++;
    }
  }

  $err_ctr = 0;
  $array_updated = 0;

  READWAIT: {

    if ($num_fh > 0) {
      # Wait for outstanding I/O on read pipes.
      $nready = select($rout = $rin, undef, undef, undef);
      if ($nready > 0) {
        # At least one pipe has available data.
        foreach $host (@hosts) {

          # Process STDOUT I/O
          $fh = $r_out{$host};
          if (vec($rout, fileno($fh), 1)) {
            $_ = <$fh>;
            if (length($_)) {
              if ($sflag) {
               print STDOUT "$_";
              }
              else {
               print STDOUT "$host:  $_";
              }
            }
            else {
              # EOF has been reached
              vec($rin, fileno($fh), 1) = 0;
              $num_fh--;
              close($fh);
            }
          }

          # Process STDERR I/O.
          $fh = $r_err{$host};
          if (vec($rout, fileno($fh), 1)) {
            $_ = <$fh>;
            if (length($_)) {
              if ($sflag) {
                $stderr_array[$err_ctr] = $_;
              }
              else {
                $stderr_array[$err_ctr] = "$host:  $_";
              }
              $err_ctr++;
              $array_updated = 1;
            }
            else {
              # EOF has been reached
              vec($rin, fileno($fh), 1) = 0;
              $num_fh--;
              close($fh);
            }
          }
        }
      }

      # Reap any completed child process, in order to maintain fanout.
      if (($child_pid = waitpid(-1, &WNOHANG)) > 0) {

        # Obtain the return code and hostname for the child process.
        $child_rc = $? >> 8;
        foreach (keys %pid) {
          if ($pid{$_} == $child_pid) {
            $child_host = $_;
            last;
          }
        }

        # Process any outstanding STDOUT I/O for the deceased child.
        $fh = $r_out{$child_host};
        if (fileno($fh) != undef) {
          do {
            $_ = <$fh>;
            if (length($_))
            {
              if ($sflag) {
                print STDOUT "$_";
              }
              else {
                print STDOUT "$child_host:  $_";
              }
            }
          } while (length($_));

          # EOF has been reached
          vec($rin, fileno($fh), 1) = 0;
          $num_fh--;
          close($fh);
        }

        # Process any outstanding STDERR I/O for the deceased child.
        $fh = $r_err{$child_host};
        if (fileno($fh) != undef) {
          do {
            $_ = <$fh>;
            if (length($_)) {
              if ($sflag) {
                $stderr_array[$err_ctr] = $_;
              }
              else {
                $stderr_array[$err_ctr] = "$child_host:  $_";
              }
              $err_ctr++;
              $array_updated = 1;
            }
          } while (length($_));

          # EOF has been reached
          vec($rin, fileno($fh), 1) = 0;
          $num_fh--;
          close($fh);
        }

        if ($child_rc != 0) {
          if ($hostCount == 1) {
            $rcode = $child_rc;   # Pass back child's rc to mmdsh caller
          }
          else {
            $rcode++;
            #print STDERR "mmdsh: $child_host rsh had exit code $child_rc\n";
            &prtMsg(365, $child_host, $child_rc);
          }
          push(@goners,$child_host);
        }

        # Remove the deceased child from @hs so we don't continually try to add
        # it to the select vector as problems may occur with sufficiently large
        # numbers of nodes.
        $offset = 0;
        foreach $host (@hs) {
          if ($host eq $child_host) {
            splice(@hs, $offset, 1);
            last;
          }
          $offset++;
        }

        # Child terminated ==> decrement fanout
        $cur_fanout--;
      }

      if (($final_read eq true) && ($num_fh > 0) ||
          ($cur_fanout >= $fanout)) {
        # Final pipes data retrieval ==> Wait/retrieve until all pipes closed
        redo READWAIT;
      }
    }
  }

  # Print out accumulated STDERR output.
  if ($array_updated) {
    for ($prt_ctr=0; $prt_ctr < $err_ctr; $prt_ctr++) {
      print STDERR "$stderr_array[$prt_ctr]";
    }
  }

}  #---------- end of read_pipes_and_wait routine ---------------------------


#----------------------------------------------------------------------------
#
# get_command
#
# Return the command from the command line or stdin.
# Return 0 if no more to be read from command line or stdin.
# If the command is in double quotes, add a double quote to the beginning.
# If the command starts with '!', execute here and read in the next command.
#
#----------------------------------------------------------------------------

sub get_command {

  local($command);
  if ($done) {
    return(0);
  }
  if (@ARGV) {
    $done = 1;
    @dsh_ARGV = @ARGV;
    shift(@ARGV);
    #
    # Save the arguments in a scalar.  The return value
    # of this routine is assigned to a scalar variable.
    #
    $dsh_return = join(' ',@dsh_ARGV);
    return($dsh_return);
  }
  GET_COMMAND: {
    -t && print STDERR "mmdsh> ";
    $command = <STDIN>;
    if (!defined($command)  ||
      $command =~ /^\s*$/ || $command =~ /^\s*exit\s*$/) {
      return(0);
    } else {
      chop $command;
      # Add a double-quote to the beginning of the command if the user
      # entered the command with double-quotes.
      if ($command =~ /^\s*"(.*)/) {
        $command = '"' . $command;
      }
      if ($command =~ /^\s*!(.*)/) {
        &do_system($1);
        redo GET_COMMAND;
      } else {
        return($command) || redo GET_COMMAND;
      }
    }
  }

}  #---------- end of get_command routine ------------------------------


#----------------------------------------------------------------------------
#
# do_system
#
# Issue system() call with default signal handling.
# Return the executed command's exit code.
#
#----------------------------------------------------------------------------

sub do_system {

  local($command) = @_;
  local(%save_sig,$rc);
  %save_sig = %SIG;
  grep($_ = 'DEFAULT', %SIG);
  $rc = system("$command 2>&1") >> 8;
  %SIG = %save_sig;
  return($rc);

}  #---------- end of do_system routine ---------------------------------


#----------------------------------------------------------------------------
#
# display_help
#
# Display help information
#
#----------------------------------------------------------------------------

sub display_help {

  $usageMsg="
Usage:

  mmdsh [flags/parms] command

Flags/parms:

  -f number     fanout number to use for concurrent execution
  -F filename   nodelist file listing nodes on which to execute command
  -I filename   file to be staged on the remote node prior to executing
                  command; the file must be in /var/mmfs/tmp
  -k            remove the file specified with the -I flag before exiting
  -L n1,n2,n3   comma-separated list of nodes on which to execute command
  -N nodespec   node specificaton that consists of a node list, a node file,
                  or a node class on which to execute the command
  -s            suppress the prepending of the hostname string to each line
                  of output generated by running the command on the remote node
  -v            verify that nodes are reachable before adding them to the
                  set of nodes on which to run the command
  -i            display the set of nodes on which the command will run
                  before command execution
";
  print STDERR "$usageMsg\n";

}  #---------- end of display_help routine -----------------------------


#----------------------------------------------------------------------------
#
# add_wc
#
# Add a host to the working collective.
#
# Input parameter is the hostname to be added.
#
# Don't add a hostname if it is already in the working collective.
#
#----------------------------------------------------------------------------

sub add_wc {

   local($host) = @_;
   local($hostname);

   $host =~ s/\s//g;

   # If the host is already in the working collective, return to the caller
   #  without adding it to the collective again (avoid redundancies).
   foreach $hostname (@wc) {
      return if $hostname eq $host;
   }

   # If we made it this far, add the host to the working collective.
   push(@wc,$host);

}    #---------- end of add_wc routine ------------------------------------


#----------------------------------------------------------------------------
#
# add_wc_parallel
#
# Add hosts to the working collective in parallel via fork.
#
# Input parameter is the opened working collective file handle.
#
# If -v was specified, only add a host to the working collective
# if it is reachable via ping, or if the user said to add it
# even though it is not reachable.
#
# Don't add a hostname if it is already in the working collective.
#
#----------------------------------------------------------------------------

sub add_wc_parallel {

  local($filehandle) = @_;
  local(@hosts_seen, $child_host);

  # Unlink the temporary file here.  Otherwise, it may be
  # left behind if we run into a problem and exit early.
  if ($hlfile) {
    `/bin/rm -f $hlfile`;
  }

  if ($fanout) {      # set fanval from user value, environment, or default
    $fanval = $fanout;
  } else {
    unless ($fanval = $ENV{'FANOUT'}) {
      $fanval = $fanout_default;
    }
  }

  HOST: while ($host = <$filehandle>) {

    $host =~ /^\s*#/ && next;
    $host =~ /^\s*$/ && next;
    $host =~ /;/ && next;
    $host =~ /\S+\s+\S+/ && next;
    chop($host);

    # Remove any leading or trailing whitespace.
    ($host, $remainder) = split(" ", $host);

    # Prevent duplicates in the working collective.  Duplicates are culled
    # here to prevent the key collisions in the %pid hashtable, which can
    # cause unexpected behavior.  For example, defined, but empty hostnames
    # being added to the working collective, leading to "Unknown host"
    # messages from the remote shell during execution attempts.
    foreach (@hosts_seen) {
      if ($_ eq $host) {
        next HOST;
      }
    }
    push(@hosts_seen, $host);

    # Create filehandles for pipe ends.
    $Rout{$host} = "READ_STDOUT__" . $host;
    $Wout{$host} = "WRITE_STDOUT__" . $host;
    $Rerr{$host} = "READ_STDERR__" . $host;
    $Werr{$host} = "WRITE_STDERR__" . $host;

    # Open pipes for this host's stdout and stderr from ping.
    if (pipe($Rout{$host}, $Wout{$host}) < 0) {
      #die "mmdsh: Couldn't pipe: $!\n";
      &prtMsg(368, "\'$!\'");
      exit(-1);
    }
    if (pipe($Rerr{$host}, $Werr{$host}) < 0) {
      #die "mmdsh: Couldn't pipe: $!\n";
      &prtMsg(368, "\'$!\'");
      exit(-1);
    }

    # Fork a child to execute the ping to the current host to be added.
    ADDFORK: {

      if ($pid{$host} = fork) {

        # parent code -
        # close unneeded ends of pipes
        # parent will wait for child processes if fanout limit reached

        close($Wout{$host});
        close($Werr{$host});

        if (++$cur_fanout >= $fanval) {
          &wait_for_kids_and_add_hosts(false);
        }

      } elsif (defined $pid{$host}) {

        # child code -
        # close unneeded ends of pipes
        # redirect stdout and stderr to output pipes
        # exec the ping command
        # stdout/stderr will go to pipes to be read by parent

        close($Rout{$host});
        close($Rerr{$host});

        # If the -v flag was specified, check whether the host can be ping'd.
        # If the 1st ping fails, try a few more times to avoid false node down.
        if ($verify) {
          $pingCount = 0;
          $pingRc = 1;
          while ($pingRc != 0 && ++$pingCount <= $maxPingCount) {
            `$ping -w $pingTimeout -c 1 $host 2>/dev/null`;
            $pingRc = $? >> 8;
            if ($pingRc > 0 && $iflag) {
              printf STDERR "ping %s failed (%s); pingRc=%s\n", $host, $pingCount, $pingRc;
            }
            ++$pingTimeout;
          }
        }

        unless (open(STDOUT, ">&$Wout{$host}")) {
          #die "mmdsh: Cannot redirect STDOUT: $!\n";
          &prtMsg(369, 'STDOUT', "\'$!\'");
          exit(-1);
        }
        unless (open(STDERR, ">&$Werr{$host}")) {
          #die "mmdsh: Cannot redirect STDERR: $!\n";
          &prtMsg(369, 'STDERR', "\'$!\'");
          exit(-1);
        }

        select(STDOUT); $| = 1;
        select(STDERR); $| = 1;

        exit($pingRc);

      } else {

        # Try again.  The fork must have failed due to a resource problem.

        sleep 5;
        redo ADDFORK;
      }
    }
  }

  # Parent continues here after forking all the children for this command.
  # Get the results of any remaining ping's (the number of hosts in the
  # working collective may not be a multiple of the fanout value).
  # Get rid of any hosts that have failed, reporting all removed hosts as
  # directed by the [-R reportfile] and [-r] flags.
  &wait_for_kids_and_add_hosts(true);
  if ($reportFile && ($#goners >= 0)) {
    # Write removed hosts to specified file.  The file is not created if
    # the goners array is empty (i.e., $#goners = -1).
    open(REPORTFILE, ">$reportFile");
    foreach $child_host (@goners) {
      print REPORTFILE "$child_host\n";
    }
    close(REPORTFILE);
  }
  if ($rflag) {
    # Write removed hosts to standard error.
    foreach $child_host (@goners) {
      print STDERR "$child_host\n";
    }
  }
  &delete_hosts;
  unless ($done) {
    $cur_fanout = 0;
    &check_wc;
  }

}  #---------- end of add_wc_parallel routine ----------------------------


#----------------------------------------------------------------------------
#
# parse
#
# Parse the command line.
#
#----------------------------------------------------------------------------

sub parse {

   local(@indices,@temp,$Findex,$findex,$Iindex,$Lindex,$Nindex);
   local($fn,$wfile,$host,@hostlist,$ht,$hl,$Narg,$nodesToUse);

   if ($ARGV[0] eq "-\?") {
      &display_help;
      exit;
   }

   while ($ARGV[0] =~ /^-/) {    # while current parm starts with "-"

      if ($ARGV[0] =~ /[FfILNR](\S+)/) { # if flag is one of F, f, I, L, N, or R
                                         # and there is no whitespace
                                         # immediately after the flag
         $Findex = index($ARGV[0],"F");
         $findex = index($ARGV[0],"f");
         $Iindex = index($ARGV[0],"I");
         $Lindex = index($ARGV[0],"L");
         $Nindex = index($ARGV[0],"N");
         $Rindex = index($ARGV[0],"R");
         @indices = ($Findex, $findex, $Iindex, $Lindex, $Nindex, $Rindex);

         @indices = sort @indices;
         @temp = @indices;
         foreach (@temp) {            # loop moves parm pos value to indices[0]
            $_ == -1 && shift(@indices);
         }

         if ($indices[0] == $Findex) {  # if the flag was "F"
            if (!$fn) {
               $fn = $1;
               unless ($wfile = $fn) {
                  #die "mmdsh: Missing argument\n";
                  &prtMsg(204, "-F");
                  &display_help;
                  exit(-1);
               }
            } else {
               #die "mmdsh:  F flag specified twice.\n";
               &prtMsg(36, "F");
               &display_help;
               exit(-1);
            }

         } elsif ($indices[0] == $findex) {  # if the flag was "f"
            if (!$fanout) {
               $fanout = $1;
               if ($fanout =~ /\D/) {
                  #die "mmdsh: Incorrect argument - $fanout\n";
                  &prtMsg(40, "-f", $fanout);
                  &display_help;
                  exit(-1);
               }
            } else {
               #die "mmdsh:  f flag specified twice.\n";
               &prtMsg(36, "f");
               &display_help;
               exit(-1);
            }

         } elsif ($indices[0] == $Iindex) {  # if the flag was "I"
            if (!$stageFile) {
               $stageFile = $1;
               unless ($fileToCopy = $stageFile) {
                  #die "mmdsh: Missing argument\n";
                  &prtMsg(204, "-I");
                  &display_help;
                  exit(-1);
               }
            } elsif (!$stageFile2) {
               $stageFile2 = $1;
               unless ($fileToCopy2 = $stageFile2) {
                  #die "mmdsh: Missing argument\n";
                  &prtMsg(204, "-I");
                  &display_help;
                  exit(-1);
               }
            } else {
               #die "mmdsh:  I flag specified twice.\n";
               &prtMsg(36, "I");
               &display_help;
               exit(-1);
            }

         } elsif ($indices[0] == $Lindex) {  # if the flag was "L"
            if (!$hl) {
               $hl = $1;
               if ($hl =~ /^,|,,|,$/) {
                  #die "Incorrect argument - $hl\n";
                  &prtMsg(153, "-L");
                  &display_help;
                  exit(-1);
               }
               if ($hl eq "-") {
                  # Get names from stdin
                  while (<STDIN>) {
                     /^\s*#/ && next;
                     /^\s*$/ && next;
                     /;/ && next;
                     /\S+ \S+/ && next;
                     s/ //g;
                     chop;
                     push(@hostlist,$_);
                  }
               } else {
                  @hostlist = split(/,/,$hl);
               }
            } else {
               #die "mmdsh:  L flag specified twice.\n";
               &prtMsg(36, "L");
               &display_help;
               exit(-1);
            }

         } elsif ($indices[0] == $Nindex) {  # if the flag was "N"
            if (!$Narg) {
               $Narg = $1;
               unless ($nodesToUse = $Narg) {
                  #die "mmdsh: Missing argument\n";
                  &prtMsg(204, "-N");
                  &display_help;
                  exit(-1);
               }
            } else {
               #die "mmdsh:  N flag specified twice.\n";
               &prtMsg(36, "N");
               &display_help;
               exit(-1);
            }

         } elsif ($indices[0] == $Rindex) {  # if the flag was "R"
            if (!$reportFile) {
               unless ($reportFile = $1) {
                  #die "mmdsh: Missing argument\n";
                  &prtMsg(204, "-R");
                  &display_help;
                  exit(-1);
               }
            } else {
               #die "mmdsh:  R flag specified twice.\n";
               &prtMsg(36, "R");
               &display_help;
               exit(-1);
            }
         }
         $ARGV[0] = substr($ARGV[0], 0, $indices[0] + 1);

      } elsif ($ARGV[0] =~ /F$/) {  # otherwise, if the flag is an "F"
                                    #  followed by whitespace
         if (!$fn) {
            $fn = $ARGV[1];
            unless ($wfile = $fn) {
               #die "mmdsh: Missing argument\n";
               &prtMsg(204, "-F");
               &display_help;
               exit(-1);
            }
            $shiftflag++;
         } else {
            #die "mmdsh:  F flag specified twice.\n";
            &prtMsg(36, "F");
            &display_help;
            exit(-1);
         }

      } elsif ($ARGV[0] =~ /f$/) {  # otherwise, if the flag is an "f"
                                    #  followed by whitespace
         if (!$fanout) {
            $fanout = $ARGV[1];
            unless ($fanout) {
               #die "mmdsh: Missing argument\n";
               &prtMsg(204, "-f");
               &display_help;
               exit(-1);
            }
            if ($fanout =~ /\D/) {
               #die "mmdsh: Incorrect argument - $fanout\n";
               &prtMsg(40, "-f", $fanout);
               &display_help;
               exit(-1);
            }
            $shiftflag++;
         } else {
            #die "mmdsh:  f flag specified twice.\n";
            &prtMsg(36, "f");
            &display_help;
            exit(-1);
         }

      } elsif ($ARGV[0] =~ /I$/) {  # otherwise, if the flag is an "I"
                                    #  followed by whitespace
         if (!$stageFile) {
            $stageFile = $ARGV[1];
            unless ($fileToCopy = $stageFile) {
               #die "mmdsh: Missing argument\n";
               &prtMsg(204, "-I");
               &display_help;
               exit(-1);
            }
            $shiftflag++;
         } elsif (!$stageFile2) {
            $stageFile2 = $ARGV[1];
            unless ($fileToCopy2 = $stageFile2) {
               #die "mmdsh: Missing argument\n";
               &prtMsg(204, "-I");
               &display_help;
               exit(-1);
            }
            $shiftflag++;
         } else {
            #die "mmdsh:  I flag specified twice.\n";
            &prtMsg(36, "I");
            &display_help;
            exit(-1);
         }

      } elsif ($ARGV[0] =~ /L$/) {  # otherwise, if the flag is an "L"
                                    #  followed by whitespace
         if (!$hl) {
            $hl = $ARGV[1];
            unless ($hl) {
              #die "mmdsh: Missing argument\n";
               &prtMsg(204, "-L");
               &display_help;
               exit(-1);
            }
            if ($hl =~ /^,|,,|,$/) {
               #die "Incorrect argument - $hl\n";
               &prtMsg(153, "-L");
               &display_help;
               exit(-1);
            }
            if ($hl eq "-") {
               while (<STDIN>) {
                  /^\s*#/ && next;
                  /^\s*$/ && next;
                  /;/ && next;
                  /\S+ \S+/ && next;
                  s/ //g;
                  chop;
                  push(@hostlist,$_);
               }
            } else {
               @hostlist = split(/,/,$hl);
            }
            $shiftflag++;
         } else {
            #die "mmdsh:  L flag specified twice.\n";
            &prtMsg(36, "L");
            &display_help;
            exit(-1);
         }

      } elsif ($ARGV[0] =~ /N$/) {  # otherwise, if the flag is an "N"
                                    #  followed by whitespace
         if (!$Narg) {
            $Narg = $ARGV[1];
            unless ($nodesToUse = $Narg) {
               #die "mmdsh: Missing argument\n";
               &prtMsg(204, "-N");
               &display_help;
               exit(-1);
            }
            $shiftflag++;
         } else {
            #die "mmdsh:  N flag specified twice.\n";
            &prtMsg(36, "N");
            &display_help;
            exit(-1);
         }

      } elsif ($ARGV[0] =~ /R$/) {  # otherwise, if the flag is an "R"
                                    #  followed by whitespace
         if (!$reportFile) {
            unless ($reportFile = $ARGV[1]) {
              #die "mmdsh: Missing argument\n";
              &prtMsg(204, "-R");
              &display_help;
              exit(-1);
            }
            $shiftflag++;
         } else {
            #die "mmdsh:  R flag specified twice.\n";
            &prtMsg(36, "R");
            &display_help;
            exit(-1);
         }
      }  # end of if ($ARGV[0] =~ /[FfILNR](\S+)/)

      if (index($ARGV[0], 'i') >= $[) {   # if there is an "i" flag present
         if (rindex($ARGV[0], 'i') == index($ARGV[0], 'i')) {
            if ($iflag++) {
               #die "mmdsh:  i flag specified twice.\n";
               &prtMsg(36, "i");
               &display_help;
               exit(-1);
            }
         } else {
            #die "mmdsh:  i flag specified twice.\n";
            &prtMsg(36, "i");
            &display_help;
            exit(-1);
         }
      }

      if (index($ARGV[0], 'k') >= $[) {   # if there is a "k" flag present
         if (rindex($ARGV[0], 'k') == index($ARGV[0], 'k')) {
            if ($kflag++) {
               #die "mmdsh:  k flag specified twice.\n";
               &prtMsg(36, "k");
               &display_help;
               exit(-1);
            }
         } else {
            #die "mmdsh:  k flag specified twice.\n";
            &prtMsg(36, "k");
            &display_help;
            exit(-1);
         }
      }

      if (index($ARGV[0], 'r') >= $[) {   # if there is an "r" flag present
         if (rindex($ARGV[0], 'r') == index($ARGV[0], 'r')) {
            if ($rflag++) {
               #die "mmdsh:  r flag specified twice.\n";
               &prtMsg(36, "r");
               &display_help;
               exit(-1);
            }
         } else {
            #die "mmdsh:  r flag specified twice.\n";
            &prtMsg(36, "r");
            &display_help;
            exit(-1);
         }
      }

      if (index($ARGV[0], 's') >= $[) {   # if there is an "s" flag present
         if (rindex($ARGV[0], 's') == index($ARGV[0], 's')) {
            if ($sflag++) {
               #die "mmdsh:  s flag specified twice.\n";
               &prtMsg(36, "s");
               &display_help;
               exit(-1);
            }
         } else {
            #die "mmdsh:  s flag specified twice.\n";
            &prtMsg(36, "s");
            &display_help;
            exit(-1);
         }
      }

      if (index($ARGV[0], 'v') >= $[) {   # if there is a "v" flag present
         if (rindex($ARGV[0], 'v') == index($ARGV[0], 'v')) {
            if ($verify++) {
               #die "mmdsh:  v flag specified twice.\n";
               &prtMsg(36, "v");
               &display_help;
               exit(-1);
            }
         } else {
            #die "mmdsh:  v flag specified twice.\n";
            &prtMsg(36, "v");
            &display_help;
            exit(-1);
         }
      }

      if ($ARGV[0] =~ /^-.*([^fFiIkLNRrsv]).*/) {  # fail if any unknown flags
         #die "mmdsh: Incorrect option - $1\n";
         &prtMsg(13, $1);
         &display_help;
         exit(-1);
      }

      if ($ARGV[0] =~ /^-$/) {                   # fail if no flag after "-"
         #die "mmdsh: Missing option\n";
         &prtMsg(168);
         &display_help;
         exit(-1);
      }

      shift(@ARGV);      # shift to next parameter
      if ($shiftflag) {
         shift(@ARGV);   # shift again if parameter pertains to previous flag
         $shiftflag--;   # reset the shift flag
      }
   }

   # Process any hosts that were specified via the -L flag.
   if (@hostlist) {
      $wfound++;
      # If the -v flag was specified, call routine to
      # only add pingable hosts; otherwise, add them all.
      if ($verify) {
         # Put the hosts specified by the "L" flag into a temporary file.
         $hlfile = "/var/mmfs/tmp/hostlistFile.mmdsh.$$";
         open(HLFILE, ">$hlfile");
         foreach $ht (@hostlist) {
            print HLFILE "$ht\n";
            $hostCount++;
            $targetHost = $ht;
         }
         close(HLFILE);

         # Pass the created file to routine that only adds pingable hosts.
         # The temp file is cleaned up inside the add_wc_parallel routine.
         unless (open(HLFILE, "$hlfile")) {
            #die "mmdsh: Cannot open hostlist file $hlfile: $!\n";
            &prtMsg(364, $hlfile, "\'$!\'");
            `/bin/rm -f $hlfile`;
            exit(-1);
         }
         &add_wc_parallel(HLFILE);
         close(HLFILE);
         `/bin/rm -f $hlfile`;

      } else {
         foreach $ht (@hostlist) {  # add any hosts specified by the "L" flag
            &add_wc($ht);
            $hostCount++;
            $targetHost = $ht;
         }
      }
   }

   # Process any hosts that were specified via the -F flag.
   if ($wfile) {
      $wfound++;
      unless (open(WCFILE, $wfile)) {
         #die "mmdsh: Cannot open working collective file $wfile: $!\n";
         &prtMsg(364, $wfile, "\'$!\'");
         exit(-1);
      }
      # If the -v flag was specified, call routine to
      # only add pingable hosts; otherwise, add them all.
      if ($verify) {
         &add_wc_parallel(WCFILE);
      } else {
         while ($new_host = <WCFILE>) {
            $new_host =~ /^\s*#/ && next;
            $new_host =~ /^\s*$/ && next;
            $new_host =~ /;/ && next;
            $new_host =~ /\S+\s+\S+/ && next;
            chop($new_host);
            &add_wc($new_host);
         }
      }
      close(WCFILE);
   }

   # Process any nodes that were specified via the -N flag.
   if ($nodesToUse) {
      # Convert the node specification into a verified file of nodes.
      $hlfile = "/var/mmfs/tmp/hostlistFile.mmdsh.$$";
      $cmd = "$MMCOMMON run createVerifiedNodefile $nodesToUse $REL_HOSTNAME_Field no $hlfile";
      if ($rc = system($cmd))
      {
         $rc = &normalizeRC($rc);
         &prtMsg(206, $rc);
         `/bin/rm -f $hlfile`;
         exit(-1);
      }

      # Process the nodes in the file that was created.
      if ($hlfile) {
         $wfound++;
         unless (open(HLFILE, $hlfile)) {
            #die "mmdsh: Cannot open node file $hlfile: $!\n";
            &prtMsg(364, $hlfile, "\'$!\'");
            exit(-1);
         }
         # If the -v flag was specified, call routine to
         # only add pingable hosts; otherwise, add them all.
         if ($verify) {
            &add_wc_parallel(HLFILE);
         } else {
            while ($new_host = <HLFILE>) {
               $new_host =~ /^\s*#/ && next;
               $new_host =~ /^\s*$/ && next;
               $new_host =~ /;/ && next;
               $new_host =~ /\S+\s+\S+/ && next;
               chop($new_host);
               &add_wc($new_host);
            }
         }
         close(HLFILE);
         `/bin/rm -f $hlfile`;
      }
   }

   # Verify the files to be staged exist and are in a subdir of /var/mmfs.
   if ($stageFile) {
     unless ( $fileToCopy =~ /^\/var\/mmfs\//) {
       die "mmdsh: Files may only be copied to /var/mmfs.\n";
     }
     unless ( -f $fileToCopy ) {
       #die "mmdsh: File $fileToCopy not found\n";
       &prtMsg(364, $fileToCopy, "\'$!\'");
       exit(-1);
     }
   }

   if ($stageFile2) {
     unless ( $fileToCopy2 =~ /^\/var\/mmfs\//) {
       die "mmdsh: Files may only be copied to /var/mmfs.\n";
     }
     unless ( -f $fileToCopy2 ) {
       #die "mmdsh: File $fileToCopy2 not found\n";
       &prtMsg(364, $fileToCopy2, "\'$!\'");
       exit(-1);
     }
   }

}  #---------- end of parse routine ------------------------------------


#----------------------------------------------------------------------------
#
# set_defaults
#
# Set default values for those values that have not been specified.
# If fanout was not specified, it is set to the default fanout value.
# If login name was not specified, it is set to that of the current user.
#
#----------------------------------------------------------------------------

sub set_defaults {

  unless ($fanout) {
    unless ($fanout = $ENV{'FANOUT'}) {
      $fanout = $fanout_default;
    }
  }
  unless ($login) {
    $login = (getpwuid($<))[0];
  }

}  #---------- end of set_defaults routine -----------------------------


#----------------------------------------------------------------------------
#
# readem
#
# Read the stdout and stderr pipes for all hosts in the current fanout.
#
#----------------------------------------------------------------------------

sub readem {

  local(@h) = @_;
  local($host);
  foreach $host (@h) {
    &read_pipes_and_wait(true, $host);
  }
  @h = ();

}  #---------- end of readem routine ---------------------------------


#----------------------------------------------------------------------------
#
# display_wc
#
# If requested by means of the -i option, display the set of nodes
# on which the command will be run.
#
#----------------------------------------------------------------------------

sub display_wc {

  local($i);
  if ($iflag) {
    #print STDOUT "Nodes on which the command will be run:\n";
    &prtMsg(362);
    $i = 0;
    while ($i <= $#wc) {
      printf STDERR "%-19.18s", $wc[$i];
      printf STDERR "%-19.18s", $wc[$i+1];
      printf STDERR "%-19.18s", $wc[$i+2];
      printf STDERR "%-19.18s\n", $wc[$i+3];
      $i = $i + 4;
    }
  }

}  #---------- end of display_wc routine -------------------------------


#----------------------------------------------------------------------------
#
# get_wc
#
# Determine the working collective, if not already obtained from command line.
# Look for filename in $WCOLL containing the hostnames, one per line.
#
#----------------------------------------------------------------------------

sub get_wc {

  local($wfile,$new_host);
  if (!@wc && !$wfound) {
    unless ($wfile = $ENV{'WCOLL'}) {
      #die "mmdsh: Working collective environment variable not set\n";
      &prtMsg(363);
      exit(-1);
    }
    unless (open(WCFILE, $wfile)) {
      #die "mmdsh: Cannot open working collective file $wfile: $!\n";
      &prtMsg(364, $wfile, "\'$!\'");
      exit(-1);
    }
    if ($verify) {
      &add_wc_parallel(WCFILE);
    } else {
      while ($new_host = <WCFILE>) {
        $new_host =~ /^\s*#/ && next;
        $new_host =~ /^\s*$/ && next;
        $new_host =~ /;/ && next;
        $new_host =~ /\S+\s+\S+/ && next;
        chop($new_host);
        &add_wc($new_host);
      }
    }
    close(WCFILE);
  }

}  #---------- end of get_wc routine -----------------------------------


#----------------------------------------------------------------------------
#
# set_signals
#
# HUP is ignored in the mmdsh parent and its exec'ed rsh children.
#
# STOP, CONT, and TSTP are defaulted - this means that they work on the
# parent, but are ignored (not propagated to) the exec'ed rsh children
# or the remote processes.
#
# Set the signal handler for all other signals.
# The signals will be propagated to the exec'd children and then
# the default action will be taken in the parent.
#
# rsh will propagate TERM, QUIT, and INT to the remote processes.
#
#----------------------------------------------------------------------------

sub set_signals {

  # Default STOP, CONT, TSTP signal handling
  $SIG{'STOP'} = 'DEFAULT';
  $SIG{'CONT'} = 'DEFAULT';
  $SIG{'TSTP'} = 'DEFAULT';

  # Propagate signals to forked kids.
  $SIG{'TERM'} = 'infanticide';
  $SIG{'QUIT'} = 'infanticide';
  $SIG{'INT'}  = 'infanticide';
  $SIG{'ABRT'} = 'infanticide';
  $SIG{'ALRM'} = 'infanticide';
  $SIG{'FPE'}  = 'infanticide';
  $SIG{'ILL'}  = 'infanticide';
  $SIG{'PIPE'} = 'infanticide';
  $SIG{'SEGV'} = 'infanticide';
  $SIG{'USR1'} = 'infanticide';
  $SIG{'USR2'} = 'infanticide';
  $SIG{'TTIN'} = 'infanticide';
  $SIG{'TTOU'} = 'infanticide';
  $SIG{'BUS'}  = 'infanticide';

}  #---------- end of set_signals routine -----------------------------


#----------------------------------------------------------------------------
#
# wait_for_kids_and_add_hosts
#
# When a child dies, it must be an exit after the end of his ping.
# If a negative return code, the ping failed.  Check whether we
# should give up on this host and eliminate him from the collective.
# Add the kid's host to the working collective if the ping succeeded
# or if we were told to add him anyway.
#
# Parameters:
#   wait_all -- true to "wait" on all active children, false to only "wait"
#               for completed children (fanout maintenance).
#
#----------------------------------------------------------------------------

sub wait_for_kids_and_add_hosts {

  my $wait_all = shift;

  local($child_pid, $child_rc, $child_host, $host_found, $wait_opt);

  if ($wait_all eq true) {
    # Wait for all children ==> Blocking waitpid
    $wait_opt = 0;
  }
  else {
    # Wait for completed children only ==> Non-blocking waitpid
    $wait_opt = &WNOHANG;
  }

  # Wait for any children and process accordingly.  We will wait for at
  # least one child for both "wait some" and "wait all" cases as the design
  # point is to only call this routine when the fanout limit has been reached.
  if (($child_pid = wait) != -1) {

    PINGWAIT: {

      # Obtain the return code and hostname for the child process.
      $child_rc = $? >> 8;
      foreach (keys %pid) {
        if ($pid{$_} == $child_pid) {
          $child_host = $_;
          last;
        }
      }

      # Close the pipes used for collecting data from the ping.
      close($Rout{$child_host});
      close($Rerr{$child_host});

      # If the ping failed, assume the host is not reachable;
      #  don't add it to the collective unless told to include it anyway.
      if ($child_rc != 0) {
        push(@goners,$child_host);
      }
      else {
        # Host reachable --> Add to collective (duplicates previously culled).
        push(@wc,$child_host);
      }

      # Child harvested ==> Decrement cur_fanout
      $cur_fanout--;

      # Waitpid for any child.  For "wait some" we will perform a non-blocking
      # child wait, terminating processing if none completed.  For "wait all"
      # we will perform a blocking wait, terminating processing if no children
      # remain.

      $child_pid = waitpid(-1, $wait_opt);
      if ((($wait_all eq false) && ($child_pid > 0)) ||
          (($wait_all eq true) && ($child_pid != -1)))
      {
        redo PINGWAIT;
      }
    }
  }

}  #---------- end of wait_for_kids_and_add_hosts routine --------------


#----------------------------------------------------------------------------
#
# delete_hosts
#
# Called if any hosts don't respond.
# Remove them from the working collective unless the -c flag was set.
# Input is the hostnames to remove from the working collective.
#
#----------------------------------------------------------------------------

sub delete_hosts {

  local($child_host,$h,$host_count);

  foreach $child_host (@goners) {
    $host_count = 0;
    foreach $h (@wc) {
      if ($h eq $child_host) {
        splice(@wc, $host_count, 1);
      }
      $host_count++;
    }
  }

}  #---------- end of delete_hosts routine -----------------------------


#----------------------------------------------------------------------------
#
# infanticide
#
# User has signaled the mmdsh parent - propagate TERM, INT, or QUIT to children.
# (Note - TERM, INT, and QUIT will be propagated to remote processes by rsh).
# Signal any children with SIGTERM if signal is not one of the above.
# Wait for children to manage output and prevent zombies.
# Signal self after setting default signal-handling for self.
# If still alive, exit.
# Input is the signal type.
#
#----------------------------------------------------------------------------

sub infanticide {

  local($sig) = @_;
  local($kid_sig);
  #print STDERR "mmdsh: Caught SIG$sig - terminating the kids\n";
  &prtMsg(366, $sig);
  if ($sig ne 'QUIT' && $sig ne 'INT' && $sig ne 'TERM') {
    $kid_sig = 'TERM';
    $SIG{'TERM'} = 'IGNORE';
  } else {
    $kid_sig = $sig;
  }
  $SIG{$sig} = 'DEFAULT';
  kill $kid_sig, (values %pid);
  &read_pipes_and_wait(true, @hs);
  kill $sig, $$;
  exit($rcode);

}       #---------- end of infanticide routine ----------------------------


#----------------------------------------------------------------------------
#
# check_wc
#
# Check to ensure that there are hosts in our working collective.
#
#----------------------------------------------------------------------------

sub check_wc {

  if (!@wc) {
    if ($hostCount == 1) {
      #print STDERR "mmdsh:  Node $targetHost is not available to run the command.";
      &prtMsg(573, $targetHost);
      exit(80);   # MM_HostDown
    }
    else {
      #print STDERR "There are no available nodes on which to run the command."
      &prtMsg(367);
      exit(++$rcode);
    }
  }

}     #---------- end of check_wc routine ---------------------------------


#----------------------------------------------------------------------------
#
# Mainline program
#
# The program continues while there are commands to distribute to the working
# collective via rsh.  Children are forked for each host, up to the fanout
# limit.  Stdout and stderr are piped back from the children to the parent
# who will display the results of the execed rsh's to the parent's stdout
# and stderr.
#
#----------------------------------------------------------------------------

# Determine the execution environment and initialize variables.
chop($osName=`/bin/uname -s`);
if ($osName eq "AIX") {
   $grep="/usr/bin/grep";
   $ping="/usr/sbin/ping";
   $DSPMSG="/usr/bin/dspmsg -s 32 mmfs.cat";
}
elsif ($osName eq "Linux") {
   $grep="/bin/grep";
   $ping="/bin/ping";
}
else {
   print STDERR  "mmdsh:  Unknown execution environment $osName\n";
   exit(-1);
}
$pingTimeout  = 2;       # initial single ping timeout period
$maxPingCount = 3;       # number of pings before declaring a node dead
$fanout_default = 64;    # default fanout value

# Flush output filesystem buffers after each write.
select(STDOUT); $| = 1;
select(STDERR); $| = 1;

# Get the set environment locales.
use locale;
$lang=$ENV{'LANG'};
$lcall=$ENV{'LC_ALL'};
$lccol=$ENV{'LC_COLLATE'};
$lctyp=$ENV{'LC_TYPE'};
$lcmon=$ENV{'LC_MONETARY'};
$lcnum=$ENV{'LC_NUMERIC'};
$lctim=$ENV{'LC_TIME'};
$lcmsg=$ENV{'LC_MESSAGES'};

$mmmode=$ENV{'MMMODE'};
$environmentType=$ENV{'environmentType'};
$rshPath=$ENV{'GPFS_rshPath'};
$rcpPath=$ENV{'GPFS_rcpPath'};
$mmTrace=$ENV{'mmScriptTrace'};

if ($rshPath eq "" || $rshPath eq "_DEFAULT_") {
   $rshPath="/usr/bin/rsh";
}
elsif ( ! -x $rshPath) {
#esjlrm - fix the above to strip any options
   #die "mmdsh: Invalid or missing remote shell command: $rshPath\n";
   &prtMsg(422, $rshPath);
   exit(-1);
}

if ($rcpPath eq "" || $rcpPath eq "_DEFAULT_") {
   $rcpPath="/usr/bin/rcp";
}
elsif ( ! -x $rcpPath) {
#esjlrm - fix the above to strip any options
   #die "mmdsh: Invalid or missing remote file copy command: $rcpPath\n";
   &prtMsg(423, $rcpPath);
   exit(-1);
}

$exportcmd = "export LANG=$lang; export LC_ALL=$lcall;  export LC_COLLATE=$lccol; export LC_TYPE=$lctyp; export LC_MONETARY=$lcmon; export LC_NUMERIC=$lcnum; export LC_TIME=$lctim; export LC_MESSAGES=$lcmsg; export MMMODE=$mmmode; export environmentType=$environmentType;  export GPFS_rshPath=$rshPath; export GPFS_rcpPath=$rcpPath; export mmScriptTrace=$mmTrace; ";


# Set signal handling for forked (child) processes,
#  parse the command line, and determine our working collective.
&set_signals;
&parse;
&get_wc;
&display_wc;
&set_defaults;
&check_wc;

# Perform each command on the nodes in our working collective.
while ($command = &get_command) {

   if ($iflag) {
     print STDERR "mmdsh: Command to run: $command\n\n";
   }

   # If the special command _SELECT_FROM_FILE_ is specified,
   # the command to be executed by each of the nodes is given
   # in the file specified in the mmdshCommandsFile variable.
   if ($command eq "_SELECT_FROM_FILE_") {
     $selectCommandFromFile++;
     unless ($mmdshCommandsFile = $ENV{'mmdshCommandsFile'}) {
       die "mmdsh: mmdshCommandsFile environment variable not set\n";
     }
     unless ( -f $mmdshCommandsFile ) {
       #die "mmdsh: File $mmdshCommandsFile not found\n";
       &prtMsg(364, $mmdshCommandsFile, "\'$!\'");
       exit(-1);
     }
   }

   # Clear out the host array and pid hash (also used in host verification)
   undef @hs;
   undef %pid;

   # The working collective, @wc, at this point is guaranteed to contain only
   # unique entries.  This prevents key collisions on the %pid hash table.
   foreach $host (@wc) {

      # If the special command _SELECT_FROM_FILE_ is specified,
      # retrieve the command string to be executed by the node.

      if ($selectCommandFromFile) {
        $command = `$grep -w ^$host $mmdshCommandsFile`;
        chop($command);

        # If there is no command for this host, move on.
        unless ($command) {
          if ($iflag) {
            print STDERR "mmdsh: $host: No command found.\n";
          }
          next;
        }

        # Remove the hostname from the command string.
        @dsh_ARGV = split(' ',$command);
        shift(@dsh_ARGV);
        $command = join(' ',@dsh_ARGV);

        if ($iflag) {
          print STDERR "mmdsh: $host: $command \n";
        }
      }


      # Create filehandles for pipe ends.
      $r_out{$host} = "READ_STDOUT_" . $host;
      $w_out{$host} = "WRITE_STDOUT_" . $host;
      $r_err{$host} = "READ_STDERR_" . $host;
      $w_err{$host} = "WRITE_STDERR_" . $host;

      # Open pipes for this host's stdout and stderr from rsh.
      if (pipe($r_out{$host}, $w_out{$host}) < 0) {
         #die "mmdsh: Couldn't pipe: $!\n";
         &prtMsg(368, "\'$!\'");
         exit(-1);
      }
      if (pipe($r_err{$host}, $w_err{$host}) < 0) {
         #die "mmdsh: Couldn't pipe: $!\n";
         &prtMsg(368, "\'$!\'");
         exit(-1);
      }

      # Fork a child to exec the rsh.
      MAINFORK: {

         if ($pid{$host} = fork) {

            # Parent code -
            # Close unneeded ends of pipes.
            # Parent will wait for child processes if fanout limit reached.
            close($w_out{$host});
            close($w_err{$host});

            push(@hs,$host);

            if (++$cur_fanout >= $fanout) {
               &read_pipes_and_wait(false, @hs);
            }

         } elsif (defined $pid{$host}) {

            # Child code -
            # Close unneeded ends of pipes.
            # Redirect stdout and stderr to output pipes.
            # Exec rsh.
            # stdout/stderr will go to pipes to be read by parent.
            close($r_out{$host});
            close($r_err{$host});

            unless (open(STDOUT, ">&$w_out{$host}")) {
               #die "mmdsh: Cannot redirect STDOUT: $!\n";
               &prtMsg(369, 'STDOUT', "\'$!\'");
               exit(-1);
            }
            unless (open(STDERR, ">&$w_err{$host}")) {
               #die "mmdsh: Cannot redirect STDERR: $!\n";
               &prtMsg(369, 'STDERR', "\'$!\'");
               exit(-1);
            }

            select(STDOUT); $| = 1;
            select(STDERR); $| = 1;

            # If the -I option is specified, copy the specified files
            # to the remote host prior to invoking the actual command.
            if ($stageFile) {
              $rcpCommandString = "$rcpPath -p $fileToCopy $host:$fileToCopy";
              if ($iflag) {
                print STDERR "mmdsh: $rcpCommandString \n";
              }
              $rcpOutput = `LC_ALL=C $rcpCommandString 2>&1`;
              $rcpRc = $? >> 8;

              # If the rcopy fails, move to the next host.
              if ($rcpRc > 0 && $rcpOutput !~ /refer to the same file/) {
                if ($rcpOutput) {
                  printf STDERR "%s", $rcpOutput;
                }
                &prtMsg(171, $rcpCommandString, $rcpRc);
                die "mmdsh: $command will not be executed on $host.\n";
              }
            }

            if ($stageFile2) {
              $rcpCommandString = "$rcpPath -p $fileToCopy2 $host:$fileToCopy2";
              if ($iflag) {
                print STDERR "mmdsh: $rcpCommandString \n";
              }
              $rcpOutput = `LC_ALL=C $rcpCommandString 2>&1`;
              $rcpRc = $? >> 8;

              # If the rcopy fails, move to the next host.
              if ($rcpRc > 0 && $rcpOutput !~ /refer to the same file/) {
                if ($rcpOutput) {
                  printf STDERR "%s", $rcpOutput;
                }
                &prtMsg(171, $rcpCommandString, $rcpRc);
                die "mmdsh: $command will not be executed on $host.\n";
              }
            }

            # Execute the command on the remote host.
            if (!@dsh_ARGV) {
               # mmdsh was invoked without specifying a command.
               # The command to execute comes from the mmdsh prompt.
	       if ($ENV{'DSHPATH'}) {
                  exec ($rshPath,
                        $host, '-n', '-l', $login,
                        "export PATH=$ENV{'DSHPATH'};",
                        $exportcmd, $command)                 ||
                  die "mmdsh: rsh: $host $command: $!\n";
	       } else {
                  exec ($rshPath,
                        $host, '-n', '-l', $login,
                        $exportcmd, $command)                 ||
                  die "mmdsh: rsh: $host $command: $!\n";
               }
            } else {
               # The command to execute comes from the mmdsh command line.
               # This is the path taken by the mm commands.
	       if ($ENV{'DSHPATH'}) {
                  exec ($rshPath,
                        $host, '-n', '-l', $login,
                        "export PATH=$ENV{'DSHPATH'};",
                        $exportcmd, @dsh_ARGV)                ||
                  die "mmdsh: rsh: $host @dsh_ARGV: $!\n";
	       } else {
                  exec ($rshPath,
                        $host, '-n', '-l', $login,
                        $exportcmd, @dsh_ARGV)                ||
                  die "mmdsh: rsh: $host @dsh_ARGV: $!\n";
               }
	    }

         } else {

           # Try again.  The fork must have failed due to a resource problem.
           if ($iflag) {
             print STDERR "mmdsh: fork() failed; will retry in 5 seconds.\n";
           }
           sleep 5;
           redo MAINFORK;
         }
      }
   }

   # Parent continues here after forking all the children for this command.
   # Get the results of any remaining rsh's (the number of hosts in our
   #  working collective may not be a multiple of the fanout value).
   # Get rid of any hosts that have failed.
   # Display the working collective and command.
   &read_pipes_and_wait(true, @hs);
   &delete_hosts;
   unless ($done) {
      $cur_fanout = 0;
      &check_wc;
      &display_wc;
   }
}

# If requested, delete the -I files.
if ($stageFile && $kflag) {
  `/bin/rm -f $fileToCopy $fileToCopy2 >/dev/null 2>&1`;
}

# exit

exit($rcode);       #---------- end of mainline program ----------------