#!/bin/csh
# CVS: $Id$

# Name:   copy-snapshots
# Author: wd (Wolfgang.Dobler@kis.uni-freiburg.de)
# Date:   24-Sep-2002
# Description:
#   Copy snapshots VAR# and TAVG# from /scratch to $PBS_O_WORKDIR (from
#   where the code started).
#     Relies on PBS (and on tcsh)  and is needed on Horseshoe (the Odense
#   cluster).
#     Should better be a Perl script.

set debug = 0
foreach argv_scalar ($argv)
  if (($argv_scalar == "-v") || ($argv_scalar == "--verbose")) then
    set debug = 1          # for debugging output (recommended)
  else if (($argv_scalar == "-1")) then
    set run_only_once = 1  # check only once for files to copy
  else
    set varfile = $argv_scalar
    if ($debug) echo "varfile = <$varfile>"
  endif
end
##
## Set some necessary filesystem variables...
##
set pwd = `pwd`
set targetdir = $pwd/data
set nodelist = (`echo $NODELIST | sed 's/:/ /g'`) # unpack NODELIST
##
## ... and print out their values.
##
if ($debug) then
  echo "SCRATCH_DIR = <$SCRATCH_DIR>"
  echo "targetdir   = <$targetdir>"
  echo "nodelist    = <$nodelist>"
endif
##
## Produce debug output file in data directory
##
if($debug) then
  echo "copy-snapshots: pid=$$ starting" >>! $targetdir/copy-snaps_was_here.log
  echo "copy-snapshots: pid=$$ args=<$*>" >>! $targetdir/copy-snaps_was_here.log
endif
##
## If explicit filename is given, copy that file from local scratch discs
##
if ($?varfile) then
  echo "======================== $varfile ========================"
  foreach node ($nodelist)
    echo "------------------ working on node: $node ----------------------"
##
## Output directories and whereabouts of $varfile at the root processor's
## scratch disc
##
    if ($debug) then
      printf "\n  ##Checking if scratch directory is accessible\n"
      printf "\n$SSH $node ls -ltd $SCRATCH_DIR :\n"
      $SSH $node "ls -ltd $SCRATCH_DIR"
      printf "\n\n  ##Checking for presence of $varfile on scratch disc\n"
      printf "\n$SSH $node ls -ltd $SCRATCH_DIR/proc*/$varfile $SCRATCH_DIR/allprocs/$varfile :\n"
      $SSH $node "ls -ltd $SCRATCH_DIR/proc*/$varfile $SCRATCH_DIR/allprocs/$varfile"
      printf "\n  ##Checking for presence of $varfile in data directory\n"
      printf "\n$SSH $node ls -ltd $targetdir/proc*/$varfile $targetdir/allprocs/$varfile :\n"
      $SSH $node "ls -ltd $targetdir/proc*/$varfile $targetdir/allprocs/$varfile"
      echo
    endif
##
## Copy all $varfile you find (on all nodes) to the data directory
## (not efficient on dual-CPU systems, unless we would delete files
## after copying..)
##
    if ($debug) printf "  ##Copying proc*/$varfile and allprocs/$varfile to data directory\n\n"

    set remcmd = 'cd '"$SCRATCH_DIR"'; for f in `ls proc*/'"$varfile"' allprocs/'"$varfile"' 2> /dev/null`; do echo "Copying '"$SCRATCH_DIR"'/$f to '"$targetdir"'/$f"; cp '"$SCRATCH_DIR"'/$f '"$targetdir"'/$f; done'

    if ($debug) echo "Now running <$SSH $node sh -c $remcmd>"

    $SSH $node sh -c "'$remcmd'"

    if ($debug) then
      printf "\n  ##Checking for changes to $varfile in data directory\n"
      printf "\n$SSH $node ls -ltd $targetdir/proc*/$varfile $targetdir/allprocs/$varfile :\n"
      $SSH $node "ls -ltd $targetdir/proc*/$varfile $targetdir/allprocs/$varfile"
    endif

  end  # loop over nodes
  echo "========================================================"
else
##
## If no explicit filename is given, copy VARN, TIMEAVGN
##
  echo
  echo "=================== VARN, TIMEAVGN ====================="
##
## Check for relevant files at regular intervals
##
  while (1)
    echo
    echo "=========  "`date`"  ========="
##
## Is there something to copy? (It is sufficient to copy once the
## files show up for the master process of this job -- might depend on
## the queuing system etc.)
##
    if { ( (ls $SCRATCH_DIR/proc0/VAR[0-9]*     || \
            ls $SCRATCH_DIR/proc0/TIMEAVG[0-9]* || \
            ls $SCRATCH_DIR/allprocs/VAR[0-9]*  || \
            ls $SCRATCH_DIR/allprocs/TIMEAVG[0-9]* ) >& /dev/null ) } then
##
## Some debug output reporting that copy-snapshots found something to copy
##
      if ($debug) then
        printf "  ##Found something to copy on the root processor's scratch disc\n"
        echo "ls -l $SCRATCH_DIR/proc0/; ls -l $SCRATCH_DIR/allprocs/ :"
        foreach node ($nodelist)
          echo $node
          $SSH $node "ls -l $SCRATCH_DIR/proc*/; ls -l $SCRATCH_DIR/allprocs/"
        end
        if (-e $SCRATCH_DIR/proc0/varN.list) then
          printf "  ##Contents of $SCRATCH_DIR/proc0/varN.list :\n"
          echo `cat $SCRATCH_DIR/proc0/varN.list`
        endif
          if (-e $SCRATCH_DIR/allprocs/varN.list) then
          printf "  ##Contents of $SCRATCH_DIR/allprocs/varN.list :\n"
          echo `cat $SCRATCH_DIR/allprocs/varN.list`
        endif
        printf "\n"
        printf " ##Next copy-snapshots is going to move the filenames found\n"
        printf " ##in proc*/varN.list from all nodes to the data directory.\n"
      endif
##
##  Go to all nodes and copy everything from proc*/varN.list to the
##  data directory (this is the so-called new method for copying snapshots).
##
      foreach node ($nodelist)
        echo "------------------ working on node: $node ----------------------"
##
## Construct remote command that will do approximately the
## following thing (strongly complicated by the pathetic shell
## quoting rules. ceterum censeo: This script should be written
## in Perl):
##
##   for d in `cd $SCRATCH_DIR; ls -d allprocs proc*`; do
##       fdir="$SCRATCH_DIR/$d"
##       for f in `cat $fdir/*.list`; do
##           file=$fdir/$f
##           if [ -e $file ]; then
##               echo "Moving $file to $targetdir/$d/"
##               mv -f $file $targetdir/$d/"
##           fi
##       done
##   done
##
## This runs as a POSIX (Bourne) shell script on the remote
## side, so wo can write a full for(each) loop in one line.
##
        set remcmd = 'for d in `cd "'"$SCRATCH_DIR"'"; ls -d allprocs proc* 2> /dev/null`; do fdir="'"$SCRATCH_DIR"'/$d"; for f in `cat $fdir/*.list 2> /dev/null`; do file=$fdir/$f; if [ -e $file ]; then echo "Moving $file to '"$targetdir"'/$d/"; mv -f $file "'"$targetdir"'/$d/"; fi; done; done;'
        if ($debug) echo "Now running <$SSH $node sh -c $remcmd>"
        $SSH $node sh -c "'$remcmd'"
            echo "--------------------------------------------------------------"
      end

      if (`ps -p $PARENT_PID | fgrep -c $PARENT_PID` <= 0) then
        echo "No parent process (pid $PARENT_PID) -- exiting"
        exit 1
      endif

    else
##
## Output message if no interesting files found
##
      printf "  ##Nothing to copy:\n"
      printf "  ##no {proc0,allprocs}/{VAR,TIMEAVG}[0-9]* at the root processor's scratch disc\n"
    endif
##
## Break infinite while loop if set to check only once for files to copy
##
    if ($?run_only_once) break
##
## Check only every 60 seconds
##
    if (! $?varfile) then
      sleep 60
    endif

  end # while(1)

endif # no input filename given
##
## Finishing output
##
if ($debug) echo "copy-snapshots: done"
echo "------------------------------------------------------ "

#TEST-BEGIN
if($debug) then
  echo "copy-snapshots: pid=$$ finished" >>! $targetdir/copy-snaps_was_here.log
endif
#TEST-END

# End of file copy-snapshots