#!/bin/sh
#  -*-Perl-*-  (for Emacs)    vim:set filetype=perl:  (for vim)
#======================================================================#
# Run the right perl version:
if [ -x /usr/local/bin/perl ]; then
  perl=/usr/local/bin/perl
elif [ -x /usr/bin/perl ]; then
  perl=/usr/bin/perl
else
  perl=`which perl| sed 's/.*aliased to *//'`
fi

exec $perl -x -S $0 "$@"     # -x: start from the following line
#======================================================================#
#! /Good_Path/perl -w

# Name:   llsub
# Author: Tobias Heinemann (theine@nordita.dk)
# Date:   23-Apr-2006
# CVS:    $Id$
# Description:
#   Wrapper script to pass start.csh and run.csh (or both) to IBM Loadleveler
#   God knows why it does not take any useful command line options on its
#   own. (Or does it?)
# To Do:
# - Probably lots.

use strict;
use Getopt::Long;
use integer;

my $usage =
"Usage: $0 [options] script.csh
  Wrapper script to pass start.csh and run.csh (or both) to IBM Loadleveler.

Options:
  -h,  --help             \tPrint Usage information
  -n,  --ncpus            \tNumber of processors
  -t,  --tasks            \tNumber of tasks per node
  -q,  --queue            \tCluster queue
  -w,  --walltime         \tMaximum wall clock time
  -a,  --account          \tAccount number
  -m,  --mail             \tEmail notification ('always', 'never')
  -o,  --output           \tPipe the resulting script to standard output
  -r,  --restart          \tRequeue a vacated job
";

# Default values
# (for testing on steno.dcsc.ku.dk -- feel free to modify and commit)
my $help = 0;
my $ncpus = 0;
my $tasks = 4;
my $queue = 'parallel_2gb';
my $walltime = '00:30:00';
my $account = '';
my $mail = 'never';
my $output = 0;
my $restart = 0;

my $pipe = "| llsubmit -";

# Get command line options
GetOptions('help'       => \$help,
           'ncpus=i'    => \$ncpus,
           'tasks=i'    => \$tasks,
           'queue=s'    => \$queue,
           'walltime=s' => \$walltime,
           'account=s'  => \$account,
           'mail=s'     => \$mail,
           'output'     => \$output,
           'restart'    => \$restart
          ) or die "Aborting.\n";

die $usage if (@ARGV == 0 or $help);

# Check whether we want to pipe to standard output
$pipe = "| cat" if ($output);

# Parse src/cparam.local for ncpus if it's not given as an argument
if ($ncpus eq 0) {
  open (CPARAM,'src/cparam.local');
  while (<CPARAM>) {
    if ($_ =~ "ncpus *= *([0-9]+)") {
      $ncpus = $1;
    }
  }
  close (CPARAM);
}

# tasks stands for "cpus per node" and ncpus should be a multiple thereof
my $message = "ncpus (-n) should be a multiple of tasks (-t)\n";
die $message unless ($ncpus % $tasks == 0);
my $nodes = $ncpus / $tasks;

# Working directory
my $pwd = `pwd`;
chomp ($pwd);

# Header instructions for Loadleveler
my $header =
"\#\!/bin/csh
# @ output = $pwd/\$(jobid).out
# @ error = \$(output)
# @ wall_clock_limit = $walltime
# @ notification  = $mail
# @ node_usage = not_shared
# @ job_type = parallel
# @ environment = COPY_ALL
# @ node = $nodes
# @ tasks_per_node = $tasks
# @ class = $queue".
(($account ne '')?"\n# @ account_no = $account":"")."
# @ restart = ".($restart?"yes":"no")."
# @ queue\n";

# Read submit scripts and remove all comments
# (Comments easily interfere with Loadleveler directives)
# For now, also remove exit statements so that passing
# both start.csh and run.csh to llsub actually works
my $text;
my $script;
foreach $script (@ARGV) {
  open (SCRIPT,$script) or die "Couldn't open $script\n";
  while ( <SCRIPT> ) {
    $text = $text . $_ unless ($_ =~ '^#' or $_ =~ '^exit');
  }
  close (SCRIPT);
}

# Pass everything to llsubmit (or standard output)
open (PIPE, $pipe);
print PIPE $header;
print PIPE $text;
close (PIPE);