#-------------------------------------------------------------------------------
# The batch scheduler allocates 1.9 Gbytes of memory with each processor.
# Increase available CPU or memory values to maintain that ratio.
#-------------------------------------------------------------------------------

if ($queue ne "viz") {

# We want at least one processor
   if ($ncpus == 0) {
     $ncpus=1;
   }

# determine if the job needs to be scheduled on the big memory system

   $system="co-compute1";
   if ($queue eq "dedicated") {

      if ($mem > $MAX_DED_MEM) {
         print STDERR "qsub: memory is limited to $MAX_DED_MEM Gbytes for dedicated jobs\n";
         print STDERR " *** Job not submitted ***\n";
         exit 1;

# dedicated job has to run on the big memory system

      } elsif ( $mem > ($MAX_DED_MEM/2) ) {
        $system="co-compute2";
      }

# queue is not dedicated...

   } else {
      if ($mem > $MAX_NONDED_MEM) {
         print STDERR "qsub: memory is limited to $MAX_NONDED_MEM Gbytes for non-dedicated jobs\n";
         print STDERR " *** Job not submitted ***\n";
         exit 1;

# non-dedicated job has to run on the big memory system

      } elsif ( $mem > ($MAX_NONDED_MEM/2) ) {
         $system="co-compute2";
      }
   }
# If we need more memory per node than is available on co-compute1, force
# the job to run on co-compute2 instead (to save processors)

   $round_ncpus = int ($ncpus/2+0.9999999)*2;  # round to even number
   $mem_per_cpu = $mem/$round_ncpus;
#print "mem=$mem  round_ncpus=$round_ncpus mem_per_cpu=$mem_per_cpu \n";
   if ($system == "co-compute1" and $mem/$round_ncpus > $GB_PER_PROC{$system}) {
#print "moving job to co-compute2\n";
      $system="co-compute2";
      $qsub_system="-l nodes=1:himem";
   }

# the cpus needed to satisfy the memory request

   if ($ncpus > $mem/$GB_PER_PROC{$system}) {
      $avail_ncpus = int ($ncpus/2+0.9999999)*2;
   } else {
      $avail_ncpus = int (($mem/$GB_PER_PROC{$system})/2+0.9999999)*2;
#print "mem=$mem  GB_PER_PROC{$system} = $GB_PER_PROC{$system}  avail_ncpus=$avail_ncpus\n";

# If the job will run on co-compute1 but there are not enough
# processors to cover the memory request, assume that the batch
# system is smart enough to run the job on co-compute2

      if ($system == "co-compute1" and $avail_ncpus > $MAX_NONDED_NCPUS) {
         $system="co-compute2";
         $avail_ncpus = int (($mem/$GB_PER_PROC{$system})/2+0.9999999)*2;
      }
   }

# check for $avail_ncpus for validity
   if ($queue eq "dedicated") {
      if ($avail_ncpus > $MAX_DED_NCPUS) {
         print STDERR "qsub: number of processors is limited to $MAX_DED_NCPUS for dedicated jobs\n";
         print STDERR " *** Job not submitted ***\n";
         exit 1;
      }
   } else {          # queue is not dedicated...
      if ($avail_ncpus > $MAX_NONDED_NCPUS) {
         print STDERR "qsub: number of processors is limited to $MAX_NONDED_NCPUS for non-dedicated jobs\n";
         print STDERR " *** Job not submitted ***\n";
         exit 1;
      }
   }

# number of available cpus for the cpuset is the maximum of $ncpus and
# check to see if too much memory is requested.

   $avail_mem = int($mem / $GB_PER_PROC{$system} + .99999999) * $GB_PER_PROC{$system};

   $max_mem = $avail_ncpus * $GB_PER_PROC{$system};
#   $max_ncpus = int ($avail_mem / $GB_PER_PROC{$system} + .5);
   $max_ncpus = int ($mem / $GB_PER_PROC{$system} + .5);
   if ($max_ncpus == 0) {
     $max_ncpus = 1;
   }

#print "GB_PER_PROC=$GB_PER_PROC{$system}  mem=$mem  avail_mem=$avail_mem  max_mem=$max_mem  ncpus=$ncpus avail_ncpus=$avail_ncpus  max_ncpus=$max_ncpus\n";
   if ($max_mem > $avail_mem) {
      $avail_mem = $max_mem;
      print STDERR "qsub: requesting a CPU Memory set with $avail_ncpus nodes\n";
   } elsif ($max_ncpus > $avail_ncpus) {
      $avail_ncpus = $max_ncpus;
      print STDERR "qsub: requesting a CPU Memory set with $avail_ncpus nodes to satisfy memory requirements\n";
# Specify memory in Mbytes or Gbytes

  $mbytes = int ($mem*1024 + .99999999);
  $memval="${mbytes}mb";
  if ($mbytes > 99999) {
     $gbytes = int ($mem + .99999999);
     $memval="${gbytes}gb";
  }
# print "memval=$memval\n";

# qsub options

  $qsub_opts="-lmem=${memval},ncpus=$ncpus,cpuset=$avail_ncpus $qsub_system";