BSUB Job Submission

Common BSUB Flags

MPI

BSUB -n 2                        # 2 processes
BSUB -R 'span[ptile=1]'          # 1 process per node
BSUB -R 'span[hosts=1]'          # all processors allocated to the job must be on the same execution host

MPI Placement

BSUB -R 'cu[pref=minavail]'      # run the job across the minimal amount of enclosures available
BSUB -R 'cu[pref=maxcus=1]'      # force the job to run within the same enclosure

Exclusive this requires EXCLUSIVE = Y to be defined in the queue (lsb.queue)

BSUB -x                          # exclusive mode, job runs on host by itself

Output

BSUB -o IMB.%J.%I.out            # standard output, appended
BSUB -oo IMB.%J.%I.out           # standard output, create new file 
BSUB -e IMB.%J.%I.err            # error output, appended
BSUB -ee IMB.%J.%I.err           # error output, create new file

Jobs Arrays

BSUB -J Job_Name
BSUB -J IMB[1-110]               # job array, launch 110 jobs as part of this job array

Checkpoint

BSUB -q chkpnt_rerun_queue                               # specify a queue to run on 
BSUB -k "/home/viglen/siesta/sbench-2009 method=blcr"    # -k for a shared checkpoint dir and use blcr

Resources

BSUB -M memlimit
BSUB -v swap/virt memlimit
BSUB -m comp000
BSUB -m "comp000 comp001"

Move Files across clusters/nodes not sharing a common user space

• Pay close attention to the redirects

# Copy input.inp from submission host to execution host
BSUB -f "/data/clustertest/case-03/input.inp > /home/user/app/input.inp"

# Copy back results file
BSUB -f "/data/clustertest/case-03/result.out < /home/user/app/result.out"

Interactive Jobs

# X forwarding
bsub -XF xterm

# Interactive Shell
bsub -Ip bash -l

IMB

#BSUB -n 2                        # 2 processes
#BSUB -R 'span[ptile=1]'          # 1 process per node
#BSUB -R 'cu[pref=minavail]'      # run the job across the minimal amount of enclosures available
#BSUB -R 'cu[maxcus=1]'           # force the job to run within the same enclosure
#BSUB -x                          # exclusive mode, job runs on host by itself
#BSUB -o IMB.%J.%I.out            # standard output, appended
#BSUB -e IMB.%J.%I.err            # error output, appended
#BSUB -J IMB[1-110]               # Job array, launch 110 jobs as part of this job array
##BSUB -L /bin/bash

#. /etc/profile

#module add PMPI/modulefile
mpirun -np $LSB_DJOB_NUMPROC /home/mbptmp/imb/src/IMB-MPI1

# Or if your running without platform mpi
mpirun -np $LSB_DJOB_NUMPROC -hostfile $LSB_DJOB_HOSTFILE /path/to/mpi-app

xHPL

#BSUB -n 8
#BSUB -R 'span[ptile=8]'
#BSUB -o xhpl.%I.%J.out
#BSUB -e xhpl.%I.%J.err
#BSUB -J xhpl[10-600:10]


#BSUB -L /bin/bash

. /etc/profile

export OMP_NUM_THREADS=1
export NCPUS=1

cd /home/mbptmp/hpl-2.0/bin/Nehalem/
mkdir $LSB_JOBINDEX
cd $LSB_JOBINDEX
sed "s/REPLACE_ME/$LSB_JOBINDEX/" /home/mbptmp/hpl-2.0/bin/Nehalem/HPL.dat > HPL.dat

module add PMPI/modulefile
mpirun -np $LSB_DJOB_NUMPROC /home/mbptmp/hpl-2.0/bin/Nehalem/xhpl