Rocks: PBS

Scripts and commands for torque / maui (pbs)

Torque/Maui Commands

    qsub submit_script.sh       # submit Job

    qmgr print server           # show the server configuration/settings
    qmgr print node compute-0-0 # show the compute nodes configuration/settings

    qstat                       # list information about queues and jobs
    qstat -q                    # list all queues on system
    qstat -Q                    # list queue limits for all queues
    qstat -a                    # list all jobs on system
    qstat -au userid            # list all jobs owned by user userid (returns jobid)
    qstat -s                    # list all jobs with status comments
    qstat -r                    # list all running jobs
    qstat -f jobid              # list all information known about specified job
    qstat -Qf queue             # list all information known about specified queue
    qstat -B                    # list summary information about the PBS server
    qstat -n                    # list nodes allocated to jobs
    qstat -ans -u {username}    # this will show the nodes, jobid's of a given {username}

    qdel jobid                  # delete job jobid
    qdel -p job (as root)       # purge job from system, last resort
    qalter jobid                # modify the attributes of the job or jobs specified by jobid

    pbsnodes -a                 # show nodes online and some node characteristics
    pbsnodes -l                 # list which nodes are currently offline
    pbsnodes -o compute-1-7     # set compute-1-7 to offline (no job will schedule to this node)
    pbsnodes -c compute-1-7     # clear compute-1-7 offline node status

    showq                       # shows the current queue status (decide by MAUI)
    showq -u {username}         # show the queue of jobs that belong to {username}
    checkjobs -v {jobid}        # show the status of job {jobid}
    showbf                      # shows the total available cpus on the cluster
    showbf -f {FEATURE}         # shows the total available cpus with feature set {FEATURE} on the cluster. 
    showstart {jobid}           # estimate when the job {jobid} will start/complete
    checkjob {jobid}            # information on the job that is running or being queued.
    showstate                   # Check if any nodes are down... queue daemon not running
    showstats                   # Stats about queue

Submit Script Flags

#PBS -I                             # submit an interactive job
#PBS -q queue_name                  # submit to a specific queue
#PBS -l nodes=6                     # request 6 nodes
#PBS -l nodes=compute-0-0           # request only node compute-0-0
#PBS -l nodes=4:ppn=2               # request 4 nodes with 2 processes per node
#PBS -j oe                          # combine both stdout and stderr
#PBS -o output_filename

Simple PBS Job Submit Script

#!/bin/bash

# Import BASH variables 
#PBS -S /bin/bash
#PBS -V

# Combine error/output and set outputfile
#PBS -j oe
#PBS -o qsub_hpl_output.log

# Request job to run on 46 nodes with 8 processes per node
#PBS -l nodes=46:ppn=8

# Change to the working directory
cd ${PBS_O_WORKDIR}

##########################################
#                                        #
#   Output some useful job information.  #
#                                        #
##########################################

NPROCS=`wc -l < $PBS_NODEFILE`
echo ------------------------------------------------------
echo ' This job is allocated on '${NPROCS}' cpu(s)'
echo 'Job is running on node(s): '
cat $PBS_NODEFILE
echo ------------------------------------------------------
echo PBS: qsub is running on $PBS_O_HOST
echo PBS: originating queue is $PBS_O_QUEUE
echo PBS: executing queue is $PBS_QUEUE
echo PBS: working directory is $PBS_O_WORKDIR
echo PBS: execution mode is $PBS_ENVIRONMENT
echo PBS: job identifier is $PBS_JOBID
echo PBS: job name is $PBS_JOBNAME
echo PBS: node file is $PBS_NODEFILE
echo PBS: current home directory is $PBS_O_HOME
echo PBS: PATH = $PBS_O_PATH
echo ------------------------------------------------------

# Launch the Job
date
/opt/openmpi/bin/mpirun -np $NPROCS -machinefile $PBS_NODEFILE ./xhpl
date

• Use showstats to get details on user jobs.

[root@fotcluster2 ~]# showstats -u | sort -n -k 3
 
         |------ Active ------|--------------------------------- Completed -----------------------------------|
asmith       0     0      0.00    1   2.27 153600.0   2.53  322.2   2.40 -----   0.00   0.00   0.00 4771.67   0.20
klangfeld    0     0      0.00   15  34.09 5567999.4  91.77 13068.4  97.45 -----   0.00   0.00   0.00 21888.98   0.22
maustin      0     0      0.00   28  63.64 345666.6   5.70   20.5   0.15 -----   0.08   0.00   0.00 5487.95   7.43
statistics initialized Thu Jan  1 01:00:00
user      Jobs Procs ProcHours Jobs    %    PHReq    %    PHDed    %   FSTgt  AvgXF  MaxXF  AvgQH  Effic  WCAcc