[torqueusers] CONTRIBUTION: pestat utility version 2.0
Ole Holm Nielsen
Ole.H.Nielsen at fysik.dtu.dk
Thu Sep 27 10:20:26 MDT 2007
Dear Torque users,
In the Torque source directory ./contrib/pestat-1.0 is a small Torque/PBS
node monitoring utility called "pestat" contributed by David Singleton
and yours truly. The "pestat" shows the status of all nodes with 1 line
per node. When I needed to update and port this tool I decided
the C-code was not the right way to go, and in stead I wrote a small shell
script which does the same thing but is much more portable and configurable.
Please see the attached script pestat-2.0 (download and rename it to pestat).
Archived versions are in ftp://ftp.fysik.dtu.dk/pub/Torque/
The output of pestat looks like this example:
# pestat
node state load pmem ncpu mem resi usrs tasks jobids/users
n002 excl 4.10 7978 4 19693 2425 1/1 4 76836 user1
n003 excl 4.12 7978 4 19693 859 1/1 4 76539 user2
n004 excl 4.00 7978 4 19693 3618 1/1 4 75720 user3
...
which you can then grep for interesting strings.
Our main use of pestat is to display only nodes that are flagged because
some node load values are out of the expected range:
# pestat -f
Listing only nodes that are flagged by *
node state load pmem ncpu mem resi usrs tasks jobids/users
m006 excl 2.53* 7978 4 19693 111 0/0 4 76446 user3
m008 excl 2.25* 7978 4 19693 112 0/0 4 76446 user3
m028 excl 2.50* 7978 4 19693 121 0/0 4 76446 user3
m031 excl 1.02* 7978 4 19693 389 4/1 4 76852 user1 76853
user1 76854 user1 76979 user1
t106 excl 0.46* 2025 1 5931 64 1/1 1 77077 user2
This is a great way to monitor the health of the jobs running on the nodes.
When we want to examine the processes on a particular node, or a set of nodes
belonging to a job, we use the scripts sshps, sshjob and sshkill which you
can also find in ftp://ftp.fysik.dtu.dk/pub/Torque/
Comments on the pestat utility are welcome ! If there is interest
I could contribute the scripts to the Torque source tree (if the developers
are willing).
--
Ole Holm Nielsen
Department of Physics, Technical University of Denmark
-------------- next part --------------
#!/bin/sh
# Name: pestat
# Usage: pestat [-f]
# Torque resource manager utility script:
# Print a 1-line summary of jobs on each node.
# The printout may be customized as needed.
# Author: Ole.H.Nielsen at fysik.dtu.dk
# Version: 2.0
# Date: 27 September 2007
# Command arguments
if test $# -ge 1
then
# Argument -f: Print only those nodes that are flagged
if test $# -eq 1 -a $1 = "-f"
then
echo Listing only nodes that are flagged by \*
listflagged=1
else
echo Usage: $0 '[-f]'
exit 1
fi
else
listflagged=0
fi
# Locations of commands used
PBSNODES=/usr/local/bin/pbsnodes
QSTAT=/usr/local/bin/qstat
AWK=/bin/awk
# Heading for printout showing:
# node: Node hostname
# state: Torque state
# load: CPU load average
# pmem: Physical memory
# ncpu: Number of CPUs
# mem: Physical+virtual memory
# resi: Resident (used) memory
# usrs: Number of sessions / Number of users
# jobs: Number of jobs
# jobids/users: Jobids and corresponding usernames of Torque jobs on this node
echo " node state load pmem ncpu mem resi usrs tasks jobids/users"
#
# Show the Torque node status and parse the results
#
$PBSNODES -a | $AWK -v listflagged=$listflagged -v QSTAT=$QSTAT '
BEGIN {
#
# First get the list of jobids versus usernames from qstat
#
QSTAT = QSTAT " -r" # Append -r flag (running jobs) to qstat.
while ((QSTAT | getline) > 0) { # Parse lines from qstat -r
if (++line>5) { # Skip first 5 header lines
split($1,b,".") # Jobid is b[1]
username[b[1]] = $2 # Username of this jobid
}
}
close(QSTAT)
}
#
# Parse the output of pbsnodes
#
NF==1 { node=$1 # 1st line is nodename
nodename[node] = node # Node name
getline # Get the next input line
numjobs[node] = 0 # Torque jobs on the node
numtasks[node] = 0 # Number of tasks started by Torque on the node
listnode=0 # Set to > 0 if this node gets flagged
while (NF >= 3) { # Read a number of non-blank lines
if ($1 == "state") {
if ($3 == "job-exclusive") state[node] = "excl"
else if ($3 == "job-exclusive,busy") state[node] = "busy"
else if ($3 == "busy") state[node] = "busy"
else if ($3 == "free") state[node] = "free"
else if ($3 == "offline") state[node] = "offl"
else if ($3 == "offline,job-exclusive") state[node] = "offl"
else if ($3 == "offline,job-exclusive,busy") state[node] = "offl"
else if ($3 == "down") state[node] = "down"
else if ($3 == "down,offline") state[node] = "down"
else if ($3 == "down,job-exclusive") state[node] = "down"
else if ($3 == "down,offline,job-exclusive") state[node] = "down"
else if ($3 == "down,offline,job-exclusive,busy") state[node] = "down"
else if ($3 == "UNKN") state[node] = "UNKN"
}
else if ($1 == "np") np[node] = $3
else if ($1 == "properties") properties[node] = $3
else if ($1 == "ntype") ntype[node] = $3
else if ($1 == "jobs") numtasks[node] = NF - 2
else if ($1 == "status") {
# Get the node status subfields
split (substr($0,15), a, ",") # Remove leading "status =", split subfields separated by ","
for (field in a) { # Process individual status subfields
split(a[field],b,"=") # Split var=value fields
if (b[1]=="arch") arch[node]=b[2]
else if (b[1]=="opsys") opsys[node]=b[2]
else if (b[1]=="sessions") sessions[node]=b[2]
else if (b[1]=="nsessions") nsessions[node]=int(b[2])
else if (b[1]=="nusers") nusers[node]=b[2]
else if (b[1]=="idletime") idletime[node]=b[2]
else if (b[1]=="totmem") totmem[node]=b[2]
else if (b[1]=="availmem") availmem[node]=b[2]
else if (b[1]=="physmem") physmem[node]=b[2]
else if (b[1]=="ncpus") ncpus[node]=b[2]
else if (b[1]=="loadave") loadave[node]=b[2]
else if (b[1]=="netload") netload[node]=b[2]
else if (b[1]=="size") size[node]=b[2]
else if (b[1]=="jobs") {
# Get the list of jobids/users for this node
if (b[2] == "? 0") b[2] = "" # Fix for a bug in pbsnodes ?
numjobs[node]=split(b[2],c)
for (i=1; i <= numjobs[node]; i++) {
split(c[i], d, ".")
# Get jobid and username
jobid = d[1]
user = username[jobid]
# Case where the node pbs_mom has a (dead job) jobid unknown to pbs_server:
if (length(user) == 0) { # Flag non-existent username
user="NONE*"
listnode++
}
# Append jobid and username to the job list
jobidlist[node] = jobidlist[node] " " jobid " " user
}
} else if (b[1]=="rectime") rectime[node]=b[2]
}
}
getline # Get the next input line
}
# Print out values that we are interested in. Flag unexpected values with a "*".
# Flag nodes with status down, offline or unknown
if (state[node] == "busy" || state[node] == "down" || state[node] == "offl" || state[node] == "UNKN") {
stateflag="*"
listnode++
} else
stateflag=" "
# Flag unexpected CPU load average
loaddiff = loadave[node] - numtasks[node]
if (loaddiff > 0.5 || loaddiff < -0.5) {
loadflag="*"
listnode++
} else
loadflag=" "
# Resident memory
resi = (totmem[node]-availmem[node])/1024
if (resi > 50 && resi > physmem[node]/1024 - 50) { # High memory usage
resiflag="*"
listnode++
} else
resiflag=" "
# Flag unexpected number of processes or users
if (nsessions[node] > 2*ncpus[node] + 1) { # More than 2 sessions per job
sessflag="*"
listnode++
} else if (nusers[node] > ncpus[node]) { # More users than nCPUs is bad
sessflag="*"
listnode++
} else
sessflag=" "
# Flag unexpected number of jobs
if (numjobs[node] > numtasks[node]) { # Should be at least 1 task per job
jobflag="*"
listnode++
} else
jobflag=" "
# CONFIGURE: Comment out the line below
# Omit down nodes from the flagged list because we do not bother to see them
# (Use "pbsnodes -l" to list down nodes).
if (state[node] == "down") listnode=0
if (!listflagged || listnode > 0)
printf (" %s %s%1s %4.2f%1s %6d %3d %6d %6d%1s %1d/%1d%1s %3d%1s %s\n",
node, state[node], stateflag, loadave[node], loadflag,
physmem[node]/1024, ncpus[node], totmem[node]/1024, resi, resiflag,
nsessions[node], nusers[node], sessflag, numtasks[node], jobflag, jobidlist[node])
}'
rm -f $JOBLIST
More information about the torqueusers
mailing list