[torqueusers] CONTRIBUTION: pestat utility version 2.0

Ole Holm Nielsen Ole.H.Nielsen at fysik.dtu.dk
Thu Sep 27 10:20:26 MDT 2007


Dear Torque users,

In the Torque source directory ./contrib/pestat-1.0 is a small Torque/PBS
node monitoring utility called "pestat" contributed by David Singleton
and yours truly.  The "pestat" shows the status of all nodes with 1 line
per node.  When I needed to update and port this tool I decided
the C-code was not the right way to go, and in stead I wrote a small shell
script which does the same thing but is much more portable and configurable.

Please see the attached script pestat-2.0 (download and rename it to pestat).
Archived versions are in ftp://ftp.fysik.dtu.dk/pub/Torque/

The output of pestat looks like this example:

# pestat
   node state  load    pmem ncpu   mem   resi usrs tasks  jobids/users
   n002  excl  4.10    7978   4  19693   2425  1/1    4    76836 user1
   n003  excl  4.12    7978   4  19693    859  1/1    4    76539 user2
   n004  excl  4.00    7978   4  19693   3618  1/1    4    75720 user3
   ...

which you can then grep for interesting strings.

Our main use of pestat is to display only nodes that are flagged because
some node load values are out of the expected range:

# pestat -f
Listing only nodes that are flagged by *
   node state  load    pmem ncpu   mem   resi usrs tasks  jobids/users
   m006  excl  2.53*   7978   4  19693    111  0/0    4    76446 user3
   m008  excl  2.25*   7978   4  19693    112  0/0    4    76446 user3
   m028  excl  2.50*   7978   4  19693    121  0/0    4    76446 user3
   m031  excl  1.02*   7978   4  19693    389  4/1    4    76852 user1 76853 
user1 76854 user1 76979 user1
   t106  excl  0.46*   2025   1   5931     64  1/1    1    77077 user2

This is a great way to monitor the health of the jobs running on the nodes.
When we want to examine the processes on a particular node, or a set of nodes
belonging to a job, we use the scripts sshps, sshjob and sshkill which you
can also find in ftp://ftp.fysik.dtu.dk/pub/Torque/

Comments on the pestat utility are welcome !  If there is interest
I could contribute the scripts to the Torque source tree (if the developers
are willing).

-- 
Ole Holm Nielsen
Department of Physics, Technical University of Denmark
-------------- next part --------------
#!/bin/sh

# Name: pestat
# Usage: pestat [-f]
# Torque resource manager utility script:
# Print a 1-line summary of jobs on each node.
# The printout may be customized as needed.

# Author: Ole.H.Nielsen at fysik.dtu.dk
# Version: 2.0
# Date: 27 September 2007

# Command arguments
if test $# -ge 1
then
	# Argument -f: Print only those nodes that are flagged
	if test $# -eq 1 -a $1 = "-f"
	then
		echo Listing only nodes that are flagged by \*
		listflagged=1
	else
		echo Usage: $0 '[-f]'
		exit 1
	fi
else
	listflagged=0
fi

# Locations of commands used
PBSNODES=/usr/local/bin/pbsnodes
QSTAT=/usr/local/bin/qstat
AWK=/bin/awk

# Heading for printout showing:
# node:		Node hostname
# state:	Torque state
# load:		CPU load average
# pmem:		Physical memory
# ncpu:		Number of CPUs
# mem:		Physical+virtual memory
# resi:		Resident (used) memory
# usrs:		Number of sessions / Number of users
# jobs:		Number of jobs
# jobids/users:	Jobids and corresponding usernames of Torque jobs on this node
echo "  node state  load    pmem ncpu   mem   resi usrs tasks  jobids/users"

#
# Show the Torque node status and parse the results
#
$PBSNODES -a | $AWK -v listflagged=$listflagged -v QSTAT=$QSTAT '
BEGIN {
	#
	# First get the list of jobids versus usernames from qstat
	#
	QSTAT = QSTAT " -r"			# Append -r flag (running jobs) to qstat.
	while ((QSTAT | getline) > 0) {		# Parse lines from qstat -r
		if (++line>5) {			# Skip first 5 header lines
			split($1,b,".")		# Jobid is b[1]
			username[b[1]] = $2	# Username of this jobid
		}
	}
	close(QSTAT)
}
#
# Parse the output of pbsnodes
#
NF==1 {	node=$1				# 1st line is nodename
	nodename[node] = node		# Node name
	getline				# Get the next input line
	numjobs[node] = 0               # Torque jobs on the node
	numtasks[node] = 0              # Number of tasks started by Torque on the node
	listnode=0			# Set to > 0 if this node gets flagged
	while (NF >= 3) {		# Read a number of non-blank lines
		if ($1 == "state") {
			if ($3 == "job-exclusive")			state[node] = "excl"
			else if ($3 == "job-exclusive,busy")		state[node] = "busy"
			else if ($3 == "busy")				state[node] = "busy"
			else if ($3 == "free")				state[node] = "free"
			else if ($3 == "offline")			state[node] = "offl"
			else if ($3 == "offline,job-exclusive")		state[node] = "offl"
			else if ($3 == "offline,job-exclusive,busy")	state[node] = "offl"
			else if ($3 == "down")				state[node] = "down"
			else if ($3 == "down,offline")			state[node] = "down"
			else if ($3 == "down,job-exclusive")		state[node] = "down"
			else if ($3 == "down,offline,job-exclusive")	state[node] = "down"
			else if ($3 == "down,offline,job-exclusive,busy")	state[node] = "down"
			else if ($3 == "UNKN")				state[node] = "UNKN"
		}
		else if ($1 == "np")		np[node] = $3
		else if ($1 == "properties")	properties[node] = $3
		else if ($1 == "ntype")		ntype[node] = $3
		else if ($1 == "jobs")		numtasks[node] = NF - 2
		else if ($1 == "status") {
			# Get the node status subfields
			split (substr($0,15), a, ",")	# Remove leading "status =", split subfields separated by ","
			for (field in a) {		# Process individual status subfields
				split(a[field],b,"=")	# Split var=value fields
				if (b[1]=="arch")		arch[node]=b[2]
				else if (b[1]=="opsys")		opsys[node]=b[2]
				else if (b[1]=="sessions")	sessions[node]=b[2]
				else if (b[1]=="nsessions")	nsessions[node]=int(b[2])
				else if (b[1]=="nusers")	nusers[node]=b[2]
				else if (b[1]=="idletime")	idletime[node]=b[2]
				else if (b[1]=="totmem")	totmem[node]=b[2]
				else if (b[1]=="availmem")	availmem[node]=b[2]
				else if (b[1]=="physmem")	physmem[node]=b[2]
				else if (b[1]=="ncpus")		ncpus[node]=b[2]
				else if (b[1]=="loadave")	loadave[node]=b[2]
				else if (b[1]=="netload")	netload[node]=b[2]
				else if (b[1]=="size")		size[node]=b[2]
				else if (b[1]=="jobs") {
					# Get the list of jobids/users for this node
					if (b[2] == "? 0") b[2] = ""	# Fix for a bug in pbsnodes ?
					numjobs[node]=split(b[2],c)
					for (i=1; i <= numjobs[node]; i++) {
						split(c[i], d, ".")
						# Get jobid and username
						jobid = d[1]
						user = username[jobid]
						# Case where the node pbs_mom has a (dead job) jobid unknown to pbs_server:
						if (length(user) == 0) {	# Flag non-existent username
							user="NONE*"
							listnode++
						}
						# Append jobid and username to the job list
						jobidlist[node] = jobidlist[node] " " jobid " " user
					}
				} else if (b[1]=="rectime")	rectime[node]=b[2]
			}
		}
		getline			# Get the next input line
	}

	# Print out values that we are interested in.  Flag unexpected values with a "*".

	# Flag nodes with status down, offline or unknown
	if (state[node] == "busy" || state[node] == "down" || state[node] == "offl" || state[node] == "UNKN") {
		stateflag="*"
		listnode++
	} else
		stateflag=" "

	# Flag unexpected CPU load average
	loaddiff = loadave[node] - numtasks[node]
	if (loaddiff > 0.5 || loaddiff < -0.5) {
		loadflag="*"
		listnode++
	} else
		loadflag=" "

	# Resident memory
	resi = (totmem[node]-availmem[node])/1024
	if (resi > 50 && resi > physmem[node]/1024 - 50) {	# High memory usage
		resiflag="*"
		listnode++
	} else
		resiflag=" "

	# Flag unexpected number of processes or users
	if (nsessions[node] > 2*ncpus[node] + 1) {	# More than 2 sessions per job
		sessflag="*"
		listnode++
	} else if (nusers[node] > ncpus[node]) {	# More users than nCPUs is bad
		sessflag="*"
		listnode++
	} else
		sessflag=" "

	# Flag unexpected number of jobs
	if (numjobs[node] > numtasks[node]) {		# Should be at least 1 task per job
		jobflag="*"
		listnode++
	} else
		jobflag=" "

	# CONFIGURE: Comment out the line below
	# Omit down nodes from the flagged list because we do not bother to see them
	# (Use "pbsnodes -l" to list down nodes).
	if (state[node] == "down") listnode=0

	if (!listflagged || listnode > 0)
		printf ("  %s  %s%1s %4.2f%1s %6d %3d %6d %6d%1s %1d/%1d%1s %3d%1s  %s\n",
			node, state[node], stateflag, loadave[node], loadflag,
			physmem[node]/1024, ncpus[node], totmem[node]/1024, resi, resiflag,
			nsessions[node], nusers[node], sessflag, numtasks[node], jobflag, jobidlist[node])
}'

rm -f $JOBLIST


More information about the torqueusers mailing list