check_busy_low_load

Check if a node marked Busy is actually doing anything

Note: it is assumed that TORQUE is used as a resource manager and Maui as a scheduler. Tested with TORQUE 2.3.5 and Maui 3.2.6p21.

Requires valid checknode and checkjob Maui clients.

Usage

check_busy_low_load -H host

e.g. check_multi -H 10.10.1.23 check node 10.10.1.23; if checknode reports a low load alert, find job ID and owner and report a WARNING.

#!/usr/bin/perl -w use strict ; use lib "/opt/nagios/libexec" ; use utils qw($TIMEOUT %ERRORS &print_revision &support) ; use vars qw($PROGNAME) ; use Getopt::Long ; $PROGNAME = "check_busy_low_load" ; my $checknode = "/opt/nagios/libexec/external/checknode" ; my $checkjob = "/opt/nagios/libexec/external/checkjob" ; my $hostname ; GetOptions( "hostname|H:s" => \$hostname ) ; unless ( -x $checknode ) { print "UNKNOWN - $checknode not found\n" ; exit $ERRORS{'UNKNOWN'} ; } unless ( -x $checkjob ) { print "UNKNOWN - $checkjob not found\n" ; exit $ERRORS{'UNKNOWN'} ; } my $joblist = 'none' ; my $load ; open CHECKNODE, "$checknode $hostname |" or exit $ERRORS{'UNKNOWN'} ; while ( <CHECKNODE> ) { chomp ; m/JobList:\s+([\d\s,]+)/ ; $joblist = $1 if $1 ; next unless m/ALERT:\s+node is in state Busy but load is low\s+\(([\d.]+)\)/ ; my @job_list = split ',', $joblist ; my $j ; for $j ( @job_list ) { open CHECKJOB, "$checkjob $j |" or next ; # job is no more? # some other unknown error? # anyway providing # incomplete info is # better than # complete failure... while ( <CHECKJOB> ) { chomp ; next unless m/Creds:\s+user:([^\s]+)/ ; my $user = $1 ; $joblist =~ s/$j/$j($user)/ ; } } $load = $1 ; print "WARNING - low load on busy node|joblist=$joblist;load=$load\n" ; exit $ERRORS{'WARNING'} ; } print "OK|joblist=$joblist\n" ; exit $ERRORS{'OK'} ;
download code