#!/bin/sh

# netapp
#
# BIG BROTHER - NETAPP SCRIPT
# Sean MacGuire - BB4 Technologies Inc.
# Version 1.6a
# Jan 9, 2001
#
# This program is Copyright (c) 2001
# BB4 Technologies Inc
# All Rights Reserved

#
# Thanks to Andrew.Tefft@trans.ge.com for fixing the 2^32 issue
# and adding disk quota support
#
# Added raid check & network Interface check
# Tal Ovadia Jalan Network SErvices 06/05/2001 tal@jalan.com
# raid: will page if any disk is not active,spare or scrubbing
# int: will page if the monitored interface or the virtual interface 
#    : changes state to down
#
# SET UP THE SNMP STUFF WE NEED
#
# MIBS=+NETWORK-APPLIANCE-MIB; export MIBS
 MIBS=+/usr/share/snmp/mibs/netapp.mib.txt; export MIBS
#
# BBHOME CAN BE SET MANUALLY WHEN TESTING.
# OTHERWISE IT SHOULD BE SET FROM THE BB ENVIRONMENT
#
# BBHOME=/home/sean/bb; export BBHOME	# FOR TESTING

if test "$SNMPGET" = ""
then
	echo "This script must be run from the snmp command"
	exit 1
fi

WARNPCT="15"
PANICPCT="40"

#Which interface is up and which one is a virtual Interface?
UPINT="6"
VIRTINT="8"
TWO32=4294967296

# GET THE OVERALL HEALTH OF THE BOX
# sysStat.misc.miscGlobalStatus.0 = ok(3)
# other(1), unknown(2), ok(3), nonCritical(4), critical(5), nonRecoverable(6)
COLOR="green"

set `$SNMPGET $HOST $COMMUNITY sysStat.misc.miscGlobalStatus.0`
RESULT="$3"

if test "$RESULT" != "ok(3)"
then
	if test "$RESULT" = "critical(5)" || test "$RESULT" = "nonRecoverable(6)"
	then
		COLOR="red"
	else
		COLOR="yellow"
	fi
fi

set `$SNMPGET $HOST $COMMUNITY sysStat.misc.miscGlobalStatusMessage.0`
shift;shift
MSG=`eval echo $*`

LINE="$MSG"

# CHECK THE TEMPERATURE
# sysStat.environment.envOverTemperature.0 = no(1)
set `$SNMPGET $HOST $COMMUNITY sysStat.environment.envOverTemperature.0`
RESULT=$3

if test "$RESULT" != "no(1)"
then
	COLOR="red"
	LINE="$LINE
&red NetApps temperature too high"	
else
	LINE="$LINE
&green NetApps temperature OK"	
fi

# CHECK THE FANS
# sysStat.environment.envFailedFanCount.0 = 0
set `$SNMPGET $HOST $COMMUNITY envFailedFanCount.0`
RESULT="$3"
set `$SNMPGET $HOST $COMMUNITY envFailedFanMessage.0`
shift; shift
MSG=`eval echo $*`

if test "$RESULT" != "0"
then
	COLOR="red"
	LINE="$LINE
&red $MSG"
else
	LINE="$LINE
&green $MSG"
fi

# CHECK THE POWER SUPPLIES
# sysStat.environment.envFailedPowerSupplyCount.0
set `$SNMPGET $HOST $COMMUNITY envFailedPowerSupplyCount.0`
RESULT="$3"
set `$SNMPGET $HOST $COMMUNITY envFailedPowerSupplyMessage.0`
shift; shift
MSG=`eval echo $*`

if test "$RESULT" != "0"
then
	COLOR="red"
	LINE="$LINE
&red $MSG"
else
	LINE="$LINE
&green $MSG"
fi

# echo "COLOR: $COLOR"
# echo "$BB $BBDISP LINE:  $LINE"

$BB $BBDISP "status ${BOX}.sys $COLOR `date` $LINE"

# --------------------------------------------------------
# NOW GET SOME CPU INFORMATION (cpu test)
#
# cpuUpTime.0 = Timeticks: (232772452) 26 days, 22:35:24.52
# --------------------------------------------------------

COLOR="green"

set `$SNMPGET $HOST $COMMUNITY cpuUpTime.0`
TICKS=`echo $4 | $SED "s/(//g" | $SED "s/)//g"`
shift; shift; shift; shift;
UPTIME="NetApp has been up for $*"

# echo "LINE IS: $LINE"

echo "$UPTIME" | $GREP "min" > /dev/null 2>&1
if test "$?" = "0"
then
        if test "$COLOR" != "red"
        then
                COLOR="yellow"
        fi
        LINE="Warning: Machine recently rebooted 
$UPTIME"
else
	LINE="$UPTIME"
fi

set `$SNMPGET $HOST $COMMUNITY cpuBusyTime.0`
BUSY=`echo $4 | $SED "s/(//g" | $SED "s/)//g"`
set `$SNMPGET $HOST $COMMUNITY cpuBusyTimePerCent.0`
BUSYPCT="$3"
set `$SNMPGET $HOST $COMMUNITY miscNfsOps.0`
NFSOPS="$3"
set `$SNMPGET $HOST $COMMUNITY miscNetRcvdKB.0`
NETRECD="$3"
set `$SNMPGET $HOST $COMMUNITY miscNetSentKB.0`
NETSENT="$3"

# NOW SUCK INFO OUT OF THE HISTORY FILE...
# NOTE - THERE SEEMS TO BE EITHER A BUGLET OR I DON'T UNDERSTAND
# WHAT NETRECD CONTAINS... IT'S SUPPOSED TO BE THE TOTAL KB REC'D
# BUT IT ROLLS OVER... AND THAT MAKES THE REST OF THE STATS GO
# NEGATIVE... WHEN THIS HAPPENS IT'S MARKED n/a

NETAPPHIST=`$TAIL -3 $BBHOME/tmp/netapp.${MACHINEMASK}.stats`  2> /dev/null

if test "$NETAPPHIST" = ""
then
	NETAPPHIST="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0"
fi

set $NETAPPHIST
if test "$#" -eq "15"
then
	# FIRST LINE IS 15 MINUTES AGO (OR SO)
	TICKS15=`expr $TICKS - $1`
	BUSY15=`expr $BUSY - $2`
	BUSY15=`expr 100 \* $BUSY15 / $TICKS15`
	NFSOPS15=`expr $NFSOPS - $3`
	NETRECD15=`expr $NETRECD - $4`
	if test "$NETRECD15" -lt "0"
	then
		NETRECD15="n/a"
	fi
	NETSENT15=`expr $NETSENT - $5`
	shift; shift; shift; shift; shift
fi

if test "$#" -eq "10"
then
	# NEXT LINE IS 10 MINUTES AGO (OR SO)
	TICKS10=`expr $TICKS - $1`
	BUSY10=`expr $BUSY - $2`
	BUSY10=`expr 100 \* $BUSY10 / $TICKS10`
	NFSOPS10=`expr $NFSOPS - $3`
	NETRECD10=`expr $NETRECD - $4`
	if test "$NETRECD10" -lt "0"
	then
		NETRECD10="n/a"
	fi
	NETSENT10=`expr $NETSENT - $5`
	shift; shift; shift; shift; shift
fi

if test "$#" -eq "5"
then
	# THIRD LINE IS 5 MINUTES AGO (OR SO)
	TICKS5=`expr $TICKS - $1`
	BUSY5=`expr $BUSY - $2`
	BUSY5=`expr 100 \* $BUSY5 / $TICKS5`
	NFSOPS5=`expr $NFSOPS - $3`
	NETRECD5=`expr $NETRECD - $4`
	if test "$NETRECD5" -lt "0"
	then
		NETRECD5="n/a"
	fi
	NETSENT5=`expr $NETSENT - $5`
	shift; shift; shift; shift; shift
fi

if test "$BUSY5" -ge "$PANICPCT"
then
	COLOR="red"
	OLDLINE="$LINE"
	LINE="NetApps load average of ${BUSY5}% (Panic at ${PANICPCT}%)
$OLDLINE"
elif test "$BUSY5" -ge "$WARNPCT"
then
	COLOR="yellow"
	LINE="NetApps load average of ${BUSY5}% (Warn at ${WARNPCT}%)
$OLDLINE"
fi

echo "$TICKS $BUSY $NFSOPS $NETRECD $NETSENT" >> $BBHOME/tmp/netapp.${MACHINEMASK}.stats

LINE="$LINE
		Total		5 Min	10 Min	15 Min
CPU busy	${BUSYPCT}% 		${BUSY5}%	${BUSY10}%	${BUSY15}%
NFS OPS 	$NFSOPS	$NFSOPS5	$NFSOPS10	$NFSOPS15
KB sent		$NETSENT	 	$NETSENT5	$NETSENT10	$NETSENT15
KB recd		$NETRECD		$NETRECD5	$NETRECD10	$NETRECD15
"
$BB $BBDISP "status ${BOX}.cpu $COLOR `date` $LINE"

# --------------------------------------------------------
# NOW GET THE DISK INFO (disk)
#
# THIS IS A LITTLE COMPLEX SINCE A SIMPLE df HAS BEEN BROKEN
# INTO N PARTS OF N COMMANDS. 
# --------------------------------------------------------
COLOR="green"
set `$SNMPWALK $HOST $COMMUNITY dfIndex | $TAIL -1`
NUMPART="$3"
LINE="NetApp has $NUMPART partitions"

COUNT="1"

while test "$COUNT" -le "$NUMPART"
do
	set `$SNMPGET $HOST $COMMUNITY dfFileSys.${COUNT}`
	DFNAME=`eval echo $3`
	set `$SNMPGET $HOST $COMMUNITY dfKBytesTotal.${COUNT}`
	DFTTL="$3"
	set `$SNMPGET $HOST $COMMUNITY dfKBytesUsed.${COUNT}`
	DFUSED="$3"
	set `$SNMPGET $HOST $COMMUNITY dfKBytesAvail.${COUNT}`
	DFAVAIL="$3"
	set `$SNMPGET $HOST $COMMUNITY dfPerCentKBytesCapacity.${COUNT}`
	DFPCT="$3"
	set `$SNMPGET $HOST $COMMUNITY dfMountedOn.${COUNT}`
	DFMOUNT=`eval echo $3`

	echo "$DFNAME 	$DFTTL $DFUSED $DFAVAIL ${DFPCT}% $DFMOUNT" >> $BBTMP/NETAPP-DF.$$
	COUNT=`expr $COUNT + 1`
done

# now do quota trees.

# Are quotas enabled? quotaStateOn(2)
set `$SNMPWALK $HOST $COMMUNITY quotaState.0`
QUOTAS=$3
if test "$QUOTAS" = "quotaStateOn(2)"
then

set `$SNMPWALK $HOST $COMMUNITY qrIndex | $TAIL -1`
NUMPART="$3"
LINE="NetApp has $NUMPART quota trees"

COUNT="1"

while test "$COUNT" -le "$NUMPART"
do
      set `$SNMPGET $HOST $COMMUNITY qrPathName.${COUNT}`
      DFMOUNT=`eval echo $3`
      set `$SNMPGET $HOST $COMMUNITY qrKBytesLimit.${COUNT}`
      DFTTL="$3"
      if test "$DFTTL" -lt "0"
      then
        DFTTL=`expr $TWO32 + $DFTTL`
        fi
      set `$SNMPGET $HOST $COMMUNITY qrKBytesUsed.${COUNT}`
      DFUSED="$3"
      if test "$DFUSED" -lt "0"
      then
        DFUSED=`expr $TWO32 + $DFUSED`
        fi
#     set `$SNMPGET $HOST $COMMUNITY dfKytesAvail.${COUNT}`
#     DFAVAIL="$3"
#     set `$SNMPGET $HOST $COMMUNITY dfPerCentKBytesCapacity.${COUNT}`
#     DFPCT="$3"
#     set `$SNMPGET $HOST $COMMUNITY dfMountedOn.${COUNT}`
#     DFMOUNT=`eval echo $3`
      DFAVAIL=`expr $DFTTL - $DFUSED`
      DFPCT=`expr 100 \* $DFUSED / $DFTTL`
      DFNAME="quota-$DFMOUNT"

      echo "$DFNAME   $DFTTL $DFUSED $DFAVAIL ${DFPCT}% $DFMOUNT" >> $BBTMP/NETAPP-DF.$$
      COUNT=`expr $COUNT + 1`
done

fi

cat $BBTMP/NETAPP-DF.$$ | {
while read line
do
    set $line
    DISKNAME=$6
    DISKDEV=$1
    DISKPCT=`echo $5 | $SED "s/%//"`

    # echo "*** DISK TEST 2 ***"
    # echo "$DISKNAME $DISKPCT"

	DFWARNVAL=$DFWARN               # USE DEFAULT WARN VALUE
	DFPANICVAL=$DFPANIC             # USE DEFAULT PANIC VALUE

	#
	# NEW IN 1.08a
	# Robert Andre Croteau's custom disk file
	# IF WE HAVE A CUSTOM PARTITION FILE, USE IT.
	# DEFAULTS TO etc/bb-dftab
	# FORMAT:	partition:warn%:panic%
	# I.E.:		/usr:96:99
	#
	# multihost bb-dftab based on an idea of Jeff Younker<jeff@mdli.com>
	if test -f "${BBDFTAB}"
	then
		# File format:  [host:]partition:WARN:PANIC
		# Set precedence levels, suggested by Larry Parmelee <parmelee@CS.Cornell.EDU>
		# Check for the partition without the leading host

		# echo "*** CHECKING FOR PARTITION $DISKNAME IN $BBDFTAB"

		# If highest precedence if host is defined
		DISKLINE=`$GREP "^${MACHINEMASK}[   ]*:[ 	]*${DISKNAME}[ 	]*:" $BBDFTAB`
		# echo "*** DISKLINE: $DISKLINE"
		if test "$DISKLINE" = ""
		then
			DISKLINE=`$GREP "^${MACHINEMASK}\..**:[ 	]*${DISKNAME}[ 	]*:" $BBDFTAB`
		fi

		# Then look for localhost:
		if test "$DISKLINE" = ""
		then
			DISKLINE=`$GREP "^localhost[ 	]*:[ 	]*${DISKNAME}[ 	]*:" $BBDFTAB`
		fi

		# Then look for a line that starts with : (equivalent of localhost:)
		if test "$DISKLINE" = ""
		then
			DISKLINE=`$GREP "^[ 	]*:[ 	]*${DISKNAME}[ 	]*:" $BBDFTAB`
			if test "$DISKLINE" != ""
			then
				DISKLINE="localhost${DISKLINE}"
			fi
		fi

		# Finally look for a line that starts with the filesystem
		if test "$DISKLINE" = ""
		then
			DISKLINE=`$GREP "^[ 	]*${DISKNAME}[ 	]*:" $BBDFTAB`
			# echo "*** DISKLINE: $DISKLINE"
			if test "$DISKLINE" != ""
			then
				DISKLINE="localhost:${DISKLINE}"
			fi
		fi

		# We have a match
		if test "$DISKLINE" != ""
		then
			# echo "*** DISKLINE NOT NULL"
			set `echo $DISKLINE | $SED "s/:/ /g"`
       			DFWARNVAL=$3            # GET CURRENT FS WARNING LEVEL
       			DFPANICVAL=$4           # GET CURRENT FS PANIC LEVEL
			# echo "*** WARN AT $DFWARNVAL - PANIC AT $DFPANICVAL"
		fi
	fi

    	MARK=""
    	if test "$DISKPCT" -ge "$DFWARNVAL"         # ABOVE WARNING LEVEL
    	then
        	if test "$DISKPCT" -ge "$DFPANICVAL"    # ABOVE PANIC, RED ALERT !!!
        	then
            		COLOR="red"
            		MARK="PANIC"
	    		MARKLEVEL=$DFPANICVAL
	    		HIGHMARK="PANIC"
			GIFCOLOR="&red"
	        	REDLINE="${GIFCOLOR} ${DISKNAME} (${DISKPCT}%) has reached the defined disk space ${MARK} level (${MARKLEVEL}%)
${REDLINE}"
        	else
            		if test "$COLOR" = "green"          # ONLY IF NOT ALREADY SET
            		then
                		COLOR="yellow"
	        		HIGHMARK="WARNING"
            		fi
            		MARK="WARNING"
	    		MARKLEVEL=$DFWARNVAL
			GIFCOLOR="&yellow"
	        	YELLOWLINE="${GIFCOLOR} ${DISKNAME} (${DISKPCT}%) has reached the defined disk space ${MARK} level (${MARKLEVEL}%)
${YELLOWLINE}"
        	fi
    	fi
done

# MAKE IT PRETTY...
DFOUT=`$AWK '{ printf "%-20s %+10s %+10s %+10s %+4s %s\n", $1, $2, $3, $4, $5, $6 }' $BBTMP/NETAPP-DF.$$`

#
# NOW SEND THIS INFORMATION TO THE BIG BROTHER DISPLAY UNIT
#
if [ "$COLOR" = "green" ]
then
    LINE="status $BOX.disk $COLOR `date` - Disk partitions on $BOX OK

$DFOUT"
else
    LINE="status $BOX.disk $COLOR `date` - Disk on $BOX at $HIGHMARK level
${REDLINE}${YELLOWLINE}
$DFOUT"
fi

$BB $BBDISP "$LINE"

$RM -f $BBTMP/NETAPP-DF.$$

############################################
#Raid Status Check 	                   #
#Tal Ovadia Jalan Network Services 6/1/2001#
#Will page if disk is not active,spare 	   #
#Or scrubbing.                             #
############################################

RAID=`$SNMPWALK $HOST $COMMUNITY raidStatus |awk -F. '{printf "%s %s\n", $7, $8}'` 
COLOR=green
TMP=0
for DEVICE in `$SNMPWALK $HOST $COMMUNITY raidStatus |awk -F. '{printf "%+20s %s\n", $7, $8}' |awk '{print $4}'` ; do
#Are we a spare disk ?
echo $DEVICE |grep spare >/dev/null
RESULT=$?
if [ $RESULT -eq 0 ]
then
        TMP=1
fi

# Are we an active disk ?
echo $DEVICE |grep active >/dev/null
RESULT=$?
if [ $RESULT -eq 0 ]
then
        TMP=1
fi

#mmmm, are we scrubbing ?
echo $DEVICE |grep scrubbing >/dev/null
RESULT=$?
if [ $RESULT -eq 0 ]
then
        TMP=1
fi

#What are we then ?
if [ $TMP -ne 1 ]
then
        COLOR=red
fi
TMP=0
done

LINE="status $BOX.raid $COLOR `date`
$RAID"
$BB $BBDISP "$LINE"
############################################
#Network Interfaces Check                  #
#Tal Ovadia Jalan Network Services 6/1/2001#
############################################

COLOR=green
LINE="status $BOX.int $COLOR `date` A physical and Virtual Interfaces are UP"
INT6=`$SNMPGET $HOST $COMMUNITY  interfaces.ifTable.ifEntry.ifOperStatus.$UPINT | awk -F. '{print $1 $5}'`
echo $INT6 | grep "up"
RESULT=$?
if [ $RESULT -ne 0 ]
then
        COLOR=red
        LINE="status $BOX.int $COLOR `date` $INT6 Physical Interface is DOWN"
fi

INT8=`$SNMPGET $HOST $COMMUNITY interfaces.ifTable.ifEntry.ifOperStatus.$VIRTINT | awk -F. '{print $1 $5}'`
echo $INT8 | grep "up"
RESULT=$?
if [ $RESULT -ne 0 ]
then
        COLOR=red
        LINE="status $BOX.int $COLOR `date` $INT8 Virtual Interface is DOWN"
fi
echo  testing $LINE
$BB $BBDISP "$LINE"
}