#!/usr/bin/perl
#
# The following line DISABLES the Nagios embedded Perl interpreter:
# nagios: -epn
#
# Nagios/Icinga plugin
#
# Monitor Dell server hardware status using Dell OpenManage Server
# Administrator, either locally via NRPE, or remotely via SNMP.
#
# $Id: check_openmanage 14630 2009-08-07 14:54:27Z trondham $
#
# Copyright (C) 2009 Trond H. Amundsen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

require 5.006;  # Perl v5.6.0 or newer is required
use strict;
use warnings;
use POSIX qw(isatty ceil);
use Getopt::Long qw(:config no_ignore_case);
use Pod::Usage;
use File::Basename qw(basename);

# Global (package) variables used throughout the code
use vars qw( $NAME $VERSION $AUTHOR $CONTACT $E_OK $E_WARNING $E_CRITICAL
	     $E_UNKNOWN $FW_LOCK $snmp_session $snmp_error $omreport
	     $component $linebreak $globalstatus $no_of_pdisks
	     $no_of_vdisks $omopt_chassis $omopt_system $blade
	     $exit_code $i_count $snmp
	     %check %opt %perfdata %reverse_exitcode %status2nagios
	     %snmp_status %snmp_probestatus %probestatus2nagios %sysinfo
	     %blacklist %nagios_alert_count %h_count
	     @controllers @enclosures
	     @report_storage @report_chassis @report_other
	  );

#---------------------------------------------------------------------
# Initialization and global variables
#---------------------------------------------------------------------

# Version and similar info
$NAME    = 'check_openmanage';
$VERSION = '3.4.9';
$AUTHOR  = 'Trond H. Amundsen';
$CONTACT = 't.h.amundsen@usit.uio.no';

# Exit codes
$E_OK       = 0;
$E_WARNING  = 1;
$E_CRITICAL = 2;
$E_UNKNOWN  = 3;

# Firmware update lock file [FIXME: location on Windows?]
$FW_LOCK = '/var/lock/.spsetup';  # default on Linux

# Options with default values
%opt = ( 'blacklist'         => [],
	 'check'             => [],
	 'critical'          => [],
	 'warning'           => [],
	 'timeout'           => 30,  # default timeout is 30 seconds
	 'verbose'           => 0,
	 'only-critical'     => 0,
	 'only-warning'      => 0,
	 'help'              => 0,
	 'man'               => 0,
	 'perfdata'          => undef,
	 'info'              => 0,
	 'extinfo'           => 0,
	 'htmlinfo'          => undef,
	 'postmsg'           => undef,
	 'state'             => 0,
	 'short-state'       => 0,
	 'okinfo'            => 0,   # default "ok" output level
	 'linebreak'         => undef,
	 'version'           => 0,
	 'global'            => 0,
	 'snmp'              => 0,
	 'port'              => 161, # default port is the well-known SNMP port 161
	 'hostname'          => undef,
	 'community'         => 'public',  # SMNP v1 or v2c
	 'protocol'          => 2,
	 'username'          => undef, # SMNP v3
	 'authpassword'      => undef, # SMNP v3
	 'authkey'           => undef, # SMNP v3
	 'authprotocol'      => undef, # SMNP v3
	 'privpassword'      => undef, # SMNP v3
	 'privkey'           => undef, # SMNP v3
	 'privprotocol'      => undef, # SMNP v3
       );

# Get options
GetOptions('b|blacklist=s'      => \@{ $opt{blacklist} },
	   'check=s'            => \@{ $opt{check} },
	   'c|critical=s'       => \@{ $opt{critical} },
	   'w|warning=s'        => \@{ $opt{warning} },
	   't|timeout=i'        => \$opt{timeout},
	   'v|verbose'          => \$opt{verbose},
	   'only-critical'      => \$opt{only_critical},
	   'only-warning'       => \$opt{only_warning},
	   'h|help'             => \$opt{help},
	   'm|man'              => \$opt{man},
	   'V|version'          => \$opt{version},
	   'p|perfdata:s'       => \$opt{perfdata},
	   'i|info'             => \$opt{info},
	   'e|extinfo'          => \$opt{extinfo},
	   'htmlinfo:s'         => \$opt{htmlinfo},
	   'postmsg=s'          => \$opt{postmsg},
	   'state'              => \$opt{state},
	   'short-state'        => \$opt{shortstate},
	   'o|ok-info=i'        => \$opt{okinfo},
	   'l|linebreak=s'      => \$opt{linebreak},
	   'g|global'           => \$opt{global},
	   's|snmp'             => \$opt{snmp},
	   'port=i'             => \$opt{port},
	   'H|hostname=s'       => \$opt{hostname},
	   'C|community=s'      => \$opt{community},
	   'P|protocol=i'       => \$opt{protocol},
	   'U|username=s'       => \$opt{username},
	   'authpassword=s'     => \$opt{authpassword},
	   'authkey=s'          => \$opt{authkey},
	   'authprotocol=s'     => \$opt{authprotocol},
	   'privpassword=s'     => \$opt{privpassword},
	   'privkey=s'          => \$opt{privkey},
	   'privprotocol=s'     => \$opt{privprotocol},
	  ) or pod2usage(-exitstatus => $E_UNKNOWN, -verbose => 0);

# If user requested help
if ($opt{help}) {
    pod2usage(-exitstatus => $E_OK, -verbose => 1);
}

# If user requested man page
if ($opt{man}) {
    pod2usage(-exitstatus => $E_OK, -verbose => 2);
}

# If user requested version info
if ($opt{version}) {
    print <<"END_VERSION";
$NAME $VERSION
Copyright (C) 2009 $AUTHOR
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.

Written by $AUTHOR <$CONTACT>
END_VERSION
    exit $E_OK;
}

# Wrong option usage
if ($opt{only_critical} and $opt{only_warning}) {
    print "ERROR: Can't use both '--only-critical' and '--only-warning' options at once\n";
    exit $E_UNKNOWN;
}

# Setting timeout
$SIG{ALRM} = sub {
    print "PLUGIN TIMEOUT: $NAME timed out after $opt{timeout} seconds\n";
    exit $E_UNKNOWN;
};
alarm $opt{timeout};

# If we're using SNMP
$snmp = defined $opt{hostname} ? 1 : 0;

# SNMP session variables
$snmp_session = undef;
$snmp_error   = undef;

# The omreport command
$omreport = undef;

# Check flags, override available with the --check option
%check = ( 'storage'     => 1,   # check storage subsystem
	   'memory'      => 1,   # check memory (dimms)
	   'fans'        => 1,   # check fan status
	   'power'       => 1,   # check power supplies
	   'temperature' => 1,   # check temperature
	   'cpu'         => 1,   # check processors
	   'voltage'     => 1,   # check voltage
	   'batteries'   => 1,   # check battery probes
	   'pwrmonitor'  => 1,   # check power consumption
	   'intrusion'   => 0,   # check intrusion detection
	   'alertlog'    => 0,   # check the alert log
	   'esmlog'      => 0,   # check the ESM log (hardware log)
	   'esmhealth'   => 1,   # check the ESM log overall health
	 );

# If user has specified the '--global' option, which implies that
# everything should be checked.
if ($opt{global}) {
     $check{intrusion} = 1;  # turn on chassis intrusion check
}

# If we were called with alternate basename, check to see which
# component should be checked
$component = find_component();

# Default line break
$linebreak = isatty(*STDOUT) ? "\n" : '<br/>';

# Line break from option
if (defined $opt{linebreak}) {
    if ($opt{linebreak} eq 'REG') {
	$linebreak = "\n";
    }
    elsif ($opt{linebreak} eq 'HTML') {
	$linebreak = '<br/>';
    }
    else {
	$linebreak = $opt{linebreak};
    }
}

# Exit with status=UNKNOWN if there is firmware upgrade in progress
if (!$snmp && -f $FW_LOCK) {
    print "MONITORING DISABLED - Firmware update in progress ($FW_LOCK exists)\n";
    exit $E_UNKNOWN;
}

# List of controllers and enclosures
@controllers = ();  # controllers
@enclosures  = ();  # enclosures

# Messages
@report_storage = ();  # messages with associated nagios level (storage)
@report_chassis = ();  # messages with associated nagios level (chassis)
@report_other   = ();  # messages with associated nagios level (other)

# Counters
$no_of_pdisks = 0;   # number of physical disks
$no_of_vdisks = 0;   # number of logical drives (virtual disks)

# Performance data
%perfdata = ();

# global health status
$globalstatus   = $E_OK;

# Nagios error levels reversed
%reverse_exitcode
  = (
     $E_OK       => 'OK',
     $E_WARNING  => 'WARNING',
     $E_CRITICAL => 'CRITICAL',
     $E_UNKNOWN  => 'UNKNOWN',
    );

# OpenManage (omreport) and SNMP error levels
%status2nagios
  = (
     'Unknown'         => $E_CRITICAL,
     'Critical'        => $E_CRITICAL,
     'Non-Critical'    => $E_WARNING,
     'Ok'              => $E_OK,
     'Non-Recoverable' => $E_CRITICAL,
     'Other'           => $E_CRITICAL,
    );

# Status via SNMP
%snmp_status
  = (
     1 => 'Other',
     2 => 'Unknown',
     3 => 'Ok',
     4 => 'Non-Critical',
     5 => 'Critical',
     6 => 'Non-Recoverable',
    );

# Probe Status via SNMP
%snmp_probestatus
  = (
     1  => 'Other',               # probe status is not one of the following:
     2  => 'Unknown',             # probe status is unknown (not known or monitored)
     3  => 'Ok',                  # probe is reporting a value within the thresholds
     4  => 'nonCriticalUpper',    # probe has crossed upper noncritical threshold
     5  => 'criticalUpper',       # probe has crossed upper critical threshold
     6  => 'nonRecoverableUpper', # probe has crossed upper non-recoverable threshold
     7  => 'nonCriticalLower',    # probe has crossed lower noncritical threshold
     8  => 'criticalLower',       # probe has crossed lower critical threshold
     9  => 'nonRecoverableLower', # probe has crossed lower non-recoverable threshold
     10 => 'failed',              # probe is not functional
    );

# Probe status translated to Nagios alarm levels
%probestatus2nagios
  = (
     'Other'               => $E_CRITICAL,
     'Unknown'             => $E_CRITICAL,
     'Ok'                  => $E_OK,
     'nonCriticalUpper'    => $E_WARNING,
     'criticalUpper'       => $E_CRITICAL,
     'nonRecoverableUpper' => $E_CRITICAL,
     'nonCriticalLower'    => $E_WARNING,
     'criticalLower'       => $E_CRITICAL,
     'nonRecoverableLower' => $E_CRITICAL,
     'failed'              => $E_CRITICAL,
    );

# System information gathered
%sysinfo
  = (
     'bios'     => 'N/A',  # BIOS version
     'biosdate' => 'N/A',  # BIOS release date
     'serial'   => 'N/A',  # serial number (service tag)
     'model'    => 'N/A',  # system model
     'osname'   => 'N/A',  # OS name
     'osver'    => 'N/A',  # OS version
     'om'       => 'N/A',  # OMSA version
     'bmc'      => 0,      # HAS baseboard management controller (BMC)
     'rac'      => 0,      # HAS remote access controller (RAC)
     'rac_name' => 'N/A',  # remote access controller (RAC)
     'bmc_fw'   => 'N/A',  # BMC firmware
     'rac_fw'   => 'N/A',  # RAC firmware
    );

# Adjust which checks to perform
adjust_checks() if defined $opt{check};

# Blacklisted components
%blacklist = defined $opt{blacklist} ? %{ get_blacklist() } : ();

# If blacklisting is in effect, the --global option is negated
if (scalar keys %blacklist > 0 && $opt{global}) {
    $opt{global} = 0;
}

# Take into account new hardware and blades
$omopt_chassis = 'chassis';  # default "chassis" option to omreport
$omopt_system  = 'system';   # default "system" option to omreport
$blade         = 0;          # if this is a blade system

# Some initializations and checking before we begin
if ($snmp) {
    snmp_initialize();    # initialize SNMP
    snmp_check();         # check that SNMP works
    snmp_detect_blade();  # detect blade via SNMP
}
else {
    # Find the omreport binary
    find_omreport();
    # Check help output from omreport, see which options are available.
    # Also detecting blade via omreport.
    check_omreport_options();
}


#---------------------------------------------------------------------
# Helper functions
#---------------------------------------------------------------------

#
# Store a message in one of the message arrays
#
sub report {
    my ($type, $msg, $exval, $id) = @_;
    defined $id or $id = q{};

    my %type2array
      = (
	 'storage' => \@report_storage,
	 'chassis' => \@report_chassis,
	 'other'   => \@report_other,
	);

    return push @{ $type2array{$type} }, [ $msg, $exval, $id ];
}


#
# Run command, put resulting output lines in an array and return a
# pointer to that array
#
sub run_command {
    my $command = shift;

    open my $CMD, '-|', $command
      or do { report('other', "Couldn't run command '$command': $!", $E_UNKNOWN)
		and return [] };
    my @lines = <$CMD>;
    close $CMD
      or do { report('other', "Couldn't close filehandle for command '$command': $!", $E_UNKNOWN)
		and return \@lines };
    return \@lines;
}

#
# Run command, put resulting output in a string variable and return it
#
sub slurp_command {
    my $command = shift;

    open my $CMD, '-|', $command
      or do { report('other', "Couldn't run command '$command': $!", $E_UNKNOWN) and return };
    my $rawtext = do { local $/ = undef; <$CMD> }; # slurping
    close $CMD;

    # NOTE: We don't check the return value of close() since omreport
    # does something weird sometimes.

    return $rawtext;
}

#
# If we were called with alternate basename, check to see which
# component should be checked
#
sub find_component {
    my $self = basename($0);
    my $comp = undef;
    if ($self =~ m/\A ${NAME}_(.+?)(\.exe)? \z/xms) {  # matches "$NAME_foo" and "$NAME_foo.exe"
	$comp = $1;
	if (!exists $check{$comp}) {
	    print "CONFIGURATION ERROR: Unknown component '$comp'. Check plugin filename\n";
	    exit $E_UNKNOWN;
	}
    }
    return $comp;
}


#
# Initialize SNMP
#
sub snmp_initialize {
    # Legal SNMP v3 protocols
    my $snmp_v3_privprotocol = qr{\A des|aes|aes128|3des|3desde \z}xms;
    my $snmp_v3_authprotocol = qr{\A md5|sha \z}xms;

    # Parameters to Net::SNMP->session()
    my %param
      = (
	 '-port'     => $opt{port},
	 '-hostname' => $opt{hostname},
	 '-version'  => $opt{protocol},
	);

    # Parameters for SNMP v3
    if ($opt{protocol} == 3) {

	# Username is mandatory
	if (defined $opt{username}) {
	    $param{'-username'} = $opt{username};
	}
	else {
	    print "SNMP ERROR: With SNMPv3 the username must be specified\n";
	    exit $E_UNKNOWN;
	}

	# Authpassword is optional
	if (defined $opt{authpassword}) {
	    $param{'-authpassword'} = $opt{authpassword};
	}

	# Authkey is optional
	if (defined $opt{authkey}) {
	    $param{'-authkey'} = $opt{authkey};
	}

	# Privpassword is optional
	if (defined $opt{privpassword}) {
	    $param{'-privpassword'} = $opt{privpassword};
	}

	# Privkey is optional
	if (defined $opt{privkey}) {
	    $param{'-privkey'} = $opt{privkey};
	}

	# Privprotocol is optional
	if (defined $opt{privprotocol}) {
	    if ($opt{privprotocol} =~ m/$snmp_v3_privprotocol/xms) {
		$param{'-privprotocol'} = $opt{privprotocol};
	    }
	    else {
		print "SNMP ERROR: Unknown privprotocol '$opt{privprotocol}', "
		  . "must be one of [des|aes|aes128|3des|3desde]\n";
		exit $E_UNKNOWN;
	    }
	}

	# Authprotocol is optional
	if (defined $opt{authprotocol}) {
	    if ($opt{authprotocol} =~ m/$snmp_v3_authprotocol/xms) {
		$param{'-authprotocol'} = $opt{authprotocol};
	    }
	    else {
		print "SNMP ERROR: Unknown authprotocol '$opt{authprotocol}', "
		  . "must be one of [md5|sha]\n";
		exit $E_UNKNOWN;
	    }
	}
    }
    # Parameters for SNMP v2c or v1
    elsif ($opt{protocol} == 2 or $opt{protocol} == 1) {
	$param{'-community'} = $opt{community};
    }
    else {
	print "SNMP ERROR: Unknown SNMP version '$opt{protocol}'\n";
	exit $E_UNKNOWN;
    }

    # Try to initialize the SNMP session
    if ( eval { require Net::SNMP; 1 } ) {
	($snmp_session, $snmp_error) = Net::SNMP->session( %param );
	if (!defined $snmp_session) {
	    printf "SNMP: %s\n", $snmp_error;
	    exit $E_UNKNOWN;
	}
    }
    else {
	print "You need perl module Net::SNMP to run $NAME in SNMP mode\n";
	exit $E_UNKNOWN;
    }
    return;
}

#
# Checking if SNMP works by probing for "chassisModelName", which all
# servers should have
#
sub snmp_check {
    my $chassisModelName = '1.3.6.1.4.1.674.10892.1.300.10.1.9.1';
    my $result = $snmp_session->get_request(-varbindlist => [$chassisModelName]);

    # Typically if remote host isn't responding
    if (!defined $result) {
	printf "SNMP CRITICAL: %s\n", $snmp_session->error;
	exit $E_CRITICAL;
    }

    # If OpenManage isn't installed or is not working
    if ($result->{$chassisModelName} =~ m{\A noSuch (Instance|Object) \z}xms) {
	print "ERROR: (SNMP) OpenManage is not installed or is not working correctly\n";
	exit $E_UNKNOWN;
    }
    return;
}

#
# Detecting blade via SNMP
#
sub snmp_detect_blade {
    my $DellBaseBoardType = '1.3.6.1.4.1.674.10892.1.300.80.1.7.1.1';
    my $result = $snmp_session->get_request(-varbindlist => [$DellBaseBoardType]);

    # Identify blade. Older models (4th and 5th gen models) and/or old
    # OMSA (4.x) don't have this OID. If we get "noSuchInstance" or
    # similar, we assume that this isn't a blade
    if ($result->{$DellBaseBoardType} eq '3') {
	$blade = 1;
    }
    return;
}

#
# Locate the omreport binary
#
sub find_omreport {
    # Possible full paths for omreport
    my @omreport_paths
      = (
	 '/usr/bin/omreport',                            # default on Linux
	 '/opt/dell/srvadmin/oma/bin/omreport.sh',       # alternate on Linux
	 '/opt/dell/srvadmin/oma/bin/omreport',          # alternate on Linux
	 'c:\progra~1\dell\sysmgt\oma\bin\omreport.exe', # default on Windows
	 'c:\progra~2\dell\sysmgt\oma\bin\omreport.exe', # default on Windows x64
	);

    # Find the one to use
  OMREPORT_PATH:
    foreach my $bin (@omreport_paths) {
	if (-x $bin) {
	    $omreport = $bin;
	    last OMREPORT_PATH;
	}
    }

    # Exit with status=UNKNOWN if OM is not installed, or we don't
    # have permission to execute the binary
    if (!defined $omreport) {
	print "ERROR: Dell OpenManage Server Administrator (OMSA) is not installed\n";
	exit $E_UNKNOWN;
    }
    return;
}

#
# Checks output from 'omreport -?' and searches for arguments to
# omreport, to accommodate deprecated options "chassis" and "system"
# (on newer hardware), as well as blade servers.
#
sub check_omreport_options {
    foreach (@{ run_command("$omreport -? 2>&1") }) {
       if (m/\A servermodule /xms) {
	   # If "servermodule" argument to omreport exists, use it
	   # instead of argument "system"
           $omopt_system = 'servermodule';
       }
       elsif (m/\A mainsystem /xms) {
	   # If "mainsystem" argument to omreport exists, use it
	   # instead of argument "chassis"
           $omopt_chassis = 'mainsystem';
       }
       elsif (m/\A modularenclosure /xms) {
	   # If "modularenclusure" argument to omreport exists, assume
	   # that this is a blade
           $blade = 1;
       }
    }
    return;
}

#
# Read the blacklist option and return a hash containing the
# blacklisted components
#
sub get_blacklist {
    my @bl = ();
    my %blacklist = ();

    if (scalar @{ $opt{blacklist} } >= 0) {
	foreach my $black (@{ $opt{blacklist} }) {
	    my $tmp = q{};
	    if (-f $black) {
		open my $BL, '<', $black
		  or do { report('other', "Couldn't open blacklist file $black: $!", $E_UNKNOWN)
			    and return {} };
		$tmp = <$BL>;
		close $BL;
		chomp $tmp;
	    }
	    else {
		$tmp = $black;
	    }
	    push @bl, $tmp;
	}
    }

    return {} if $#bl < 0;

    # Parse blacklist string, put in hash
    foreach my $black (@bl) {
	my @comps = split m{/}xms, $black;
	foreach my $c (@comps) {
	    next if $c !~ m/=/xms;
	    my ($key, $val) = split /=/xms, $c;
	    my @vals = split /,/xms, $val;
	    $blacklist{$key} = \@vals;
	}
    }

    return \%blacklist;
}

#
# Read the check option and adjust the hash %check, which is a rough
# list of components to be checked
#
sub adjust_checks {
    my @cl = ();

    if (scalar @{ $opt{check} } >= 0) {
	foreach my $check (@{ $opt{check} }) {
	    my $tmp = q{};
	    if (-f $check) {
		open my $CL, '<', $check
		  or do { report('other', "Couldn't open check file $check: $!", $E_UNKNOWN) and return };
		$tmp = <$CL>;
		close $CL;
	    }
	    else {
		$tmp = $check;
	    }
	    push @cl, $tmp;
	}
    }

    return if $#cl < 0;

    # Parse checklist string, put in hash
    foreach my $check (@cl) {
	my @checks = split /,/xms, $check;
	foreach my $c (@checks) {
	    next if $c !~ m/=/xms;
	    my ($key, $val) = split /=/xms, $c;
	    if ($opt{global} and $key !~ m/ esmlog | alertlog /xms) {
		# If the '--global' switch is specified, you're only
		# allowed to mess with the log stuff
		next;
	    }
	    $check{$key} = $val;
	}
    }
    return;
}

#
# Runs omreport and returns an array of anonymous hashes containing
# the output.
# Takes one argument: string containing parameters to omreport
#
sub run_omreport {
    my $command = shift;
    my @output  = ();
    my @keys    = ();

    # Errors that are OK. Some low-end poweredge (and blades) models
    # don't have RAID controllers, intrusion detection sensor, or
    # redundant/instrumented power supplies etc.
    my $ok_errors
      = qr{
            Intrusion\sinformation\sis\snot\sfound\sfor\sthis\ssystem  # No intrusion probe
          | No\sinstrumented\spower\ssupplies\sfound\son\sthis\ssystem # No instrumented PS (blades/low-end)
          | No\scontrollers\sfound                                     # No RAID controller
          | No\sbattery\sprobes\sfound\son\sthis\ssystem               # No battery probes
          | Invalid\scommand:\spwrmonitoring                           # Older OMSAs lack this command(?)
        }xms;

    # Errors that are OK on blade servers
    my $ok_blade_errors
      = qr{
              No\sfan\sprobes\sfound\son\sthis\ssystem   # No fan probes
      }xms;

    # Run omreport and fetch output
    my $rawtext = slurp_command("$omreport $command -fmt ssv 2>&1");
    return [] if !defined $rawtext;

    # Workaround for Openmanage BUG introduced in OMSA 5.5.0
    $rawtext =~ s/\n;/;/gxms if $command eq 'storage controller';

    # Parse output, store in array
    for ((split /\n/xms, $rawtext)) {
	if (m/\A Error/xms) {
	    next if m{$ok_errors}xms;
	    next if ($blade and m{$ok_blade_errors}xms);
	    report('other', "Problem running 'omreport $command': $_", $E_UNKNOWN);
	}

	next if !m/(.*?;){2}/xms;  # ignore lines with less than 3 fields
	my @vals = split /;/xms;
	if ($vals[0] =~ m/\A (Index|ID|Severity) \z/xms) {
	    @keys = @vals;
	}
	else {
	    my $i = 0;
	    push @output, { map { $_ => $vals[$i++] } @keys };
	}

    }

    # Finally, return the collected information
    return \@output;
}


#
# Checks if a component is blacklisted. Returns 1 if the component is
# blacklisted, 0 otherwise. Takes two arguments:
#   arg1: component name
#   arg2: component id or index
#
sub blacklisted {
    my $name = shift;  # component name
    my $id   = shift;  # component id
    my $ret  = 0;      # return value

    if (defined $blacklist{$name}) {
	foreach my $comp (@{ $blacklist{$name} }) {
	    if (defined $id and $comp eq $id) {
		$ret = 1;
	    }
	}
    }

    return $ret;
}

# Converts the NexusID from SNMP to our version
sub convert_nexus {
    my $nexus = shift;
    $nexus =~ s{\A \\}{}xms;
    $nexus =~ s{\\}{:}gxms;
    return $nexus;
}

# Sets custom temperature thresholds based on user supplied options
sub custom_temperature_thresholds {
    my $type   = shift; # type of threshold, either w (warning) or c (critical)
    my %thres  = ();    # will contain the thresholds
    my @limits = ();    # holds the input

    my @opt =  $type eq 'w' ? @{ $opt{warning} } : @{ $opt{critical} };

    if (scalar @opt >= 0) {
	foreach my $t (@opt) {
	    my $tmp = q{};
	    if (-f $t) {
		open my $F, '<', $t
		  or do { report('other', "Couldn't open temperature threshold file $t: $!",
				 $E_UNKNOWN) and return {} };
		$tmp = <$F>;
		close $F;
	    }
	    else {
		$tmp = $t;
	    }
	    push @limits, $tmp;
	}
    }

    # Parse checklist string, put in hash
    foreach my $th (@limits) {
	my @tmp = split m{,}xms, $th;
	foreach my $t (@tmp) {
	    next if $t !~ m{=}xms;
	    my ($key, $val) = split m{=}xms, $t;
	    if ($val =~ m{/}xms) {
		my ($max, $min) = split m{/}xms, $val;
		$thres{$key}{max} = $max;
		$thres{$key}{min} = $min;
	    }
	    else {
		$thres{$key}{max} = $val;
	    }
	}
    }

    return \%thres;
}


# Gets the output from SNMP result according to the OIDs checked
sub get_snmp_output {
    my ($result,$oidref) = @_;
    my @output = ();

    foreach my $oid (keys %{ $result }) {
	my @dummy = split /\./xms, $oid;
	my $id = pop @dummy;
	--$id;
	my $foo = join q{.}, @dummy;
	if (exists $oidref->{$foo}) {
	    $output[$id]{$oidref->{$foo}} = $result->{$oid};
	}
    }
    return \@output;
}


# Map the controller or other item in-place
sub map_item {
    my ($key, $val, $list)  = @_;

    foreach my $lst (@{ $list }) {
	if (!exists $lst->{$key}) {
	    $lst->{$key} = $val;
	}
    }
    return;
}

# Return the URL for official Dell documentation for a specific
# PowerEdge server
sub documentation_url {
    my $model = shift;

    # create model short form, e.g. "r710"
    $model =~ s{\A PowerEdge \s (.+?) \z}{lc($1)}exms;

    # special case for blades (e.g. M600, M710), they have common
    # documentation
    $model =~ s{\A m\d+ \z}{m}xms;

    return 'http://support.dell.com/support/edocs/systems/pe' . $model . '/';
}

# Return the URL for warranty information for a server with a given
# serial number (servicetag)
sub warranty_url {
    my $tag = shift;

    # Dell support sites for different parts of the world
    my %supportsite
      = (
	 'emea' => 'http://support.euro.dell.com/support/topics/topic.aspx/emea/shared/support/my_systems_info/',
	 'ap'   => 'http://supportapj.dell.com/support/topics/topic.aspx/ap/shared/support/my_systems_info/en/details?',
	 'glob' => 'http://support.dell.com/support/topics/global.aspx/support/my_systems_info/details?',
	);

    # warranty URLs for different country codes
    my %url
      = (
	 # EMEA
	 'at' => $supportsite{emea} . 'de/details?c=at&l=de&ServiceTag=',  # Austria
	 'be' => $supportsite{emea} . 'nl/details?c=be&l=nl&ServiceTag=',  # Belgium
	 'cz' => $supportsite{emea} . 'cs/details?c=cz&l=cs&ServiceTag=',  # Czech Republic
	 'de' => $supportsite{emea} . 'de/details?c=de&l=de&ServiceTag=',  # Germany
	 'dk' => $supportsite{emea} . 'da/details?c=dk&l=da&ServiceTag=',  # Denmark
	 'es' => $supportsite{emea} . 'es/details?c=es&l=es&ServiceTag=',  # Spain
	 'fi' => $supportsite{emea} . 'fi/details?c=fi&l=fi&ServiceTag=',  # Finland
	 'fr' => $supportsite{emea} . 'fr/details?c=fr&l=fr&ServiceTag=',  # France
	 'gr' => $supportsite{emea} . 'en/details?c=gr&l=el&ServiceTag=',  # Greece
	 'it' => $supportsite{emea} . 'it/details?c=it&l=it&ServiceTag=',  # Italy
	 'il' => $supportsite{emea} . 'en/details?c=il&l=en&ServiceTag=',  # Israel
	 'me' => $supportsite{emea} . 'en/details?c=me&l=en&ServiceTag=',  # Middle East
	 'no' => $supportsite{emea} . 'no/details?c=no&l=no&ServiceTag=',  # Norway
	 'nl' => $supportsite{emea} . 'nl/details?c=nl&l=nl&ServiceTag=',  # The Netherlands
	 'pl' => $supportsite{emea} . 'pl/details?c=pl&l=pl&ServiceTag=',  # Poland
	 'pt' => $supportsite{emea} . 'en/details?c=pt&l=pt&ServiceTag=',  # Portugal
	 'ru' => $supportsite{emea} . 'ru/details?c=ru&l=ru&ServiceTag=',  # Russia
	 'se' => $supportsite{emea} . 'sv/details?c=se&l=sv&ServiceTag=',  # Sweden
	 'uk' => $supportsite{emea} . 'en/details?c=uk&l=en&ServiceTag=',  # United Kingdom
	 'za' => $supportsite{emea} . 'en/details?c=za&l=en&ServiceTag=',  # South Africa
	 # America
	 'br' => $supportsite{glob} . 'c=br&l=pt&ServiceTag=',  # Brazil
	 'ca' => $supportsite{glob} . 'c=ca&l=en&ServiceTag=',  # Canada
	 'mx' => $supportsite{glob} . 'c=mx&l=es&ServiceTag=',  # Mexico
	 'us' => $supportsite{glob} . 'c=us&l=en&ServiceTag=',  # USA
	 # Asia/Pacific
	 'au' => $supportsite{ap} . 'c=au&l=en&ServiceTag=',  # Australia
	 'cn' => $supportsite{ap} . 'c=cn&l=zh&ServiceTag=',  # China
	 'in' => $supportsite{ap} . 'c=in&l=en&ServiceTag=',  # India
	 # default fallback
	 'XX' => $supportsite{glob} . 'ServiceTag=',  # default
	);

    if (exists $url{$opt{htmlinfo}}) {
	return $url{$opt{htmlinfo}} . $tag;
    }
    else {
	return $url{XX} . $tag;
    }
}



#---------------------------------------------------------------------
# Check functions
#---------------------------------------------------------------------

#-----------------------------------------
# Check global health status
#-----------------------------------------
sub check_global {
    my $health = $E_OK;

    if ($snmp) {
	#
	# Checks global status, i.e. both storage and chassis
	#
	my $systemStateGlobalSystemStatus = '1.3.6.1.4.1.674.10892.1.200.10.1.2.1';
	my $result = $snmp_session->get_request(-varbindlist => [$systemStateGlobalSystemStatus]);
	if (!defined $result) {
	    printf "SNMP [systemStateGlobalSystemStatus]: %s\n", $snmp_error;
	    exit $E_UNKNOWN;
	}
	$health = $status2nagios{$snmp_status{$result->{$systemStateGlobalSystemStatus}}};
    }
    else {
	#
	# NB! This does not check storage, only chassis...
	#
	foreach (@{ run_command("$omreport $omopt_system -fmt ssv") }) {
	    next if !m/;/xms;
	    next if m/\A SEVERITY;COMPONENT/xms;
	    if (m/\A (.+?);Main\sSystem(\sChassis)? /xms) {
		$health = $status2nagios{$1};
		last;
	    }
	}
    }

    # Return the status
    return $health;
}


#-----------------------------------------
# STORAGE: Check controllers
#-----------------------------------------
sub check_controllers {
    my $id       = undef;
    my $nexus    = undef;
    my $name     = undef;
    my $state    = undef;
    my $status   = undef;
    my $minfw    = undef;
    my $mindr    = undef;
    my $firmware = undef;
    my $driver   = undef;
    my @output   = ();

    if ($snmp) {
	my %ctrl_oid
	  = (
	     '1.3.6.1.4.1.674.10893.1.20.130.1.1.1'  => 'controllerNumber',
	     '1.3.6.1.4.1.674.10893.1.20.130.1.1.2'  => 'controllerName',
	     '1.3.6.1.4.1.674.10893.1.20.130.1.1.5'  => 'controllerState',
	     '1.3.6.1.4.1.674.10893.1.20.130.1.1.8'  => 'controllerFWVersion',
	     '1.3.6.1.4.1.674.10893.1.20.130.1.1.38' => 'controllerComponentStatus',
	     '1.3.6.1.4.1.674.10893.1.20.130.1.1.39' => 'controllerNexusID',
	     '1.3.6.1.4.1.674.10893.1.20.130.1.1.41' => 'controllerDriverVersion',
	     '1.3.6.1.4.1.674.10893.1.20.130.1.1.44' => 'controllerMinFWVersion',
	     '1.3.6.1.4.1.674.10893.1.20.130.1.1.45' => 'controllerMinDriverVersion',
	    );

	# We use get_table() here for the odd case where a server has
	# two or more controllers, and where some OIDs are missing on
	# one of the controllers.
	my $controllerTable = '1.3.6.1.4.1.674.10893.1.20.130.1';
	my $result = $snmp_session->get_table(-baseoid => $controllerTable);

	# No controllers is OK
	return if !defined $result;

	@output = @{ get_snmp_output($result, \%ctrl_oid) };
    }
    else {
	@output = @{ run_omreport('storage controller') };
    }

    my %ctrl_state
      = (
	 0 => 'Unknown',
	 1 => 'Ready',
	 2 => 'Failed',
	 3 => 'Online',
	 4 => 'Offline',
	 6 => 'Degraded',
	);

  CTRL:
    foreach my $out (@output) {
	if ($snmp) {
	    $id       = $out->{'controllerNumber'} - 1;
	    $name     = $out->{'controllerName'};
	    $state    = $ctrl_state{$out->{'controllerState'}};
	    $status   = $snmp_status{$out->{'controllerComponentStatus'}};
	    $minfw    = exists $out->{'controllerMinFWVersion'}
	      ? $out->{'controllerMinFWVersion'} : undef;
	    $mindr    = exists $out->{'controllerMinDriverVersion'}
	      ? $out->{'controllerMinDriverVersion'} : undef;
	    $firmware = exists $out->{controllerFWVersion}
	      ? $out->{controllerFWVersion} : 'N/A';
	    $driver   = exists $out->{controllerDriverVersion}
	      ? $out->{controllerDriverVersion} : 'N/A';
	    $nexus    = convert_nexus($out->{controllerNexusID});
	}
	else {
	    $id       = $out->{ID};
	    $name     = $out->{Name};
	    $state    = $out->{State};
	    $status   = $out->{Status};
	    $minfw    = $out->{'Minimum Required Firmware Version'} ne 'Not Applicable'
	      ? $out->{'Minimum Required Firmware Version'} : undef;
	    $mindr    = $out->{'Minimum Required Driver Version'} ne 'Not Applicable'
	      ? $out->{'Minimum Required Driver Version'} : undef;
	    $firmware = $out->{'Firmware Version'} ne 'Not Applicable'
	      ? $out->{'Firmware Version'} : 'N/A';
	    $driver   = $out->{'Driver Version'} ne 'Not Applicable'
	      ? $out->{'Driver Version'} : 'N/A';
	    $nexus    = $id;
	}

	$name =~ s{\s+\z}{}xms; # remove trailing whitespace
	push @controllers, $id;

	# Collecting some storage info
	$sysinfo{'controller'}{$id}{'id'}       = $nexus;
	$sysinfo{'controller'}{$id}{'name'}     = $name;
	$sysinfo{'controller'}{$id}{'driver'}   = $driver;
	$sysinfo{'controller'}{$id}{'firmware'} = $firmware;

	next CTRL if blacklisted('ctrl', $nexus);

	# Special case: old firmware
	if (!blacklisted('ctrl_fw', $id) && defined $minfw) {
	    chomp $firmware;
	    my $msg = sprintf 'Controller %d (%s): Firmware is out of date (%s)',
	      $id, $name, $firmware;
	    report('storage', $msg, $E_WARNING, $nexus);
	}
	# Special case: old driver
	if (!blacklisted('ctrl_driver', $id) && defined $mindr) {
	    chomp $driver;
	    my $msg = sprintf 'Controller %d (%s): Driver is out of date (%s)',
	      $id, $name, $driver;
	    report('storage', $msg, $E_WARNING, $nexus);
	}
	# Ok
	if ($status eq 'Ok' or ($status eq 'Non-Critical'
				and (defined $minfw or defined $mindr))) {
	    my $msg = sprintf 'Controller %d (%s) is %s',
	      $id, $name, $state;
	    report('storage', $msg, $E_OK, $nexus);
	}
        # Default
	else {
	    my $msg = sprintf 'Controller %d (%s) needs attention: %s',
	      $id, $name, $state;
	    report('storage', $msg, $status2nagios{$status}, $nexus);
	}
    }
    return;
}


#-----------------------------------------
# STORAGE: Check physical drives
#-----------------------------------------
sub check_physical_disks {
    return if $#controllers == -1;

    my $id       = undef;
    my $nexus    = undef;
    my $name     = undef;
    my $state    = undef;
    my $status   = undef;
    my $fpred    = undef;
    my $progr    = undef;
    my $ctrl     = undef;
    my $vendor   = undef;  # disk vendor
    my $product  = undef;  # product ID
    my $capacity = undef;  # disk length (size) in bytes
    my @output  = ();

    if ($snmp) {
	my %pdisk_oid
	  = (
	     '1.3.6.1.4.1.674.10893.1.20.130.4.1.1'  => 'arrayDiskNumber',
	     '1.3.6.1.4.1.674.10893.1.20.130.4.1.2'  => 'arrayDiskName',
	     '1.3.6.1.4.1.674.10893.1.20.130.4.1.3'  => 'arrayDiskVendor',
	     '1.3.6.1.4.1.674.10893.1.20.130.4.1.4'  => 'arrayDiskState',
	     '1.3.6.1.4.1.674.10893.1.20.130.4.1.6'  => 'arrayDiskProductID',
	     '1.3.6.1.4.1.674.10893.1.20.130.4.1.9'  => 'arrayDiskEnclosureID',
	     '1.3.6.1.4.1.674.10893.1.20.130.4.1.10' => 'arrayDiskChannel',
	     '1.3.6.1.4.1.674.10893.1.20.130.4.1.11' => 'arrayDiskLengthInMB',
	     '1.3.6.1.4.1.674.10893.1.20.130.4.1.15' => 'arrayDiskTargetID',
	     '1.3.6.1.4.1.674.10893.1.20.130.4.1.16' => 'arrayDiskLunID',
	     '1.3.6.1.4.1.674.10893.1.20.130.4.1.24' => 'arrayDiskComponentStatus',
	     '1.3.6.1.4.1.674.10893.1.20.130.4.1.26' => 'arrayDiskNexusID',
	     '1.3.6.1.4.1.674.10893.1.20.130.4.1.31' => 'arrayDiskSmartAlertIndication',
	     '1.3.6.1.4.1.674.10893.1.20.130.5.1.5'  => 'arrayDiskEnclosureConnectionEnclosureNumber',
	     '1.3.6.1.4.1.674.10893.1.20.130.5.1.7'  => 'arrayDiskEnclosureConnectionControllerNumber',
	    );
	my $result = $snmp_session->get_entries(-columns => [keys %pdisk_oid]);

	if (!defined $result) {
	    printf "SNMP [storage / pdisk]: %s.\n", $snmp_session->error;
	    $snmp_session->close;
	    exit $E_UNKNOWN;
	}

	@output = @{ get_snmp_output($result, \%pdisk_oid) };
    }
    else {
	foreach my $c (@controllers) {
	    push @output, @{ run_omreport("storage pdisk controller=$c") };
	    map_item('ctrl', $c, \@output);
	}
    }

    my %pdisk_state
      = (
	 0  => 'Unknown',
	 1  => 'Ready',
	 2  => 'Failed',
	 3  => 'Online',
	 4  => 'Offline',
	 6  => 'Degraded',
	 7  => 'Recovering',
	 11 => 'Removed',
	 15 => 'Resynching',
	 24 => 'Rebuilding',
	 25 => 'No Media',
	 26 => 'Formatting',
	 28 => 'Diagnostics',
	 34 => 'Predictive failure',
	 35 => 'Initializing',
	 39 => 'Foreign',
	 40 => 'Clear',
	 41 => 'Unsupported',
	 53 => 'Incompatible',
	);

    # Check physical disks on each of the controllers
  PDISK:
    foreach my $out (@output) {
	if ($snmp) {
	    $name   = $out->{arrayDiskName};
	    if ($name =~ m{.*\d+:\d+:\d+\z}xms) {
		$id = join q{:}, ($out->{arrayDiskChannel}, $out->{arrayDiskEnclosureID},
				 $out->{arrayDiskTargetID});
	    }
	    else {
		$id = join q{:}, ($out->{arrayDiskChannel}, $out->{arrayDiskTargetID});
	    }
	    $state    = $pdisk_state{$out->{arrayDiskState}};
	    $status   = $snmp_status{$out->{arrayDiskComponentStatus}};
	    $fpred    = $out->{arrayDiskSmartAlertIndication} == 2 ? 1 : 0;
	    $progr    = q{};
	    $ctrl     = exists $out->{arrayDiskEnclosureConnectionControllerNumber}
	      ? $out->{arrayDiskEnclosureConnectionControllerNumber} - 1
		: -1;
	    $nexus    = convert_nexus($out->{arrayDiskNexusID});
	    $vendor   = $out->{arrayDiskVendor};
	    $product  = $out->{arrayDiskProductID};
	    $capacity = $out->{arrayDiskLengthInMB} * 1024**2;
	}
	else {
	    $id       = $out->{'ID'};
	    $name     = $out->{'Name'};
	    $state    = $out->{'State'};
	    $status   = $out->{'Status'};
	    $fpred    = lc($out->{'Failure Predicted'}) eq 'yes' ? 1 : 0;
	    $progr    = ' [' . $out->{'Progress'} . ']';
	    $ctrl     = $out->{'ctrl'};
	    $nexus    = join q{:}, $out->{ctrl}, $id;
	    $vendor   = $out->{'Vendor ID'};
	    $product  = $out->{'Product ID'};
	    $capacity = $out->{'Capacity'};
	    $capacity =~ s{\A .*? \((\d+) \s bytes\) \z}{$1}xms;
	}

	next PDISK if blacklisted('pdisk', $nexus);
	$no_of_pdisks++;

        $vendor  =~ s{\s+\z}{}xms; # remove trailing whitespace
        $product =~ s{\s+\z}{}xms; # remove trailing whitespace

	# Calculate human readable capacity
	$capacity = ceil($capacity / 1000**3) >= 1000
          ? sprintf '%.1fTB', ($capacity / 1000**4)
	    : sprintf '%.0fGB', ($capacity / 1000**3);
	$capacity = '450GB' if $capacity eq '449GB';  # quick fix for 450GB disks
	$capacity = '146GB' if $capacity eq '147GB';  # quick fix for 146GB disks

	# Capitalize only the first letter of the vendor name
	$vendor = (substr $vendor, 0, 1) . lc (substr $vendor, 1, length $vendor);

	# Remove unnecessary trademark rubbish from vendor name
	$vendor =~ s{\(tm\)\z}{}xms;

	# Special case: Failure predicted
	if ($status eq 'Non-Critical' and $fpred) {
	    my $msg = sprintf '%s (%s %s, %s) on controller %d needs attention: Failure Predicted',
	      $name, $vendor, $product, $capacity, $ctrl;
	    report('storage', $msg, $E_WARNING, $nexus);
	}
	# Special case: Rebuilding
	elsif ($state eq 'Rebuilding') {
	    my $msg = sprintf '%s (%s) on controller %d is %s%s',
	      $name, $capacity, $ctrl, $state, $progr;
	    report('storage', $msg, $E_WARNING, $nexus);
	}
	# Default
	elsif ($status ne 'Ok') {
	    my $msg =  sprintf '%s (%s %s, %s) on controller %d needs attention: %s',
	      $name, $vendor, $product, $capacity, $ctrl, $state;
	    report('storage', $msg, $status2nagios{$status}, $nexus);
	}
	# Ok
	else {
	    my $msg = sprintf '%s (%s) on controller %d is %s',
	      $name, $capacity, $ctrl, $state;
	    report('storage', $msg, $E_OK, $nexus);
	}
    }
    return;
}


#-----------------------------------------
# STORAGE: Check logical drives
#-----------------------------------------
sub check_virtual_disks {
    return if $#controllers == -1;

    my $id     = undef;
    my $nexus  = undef;
    my $dev    = undef;
    my $state  = undef;
    my $status = undef;
    my $layout = undef;
    my $size   = undef;
    my $progr  = undef;
    my @output = ();

    if ($snmp) {
	my %vdisk_oid
	  = (
	     '1.3.6.1.4.1.674.10893.1.20.140.1.1.1'  => 'virtualDiskNumber',
	     '1.3.6.1.4.1.674.10893.1.20.140.1.1.2'  => 'virtualDiskName',
	     '1.3.6.1.4.1.674.10893.1.20.140.1.1.3'  => 'virtualDiskDeviceName',
	     '1.3.6.1.4.1.674.10893.1.20.140.1.1.4'  => 'virtualDiskState',
	     '1.3.6.1.4.1.674.10893.1.20.140.1.1.6'  => 'virtualDiskLengthInMB',
	     '1.3.6.1.4.1.674.10893.1.20.140.1.1.13' => 'virtualDiskLayout',
	     '1.3.6.1.4.1.674.10893.1.20.140.1.1.20' => 'virtualDiskComponentStatus',
	     '1.3.6.1.4.1.674.10893.1.20.140.1.1.21' => 'virtualDiskNexusID',
	    );
	my $result = $snmp_session->get_entries(-columns => [keys %vdisk_oid]);

	# No logical drives is OK
	return if !defined $result;

	@output = @{ get_snmp_output($result, \%vdisk_oid) };
    }
    else {
	foreach my $c (@controllers) {
	    push @output, @{ run_omreport("storage vdisk controller=$c") };
	    map_item('ctrl', $c, \@output);
	}
    }

    my %vdisk_state
      = (
	 0  => 'Unknown',
	 1  => 'Ready',
	 2  => 'Failed',
	 3  => 'Online',
	 4  => 'Offline',
	 6  => 'Degraded',
	 15 => 'Resynching',
	 16 => 'Regenerating',
	 24 => 'Rebuilding',
	 26 => 'Formatting',
	 32 => 'Reconstructing',
	 35 => 'Initializing',
	 36 => 'Background Initialization',
	 38 => 'Resynching Paused',
	 52 => 'Permanently Degraded',
	 54 => 'Degraded Redundancy',
	);

    my %vdisk_layout
      = (
	 1  => 'Concatenated',
	 2  => 'RAID-0',
	 3  => 'RAID-1',
	 7  => 'RAID-5',
	 8  => 'RAID-6',
	 10 => 'RAID-10',
	 12 => 'RAID-50',
	 19 => 'Concatenated RAID 1',
	 24 => 'RAID-60',
	);

    # Check virtual disks on each of the controllers
  VDISK:
    foreach my $out (@output) {
	if ($snmp) {
	    $id     = $out->{virtualDiskNumber} - 1;
	    $dev    = $out->{virtualDiskDeviceName};
	    $state  = $vdisk_state{$out->{virtualDiskState}};
	    $status = $snmp_status{$out->{virtualDiskComponentStatus}};
	    $layout = $vdisk_layout{$out->{virtualDiskLayout}};
	    $size   = sprintf '%.2f GB', $out->{virtualDiskLengthInMB} / 1024;
	    $progr  = q{};  # can't get this from SNMP(?)
	    $nexus  = convert_nexus($out->{virtualDiskNexusID});
	}
	else {
	    $id     = $out->{ID};
	    $dev    = $out->{'Device Name'};
	    $state  = $out->{State};
	    $status = $out->{Status};
	    $layout = $out->{Layout};
	    $size   = $out->{Size};
	    $progr  = ' [' . $out->{Progress} . ']';
	    $size   =~ s{\A (.*GB).* \z}{$1}xms;
	    $nexus  = join q{:}, $out->{ctrl}, $id;
	}

	next VDISK if blacklisted('vdisk', $nexus);
	$no_of_vdisks++;

	# Special case: Regenerating
	if ($state eq 'Regenerating') {
	    my $msg = sprintf 'Logical drive %d "%s" (%s, %s) is %s%s',
	      $id, $dev, $layout, $size, $state, $progr;
	    report('storage', $msg, $E_WARNING, $nexus);
	}
	# Default
	elsif ($status ne 'Ok') {
	    my $msg = sprintf 'Logical drive %d "%s" (%s, %s) needs attention: %s',
	      $id, $dev, $layout, $size, $state;
	    report('storage', $msg, $status2nagios{$status}, $nexus);
	}
	# Ok
	else {
	    my $msg = sprintf 'Logical drive %d "%s" (%s, %s) is %s',
	      $id, $dev, $layout, $size, $state;
	    report('storage', $msg, $E_OK, $nexus);
	}
    }
    return;
}


#-----------------------------------------
# STORAGE: Check cache batteries
#-----------------------------------------
sub check_cache_battery {
    return if $#controllers == -1;

    my $id     = undef;
    my $nexus  = undef;
    my $state  = undef;
    my $status = undef;
    my $ctrl   = undef;
    my $learn  = undef; # learn state
    my $pred   = undef; # battery's ability to be charged
    my @output = ();

    if ($snmp) {
	my %bat_oid
	  = (
	     '1.3.6.1.4.1.674.10893.1.20.130.15.1.1'  => 'batteryNumber',
	     '1.3.6.1.4.1.674.10893.1.20.130.15.1.2'  => 'batteryName',
	     '1.3.6.1.4.1.674.10893.1.20.130.15.1.4'  => 'batteryState',
	     '1.3.6.1.4.1.674.10893.1.20.130.15.1.6'  => 'batteryComponentStatus',
	     '1.3.6.1.4.1.674.10893.1.20.130.15.1.9'  => 'batteryNexusID',
	     '1.3.6.1.4.1.674.10893.1.20.130.15.1.10' => 'batteryPredictedCapacity',
	     '1.3.6.1.4.1.674.10893.1.20.130.15.1.12' => 'batteryLearnState',
	     '1.3.6.1.4.1.674.10893.1.20.130.16.1.5'  => 'batteryConnectionControllerNumber',
	    );
	my $result = $snmp_session->get_entries(-columns => [keys %bat_oid]);

	# No cache battery is OK
	return if !defined $result;

	@output = @{ get_snmp_output($result, \%bat_oid) };
    }
    else {
	foreach my $c (@controllers) {
	    push @output, @{ run_omreport("storage battery controller=$c") };
	    map_item('ctrl', $c, \@output);
	}
    }

    my %bat_state
      = (
	 0  => 'Unknown',
	 1  => 'Ready',
	 2  => 'Failed',
	 6  => 'Degraded',
	 7  => 'Reconditioning',
	 9  => 'High',
	 10 => 'Power Low',
	 12 => 'Charging',
	 21 => 'Missing',
	 36 => 'Learning',
	);

    my %bat_learn_state
      = (
	 1  => 'Failed',
	 2  => 'Active',
	 4  => 'Timed out',
	 8  => 'Requested',
	 16 => 'Idle',
	);

    my %bat_pred_cap
      = (
	 1 => 'Failed',  # The battery cannot be charged and needs to be replaced
	 2 => 'Ready',   # The battery can be charged to full capacity
	 4 => 'Unknown', # The battery is completing a Learn cycle. The charge capacity of the
                         # battery cannot be determined until the Learn cycle is complete
	);

    # Check battery on each of the controllers
  BATTERY:
    foreach my $out (@output) {
	if ($snmp) {
	    $id     = $out->{batteryNumber} - 1;
	    $state  = $bat_state{$out->{batteryState}};
	    $status = $snmp_status{$out->{batteryComponentStatus}};
	    $learn  = exists $out->{batteryLearnState}
	      ? $bat_learn_state{$out->{batteryLearnState}} : undef;
	    $pred   = exists $out->{batteryPredictedCapacity}
	      ? $bat_pred_cap{$out->{batteryPredictedCapacity}} : undef;
	    $ctrl   = $out->{batteryConnectionControllerNumber} - 1;
	    $nexus  = convert_nexus($out->{batteryNexusID});
	}
	else {
	    $id     = $out->{'ID'};
	    $state  = $out->{'State'};
	    $status = $out->{'Status'};
	    $learn  = $out->{'Learn State'};
	    $pred   = $out->{'Predicted Capacity Status'};
	    $ctrl   = $out->{'ctrl'};
	    $nexus  = join q{:}, $out->{ctrl}, $id;
	}

	next BATTERY if blacklisted('bat', $nexus);

	# Special case: Charging
	if ($state eq 'Charging') {
	    my $msg = sprintf 'Cache battery %d in controller %d is %s (%s) [probably harmless]',
	      $id, $ctrl, $state, $pred;
	    report('storage', $msg, $E_WARNING, $nexus);
	}
	# Special case: Learning (battery learns its capacity)
	elsif ($state eq 'Learning') {
	    my $msg = sprintf 'Cache battery %d in controller %d is %s (%s) [probably harmless]',
	      $id, $ctrl, $state, $learn;
	    report('storage', $msg, $E_WARNING, $nexus);
	}
	# Special case: Power Low (first part of recharge cycle)
	elsif ($state eq 'Power Low') {
	    my $msg = sprintf 'Cache battery %d in controller %d is %s [probably harmless]',
	      $id, $ctrl, $state;
	    report('storage', $msg, $E_WARNING, $nexus);
	}
	# Default
	elsif ($status ne 'Ok') {
	    my $msg = sprintf 'Cache battery %d in controller %d needs attention: %s (%s)',
	      $id, $ctrl, $state, $status;
	    report('storage', $msg, $status2nagios{$status}, $nexus);
	}
	# Ok
	else {
	    my $msg = sprintf 'Cache battery %d in controller %d is %s',
	      $id, $ctrl, $state;
	    report('storage', $msg, $E_OK, $nexus);
	}
    }
    return;
}


#-----------------------------------------
# STORAGE: Check connectors (channels)
#-----------------------------------------
sub check_connectors {
    return if $#controllers == -1;

    my $id     = undef;
    my $nexus  = undef;
    my $name   = undef;
    my $state  = undef;
    my $status = undef;
    my $type   = undef;
    my $ctrl   = undef;
    my @output = ();

    if ($snmp) {
        my %conn_oid
          = (
             '1.3.6.1.4.1.674.10893.1.20.130.2.1.1'  => 'channelNumber',
             '1.3.6.1.4.1.674.10893.1.20.130.2.1.2'  => 'channelName',
             '1.3.6.1.4.1.674.10893.1.20.130.2.1.3'  => 'channelState',
             '1.3.6.1.4.1.674.10893.1.20.130.2.1.8'  => 'channelComponentStatus',
	     '1.3.6.1.4.1.674.10893.1.20.130.2.1.9'  => 'channelNexusID',
             '1.3.6.1.4.1.674.10893.1.20.130.2.1.11' => 'channelBusType',
            );
        my $result = $snmp_session->get_entries(-columns => [keys %conn_oid]);

        if (!defined $result) {
            printf "SNMP [storage / channel]: %s.\n", $snmp_session->error;
            $snmp_session->close;
            exit $E_UNKNOWN;
        }

	@output = @{ get_snmp_output($result, \%conn_oid) };
    }
    else {
        foreach my $c (@controllers) {
            push @output, @{ run_omreport("storage connector controller=$c") };
	    map_item('ctrl', $c, \@output);
        }
    }

    my %conn_state
      = (
         0 => 'Unknown',
         1 => 'Ready',
         2 => 'Failed',
         3 => 'Online',
         4 => 'Offline',
         6 => 'Degraded',
        );

    my %conn_bustype
      = (
         1 => 'SCSI',
         2 => 'IDE',
         3 => 'Fibre Channel',
         4 => 'SSA',
         6 => 'USB',
         7 => 'SATA',
         8 => 'SAS',
        );

    # Check connectors on each of the controllers
  CHANNEL:
    foreach my $out (@output) {
        if ($snmp) {
            $id     = $out->{channelNumber} - 1;
            $name   = $out->{channelName};
            $state  = $conn_state{$out->{channelState}};
            $status = $snmp_status{$out->{channelComponentStatus}};
            $type   = $conn_bustype{$out->{channelBusType}};
	    $nexus  = convert_nexus($out->{channelNexusID});
	    $ctrl   = $nexus;
	    $ctrl   =~ s{(\d+):\d+}{$1}xms;
        }
        else {
            $id     = $out->{'ID'};
            $name   = $out->{'Name'};
            $state  = $out->{'State'};
            $status = $out->{'Status'};
            $type   = $out->{'Connector Type'};
	    $ctrl   = $out->{ctrl};
	    $nexus  = join q{:}, $out->{ctrl}, $id;
        }

        next CHANNEL if blacklisted('conn', $nexus);

	my $msg = sprintf '%s (%s) on controller %d is %s',
	  $name, $type, $ctrl, $state;
        report('storage', $msg, $status2nagios{$status}, $nexus);
    }
    return;
}


#-----------------------------------------
# STORAGE: Check enclosures
#-----------------------------------------
sub check_enclosures {
    my $id       = undef;
    my $nexus    = undef;
    my $name     = undef;
    my $state    = undef;
    my $status   = undef;
    my $firmware = undef;
    my @output   = ();

    if ($snmp) {
        my %encl_oid
          = (
             '1.3.6.1.4.1.674.10893.1.20.130.3.1.1'  => 'enclosureNumber',
             '1.3.6.1.4.1.674.10893.1.20.130.3.1.2'  => 'enclosureName',
             '1.3.6.1.4.1.674.10893.1.20.130.3.1.4'  => 'enclosureState',
	     '1.3.6.1.4.1.674.10893.1.20.130.3.1.19' => 'enclosureChannelNumber',
             '1.3.6.1.4.1.674.10893.1.20.130.3.1.24' => 'enclosureComponentStatus',
	     '1.3.6.1.4.1.674.10893.1.20.130.3.1.25' => 'enclosureNexusID',
	     '1.3.6.1.4.1.674.10893.1.20.130.3.1.26' => 'enclosureFirmwareVersion',
            );
        my $result = $snmp_session->get_entries(-columns => [keys %encl_oid]);

        # No enclosures is OK
        return if !defined $result;

	@output = @{ get_snmp_output($result, \%encl_oid) };
    }
    else {
	foreach my $c (@controllers) {
	    push @output, @{ run_omreport("storage enclosure controller=$c") };
	    map_item('ctrl', $c, \@output);
	}
    }

    my %encl_state
      = (
         0 => 'Unknown',
         1 => 'Ready',
         2 => 'Failed',
         3 => 'Online',
         4 => 'Offline',
         6 => 'Degraded',
        );

  ENCLOSURE:
    foreach my $out (@output) {
        if ($snmp) {
            $id       = $out->{'enclosureNumber'} - 1;
            $name     = $out->{'enclosureName'};
            $state    = $encl_state{$out->{'enclosureState'}};
            $status   = $snmp_status{$out->{'enclosureComponentStatus'}};
	    $firmware = exists $out->{enclosureFirmwareVersion}
	      ? $out->{enclosureFirmwareVersion} : 'N/A';
	    $nexus    = convert_nexus($out->{enclosureNexusID});
        }
        else {
            $id       = $out->{ID};
            $name     = $out->{Name};
            $state    = $out->{State};
            $status   = $out->{Status};
	    $firmware = $out->{'Firmware Version'} ne 'Not Applicable'
	      ? $out->{'Firmware Version'} : 'N/A';
	    $nexus    = join q{:}, $out->{ctrl}, $id;
        }

        $name     =~ s{\s+\z}{}xms; # remove trailing whitespace
        $firmware =~ s{\s+\z}{}xms; # remove trailing whitespace

	# store enclosure data for future use
	push @enclosures, { 'id'    => $id,
			    'ctrl'  => $out->{ctrl},
			    'name'  => $name };

	# Collecting some storage info
	$sysinfo{'enclosure'}{$nexus}{'id'}       = $nexus;
	$sysinfo{'enclosure'}{$nexus}{'name'}     = $name;
	$sysinfo{'enclosure'}{$nexus}{'firmware'} = $firmware;

        next ENCLOSURE if blacklisted('encl', $nexus);

	my $msg = sprintf 'Enclosure %s (%s) is %s',
	  $nexus, $name, $state;
        report('storage', $msg, $status2nagios{$status}, $nexus);
    }
    return;
}


#-----------------------------------------
# STORAGE: Check enclosure fans
#-----------------------------------------
sub check_enclosure_fans {
    return if $#controllers == -1;

    my $id        = undef;
    my $nexus     = undef;
    my $name      = undef;
    my $state     = undef;
    my $status    = undef;
    my $speed     = undef;
    my $encl_id   = undef;
    my $encl_name = undef;
    my @output    = ();

    if ($snmp) {
	my %fan_oid
	  = (
	     '1.3.6.1.4.1.674.10893.1.20.130.7.1.1'  => 'fanNumber',
	     '1.3.6.1.4.1.674.10893.1.20.130.7.1.2'  => 'fanName',
	     '1.3.6.1.4.1.674.10893.1.20.130.7.1.4'  => 'fanState',
	     '1.3.6.1.4.1.674.10893.1.20.130.7.1.11' => 'fanProbeCurrValue',
	     '1.3.6.1.4.1.674.10893.1.20.130.7.1.15' => 'fanComponentStatus',
	     '1.3.6.1.4.1.674.10893.1.20.130.7.1.16' => 'fanNexusID',
	     '1.3.6.1.4.1.674.10893.1.20.130.8.1.4'  => 'fanConnectionEnclosureName',
	     '1.3.6.1.4.1.674.10893.1.20.130.8.1.5'  => 'fanConnectionEnclosureNumber',
	    );

	my $result = $snmp_session->get_entries(-columns => [keys %fan_oid]);

	# No enclosure fans is OK
	return if !defined $result;

	@output = @{ get_snmp_output($result, \%fan_oid) };
    }
    else {
	foreach my $enc (@enclosures) {
	    push @output, @{ run_omreport("storage enclosure controller=$enc->{ctrl} enclosure=$enc->{id} info=fans") };
	    map_item('ctrl', $enc->{ctrl}, \@output);
	    map_item('encl_id', $enc->{id}, \@output);
	    map_item('encl_name', $enc->{name}, \@output);
	}
    }

    my %fan_state
      = (
	 0  => 'Unknown',
	 1  => 'Ready',
	 2  => 'Failed',
	 3  => 'Online',
	 4  => 'Offline',
	 6  => 'Degraded',
	 21 => 'Missing',
	);

    # Check fans on each of the enclosures
  FAN:
    foreach my $out (@output) {
	if ($snmp) {
	    $id        = $out->{fanNumber} - 1;
	    $name      = $out->{fanName};
	    $state     = $fan_state{$out->{fanState}};
	    $status    = $snmp_status{$out->{fanComponentStatus}};
	    $speed     = $out->{fanProbeCurrValue};
	    $encl_id   = $out->{fanConnectionEnclosureNumber} - 1;
	    $encl_name = $out->{fanConnectionEnclosureName};
	    $nexus     = convert_nexus($out->{fanNexusID});
	}
	else {
	    $id        = $out->{'ID'};
	    $name      = $out->{'Name'};
	    $state     = $out->{'State'};
	    $status    = $out->{'Status'};
	    $speed     = $out->{'Speed'};
	    $encl_id   = join q{:}, $out->{ctrl}, $out->{'encl_id'};
	    $encl_name = $out->{encl_name};
	    $nexus     = join q{:}, $out->{ctrl}, $out->{'encl_id'}, $id;
	}

	next FAN if blacklisted('encl_fan', $nexus);

	# Default
	if ($status ne 'Ok') {
	    my $msg = sprintf '%s in enclosure %s (%s) needs attention: %s',
	      $name, $encl_id, $encl_name, $state;
	    report('storage', $msg, $status2nagios{$status}, $nexus);
	}
	# Ok
	else {
	    my $msg = sprintf '%s in enclosure %s (%s) is %s (speed=%s)',
	      $name, $encl_id, $encl_name, $state, $speed;
	    report('storage', $msg, $E_OK, $nexus);
	}
    }
    return;
}


#-----------------------------------------
# STORAGE: Check enclosure power supplies
#-----------------------------------------
sub check_enclosure_pwr {
    return if $#controllers == -1;

    my $id        = undef;
    my $nexus     = undef;
    my $name      = undef;
    my $state     = undef;
    my $status    = undef;
    my $encl_id   = undef;
    my $encl_name = undef;
    my @output    = ();

    if ($snmp) {
	my %ps_oid
	  = (
	     '1.3.6.1.4.1.674.10893.1.20.130.9.1.1'  => 'powerSupplyNumber',
	     '1.3.6.1.4.1.674.10893.1.20.130.9.1.2'  => 'powerSupplyName',
	     '1.3.6.1.4.1.674.10893.1.20.130.9.1.4'  => 'powerSupplyState',
	     '1.3.6.1.4.1.674.10893.1.20.130.9.1.9'  => 'powerSupplyComponentStatus',
	     '1.3.6.1.4.1.674.10893.1.20.130.9.1.10' => 'powerSupplyNexusID',
	     '1.3.6.1.4.1.674.10893.1.20.130.10.1.4' => 'powerSupplyConnectionEnclosureName',
	     '1.3.6.1.4.1.674.10893.1.20.130.10.1.5' => 'powerSupplyConnectionEnclosureNumber',
	    );
	my $result = $snmp_session->get_entries(-columns => [keys %ps_oid]);

	# No enclosure power supplies is OK
	return if !defined $result;

	@output = @{ get_snmp_output($result, \%ps_oid) };
    }
    else {
	foreach my $enc (@enclosures) {
	    push @output, @{ run_omreport("storage enclosure controller=$enc->{ctrl} enclosure=$enc->{id} info=pwrsupplies") };
	    map_item('ctrl', $enc->{ctrl}, \@output);
	    map_item('encl_id', $enc->{id}, \@output);
	    map_item('encl_name', $enc->{name}, \@output);
	}
    }

    my %ps_state
      = (
	 0  => 'Unknown',
	 1  => 'Ready',
	 2  => 'Failed',
	 5  => 'Not Installed',
	 6  => 'Degraded',
	 11 => 'Removed',
	 21 => 'Missing',
	);

    # Check power supplies on each of the enclosures
  PS:
    foreach my $out (@output) {
	if ($snmp) {
	    $id        = $out->{powerSupplyNumber};
	    $name      = $out->{powerSupplyName};
	    $state     = $ps_state{$out->{powerSupplyState}};
	    $status    = $snmp_status{$out->{powerSupplyComponentStatus}};
	    $encl_id   = $out->{powerSupplyConnectionEnclosureNumber} - 1;
	    $encl_name = $out->{powerSupplyConnectionEnclosureName};
	    $nexus     = convert_nexus($out->{powerSupplyNexusID});
	}
	else {
	    $id        = $out->{'ID'};
	    $name      = $out->{'Name'};
	    $state     = $out->{'State'};
	    $status    = $out->{'Status'};
	    $encl_id   = join q{:}, $out->{ctrl}, $out->{'encl_id'};
	    $encl_name = $out->{encl_name};
	    $nexus     = join q{:}, $out->{ctrl}, $out->{'encl_id'}, $id;
	}

	next PS if blacklisted('encl_ps', $nexus);

	# Default
	if ($status ne 'Ok') {
	    my $msg = sprintf '%s in enclosure %s (%s) needs attention: %s',
	      $name, $encl_id, $encl_name, $state;
	    report('storage', $msg, $status2nagios{$status}, $nexus);
	}
	# Ok
	else {
	    my $msg = sprintf '%s in enclosure %s (%s) is %s',
	      $name, $encl_id, $encl_name, $state;
	    report('storage', $msg, $E_OK, $nexus);
	}
    }
    return;
}


#-----------------------------------------
# STORAGE: Check enclosure temperatures
#-----------------------------------------
sub check_enclosure_temp {
    return if $#controllers == -1;

    my $id        = undef;
    my $nexus     = undef;
    my $name      = undef;
    my $state     = undef;
    my $status    = undef;
    my $reading   = undef;
    my $unit      = undef;
    my $max_warn  = undef;
    my $max_crit  = undef;
    my $encl_id   = undef;
    my $encl_name = undef;
    my @output    = ();

    if ($snmp) {
	my %temp_oid
	  = (
	     '1.3.6.1.4.1.674.10893.1.20.130.11.1.1'  => 'temperatureProbeNumber',
	     '1.3.6.1.4.1.674.10893.1.20.130.11.1.2'  => 'temperatureProbeName',
	     '1.3.6.1.4.1.674.10893.1.20.130.11.1.4'  => 'temperatureProbeState',
	     '1.3.6.1.4.1.674.10893.1.20.130.11.1.6'  => 'temperatureProbeUnit',
	     '1.3.6.1.4.1.674.10893.1.20.130.11.1.9'  => 'temperatureProbeMaxWarning',
	     '1.3.6.1.4.1.674.10893.1.20.130.11.1.10' => 'temperatureProbeMaxCritical',
	     '1.3.6.1.4.1.674.10893.1.20.130.11.1.11' => 'temperatureProbeCurValue',
	     '1.3.6.1.4.1.674.10893.1.20.130.11.1.13' => 'temperatureProbeComponentStatus',
	     '1.3.6.1.4.1.674.10893.1.20.130.11.1.14' => 'temperatureProbeNexusID',
	     '1.3.6.1.4.1.674.10893.1.20.130.12.1.4'  => 'temperatureConnectionEnclosureName',
	     '1.3.6.1.4.1.674.10893.1.20.130.12.1.5'  => 'temperatureConnectionEnclosureNumber',
	    );
	my $result = $snmp_session->get_entries(-columns => [keys %temp_oid]);

	# No enclosure temperature probes is OK
	return if !defined $result;

	@output = @{ get_snmp_output($result, \%temp_oid) };
    }
    else {
	foreach my $enc (@enclosures) {
	    push @output, @{ run_omreport("storage enclosure controller=$enc->{ctrl} enclosure=$enc->{id} info=temps") };
	    map_item('ctrl', $enc->{ctrl}, \@output);
	    map_item('encl_id', $enc->{id}, \@output);
	    map_item('encl_name', $enc->{name}, \@output);
	}
    }

    my %temp_state
      = (
	 0  => 'Unknown',
	 1  => 'Ready',
	 2  => 'Failed',
	 4  => 'Offline',
	 6  => 'Degraded',
	 9  => 'Inactive',
	 21 => 'Missing',
	);

    # Check temperature probes on each of the enclosures
  TEMP:
    foreach my $out (@output) {
	if ($snmp) {
	    $id        = $out->{temperatureProbeNumber} - 1;
	    $name      = $out->{temperatureProbeName};
	    $state     = $temp_state{$out->{temperatureProbeState}};
	    $status    = $snmp_status{$out->{temperatureProbeComponentStatus}};
	    $unit      = $out->{temperatureProbeUnit};
	    $reading   = $out->{temperatureProbeCurValue};
	    $max_warn  = $out->{temperatureProbeMaxWarning};
	    $max_crit  = $out->{temperatureProbeMaxCritical};
	    $encl_id   = $out->{temperatureConnectionEnclosureNumber} - 1;
	    $encl_name = $out->{temperatureConnectionEnclosureName};
	    $nexus     = convert_nexus($out->{temperatureProbeNexusID});
	}
	else {
	    $id        = $out->{'ID'};
	    $name      = $out->{'Name'};
	    $state     = $out->{'State'};
	    $status    = $out->{'Status'};
	    $unit      = 'FIXME';
	    $reading   = $out->{'Reading'}; $reading =~ s{\s*C}{}xms;
	    $max_warn  = $out->{'Maximum Warning Threshold'}; $max_warn =~ s{\s*C}{}xms;
	    $max_crit  = $out->{'Maximum Failure Threshold'}; $max_crit =~ s{\s*C}{}xms;
	    $encl_id   = join q{:}, $out->{ctrl}, $out->{'encl_id'};
	    $encl_name = $out->{encl_name};
	    $nexus     = join q{:}, $out->{ctrl}, $out->{'encl_id'}, $id;
	}

	next TEMP if blacklisted('encl_temp', $nexus);

	# Default
	if ($status ne 'Ok') {
	    my $msg = sprintf '%s in enclosure %s (%s) is %s at %s (%s max)',
	      $name, $encl_id, $encl_name, $state, $reading, $max_crit;
	    report('storage', $msg, $status2nagios{$status}, $nexus);
	}
	# Ok
	else {
	    my $msg = sprintf '%s in enclosure %s (%s): %s (%s max)',
	      $name, $encl_id, $encl_name, $reading, $max_crit;
	    report('storage', $msg, $E_OK, $nexus);
	}

	# Collect performance data
	if (defined $opt{'perfdata'}) {
	    $name =~ s{\A Temperature\sProbe\s(\d+) \z}{temp_$1}gxms;
	    my $pkey = "enclosure_${encl_id}_${name}";
	    my $pval = join q{;}, "${reading}C", $max_warn, $max_crit;
	    $perfdata{$pkey} = $pval;
	}
    }
    return;
}


#-----------------------------------------
# STORAGE: Check enclosure management modules (EMM)
#-----------------------------------------
sub check_enclosure_emms {
    return if $#controllers == -1;

    my $id        = undef;
    my $nexus     = undef;
    my $name      = undef;
    my $state     = undef;
    my $status    = undef;
    my $encl_id   = undef;
    my $encl_name = undef;
    my @output    = ();

    if ($snmp) {
	my %emms_oid
	  = (
	     '1.3.6.1.4.1.674.10893.1.20.130.13.1.1'  => 'enclosureManagementModuleNumber',
	     '1.3.6.1.4.1.674.10893.1.20.130.13.1.2'  => 'enclosureManagementModuleName',
	     '1.3.6.1.4.1.674.10893.1.20.130.13.1.4'  => 'enclosureManagementModuleState',
	     '1.3.6.1.4.1.674.10893.1.20.130.13.1.11' => 'enclosureManagementModuleComponentStatus',
	     '1.3.6.1.4.1.674.10893.1.20.130.13.1.12' => 'enclosureManagementModuleNexusID',
	     '1.3.6.1.4.1.674.10893.1.20.130.14.1.4'  => 'enclosureManagementModuleConnectionEnclosureName',
	     '1.3.6.1.4.1.674.10893.1.20.130.14.1.5'  => 'enclosureManagementModuleConnectionEnclosureNumber',
	    );
	my $result = $snmp_session->get_entries(-columns => [keys %emms_oid]);

	# No enclosure EMMs is OK
	return if !defined $result;

	@output = @{ get_snmp_output($result, \%emms_oid) };
    }
    else {
	foreach my $enc (@enclosures) {
	    push @output, @{ run_omreport("storage enclosure controller=$enc->{ctrl} enclosure=$enc->{id} info=emms") };
	    map_item('ctrl', $enc->{ctrl}, \@output);
	    map_item('encl_id', $enc->{id}, \@output);
	    map_item('encl_name', $enc->{name}, \@output);
	}
    }

    my %emms_state
      = (
	 0  => 'Unknown',
	 1  => 'Ready',
	 2  => 'Failed',
	 3  => 'Online',
	 4  => 'Offline',
	 5  => 'Not Installed',
	 6  => 'Degraded',
	 21 => 'Missing',
	);

    # Check temperature probes on each of the enclosures
  EMM:
    foreach my $out (@output) {
	if ($snmp) {
	    $id        = $out->{enclosureManagementModuleNumber} - 1;
	    $name      = $out->{enclosureManagementModuleName};
	    $state     = $emms_state{$out->{enclosureManagementModuleState}};
	    $status    = $snmp_status{$out->{enclosureManagementModuleComponentStatus}};
	    $encl_id   = $out->{enclosureManagementModuleConnectionEnclosureNumber} - 1;
	    $encl_name = $out->{enclosureManagementModuleConnectionEnclosureName};
	    $nexus     = convert_nexus($out->{enclosureManagementModuleNexusID});
	}
	else {
	    $id        = $out->{'ID'};
	    $name      = $out->{'Name'};
	    $state     = $out->{'State'};
	    $status    = $out->{'Status'};
	    $encl_id   = join q{:}, $out->{ctrl}, $out->{'encl_id'};
	    $encl_name = $out->{encl_name};
	    $nexus     = join q{:}, $out->{ctrl}, $out->{'encl_id'}, $id;
	}

	next EMM if blacklisted('encl_emm', $nexus);

	# Default
	if ($status ne 'Ok') {
	    my $msg = sprintf '%s in enclosure %s (%s) needs attention: %s',
	      $name, $encl_id, $encl_name, $state;
	    report('storage', $msg, $status2nagios{$status}, $nexus);
	}
	# Ok
	else {
	    my $msg = sprintf '%s in enclosure %s (%s) is %s',
	      $name, $encl_id, $encl_name, $state;
	    report('storage', $msg, $E_OK, $nexus);
	}
    }
    return;
}


#-----------------------------------------
# CHASSIS: Check memory modules
#-----------------------------------------
sub check_memory {
    my $index    = undef;
    my $status   = undef;
    my $location = undef;
    my $size     = undef;
    my $modes    = undef;
    my @failures = ();
    my @output   = ();

    if ($snmp) {
	my %dimm_oid
	  = (
	     '1.3.6.1.4.1.674.10892.1.1100.50.1.2.1'  => 'memoryDeviceIndex',
	     '1.3.6.1.4.1.674.10892.1.1100.50.1.5.1'  => 'memoryDeviceStatus',
	     '1.3.6.1.4.1.674.10892.1.1100.50.1.8.1'  => 'memoryDeviceLocationName',
	     '1.3.6.1.4.1.674.10892.1.1100.50.1.14.1' => 'memoryDeviceSize',
	     '1.3.6.1.4.1.674.10892.1.1100.50.1.20.1' => 'memoryDeviceFailureModes',
	    );
	my $result = $snmp_session->get_entries(-columns => [keys %dimm_oid]);

	if (!defined $result) {
	    printf "SNMP [memory]: %s.\n", $snmp_session->error;
	    $snmp_session->close;
	    exit $E_UNKNOWN;
	}

	@output = @{ get_snmp_output($result, \%dimm_oid) };
    }
    else {
	@output = @{ run_omreport("$omopt_chassis memory") };
    }

    # Note: These values are bit masks, so combination values are
    # possible. If value is 0 (zero), memory device has no faults.
    my %failure_mode
      = (
	 1  => 'ECC single bit correction warning rate exceeded',
	 2  => 'ECC single bit correction failure rate exceeded',
	 4  => 'ECC multibit fault encountered',
	 8  => 'ECC single bit correction logging disabled',
	 16 => 'device disabled because of spare activation',
	);

    my $count_dimms = 0;

  DIMM:
    foreach my $out (@output) {
	@failures = ();  # Initialize
	if ($snmp) {
	    $index    = $out->{memoryDeviceIndex};
	    $status   = $snmp_status{$out->{memoryDeviceStatus}};
	    $location = $out->{memoryDeviceLocationName};
	    $size     = sprintf '%d MB', $out->{memoryDeviceSize}/1024;
	    $modes    = $out->{memoryDeviceFailureModes};
	    if ($modes > 0) {
		foreach my $mask (sort keys %failure_mode) {
		    if (($modes & $mask) != 0) { push @failures, $failure_mode{$mask}; }
		}
	    }
	}
	else {
	    $index    = $out->{'Type'} eq '[Not Occupied]' ? undef : $out->{'Index'};
	    $status   = $out->{'Status'};
	    $location = $out->{'Connector Name'};
	    $size     = $out->{'Size'};
	    if (defined $size) {
		$size =~ s{\s\s}{ }gxms;
	    }
	    # Run 'omreport chassis memory index=X' to get the failures
	    if ($status ne 'Ok' && defined $index) {
		foreach (@{ run_command("$omreport $omopt_chassis memory index=$index -fmt ssv") }) {
		    if (m/\A Failures; (.+?) \z/xms) {
			chop(my $fail = $1);
			push @failures, split m{\.}xms, $fail;
		    }
		}
	    }
	}
	$location =~ s{\A \s*(.*?)\s* \z}{$1}xms;

	next DIMM if blacklisted('dimm', $index);

	# Ignore empty memory slots
	next DIMM if !defined $index;
	$count_dimms++;

	if ($status ne 'Ok') {
	    my $msg = undef;
	    if (scalar @failures == 0) {
		$msg = sprintf 'Memory module %d (%s, %s) needs attention (%s)',
		  $index, $location, $size, $status;
	    }
	    else {
		$msg = sprintf 'Memory module %d (%s, %s) needs attention: %s',
		  $index, $location, $size, (join q{, }, @failures);
	    }

	    report('chassis', $msg, $status2nagios{$status}, $index);
	}
	# Ok
	else {
	    my $msg = sprintf 'Memory module %d (%s, %s) is %s',
	      $index, $location, $size, $status;
	    report('chassis', $msg, $E_OK, $index);
	}
    }
    return $count_dimms;
}


#-----------------------------------------
# CHASSIS: Check fans
#-----------------------------------------
sub check_fans {
    my $index    = undef;
    my $status   = undef;
    my $reading  = undef;
    my $location = undef;
    my $max_crit = undef;
    my $max_warn = undef;
    my @output   = ();

    if ($snmp) {
	my %cool_oid
	  = (
	     '1.3.6.1.4.1.674.10892.1.700.12.1.2.1'  => 'coolingDeviceIndex',
	     '1.3.6.1.4.1.674.10892.1.700.12.1.5.1'  => 'coolingDeviceStatus',
	     '1.3.6.1.4.1.674.10892.1.700.12.1.6.1'  => 'coolingDeviceReading',
	     '1.3.6.1.4.1.674.10892.1.700.12.1.8.1'  => 'coolingDeviceLocationName',
	     '1.3.6.1.4.1.674.10892.1.700.12.1.10.1' => 'coolingDeviceUpperCriticalThreshold',
	     '1.3.6.1.4.1.674.10892.1.700.12.1.11.1' => 'coolingDeviceUpperNonCriticalThreshold',
	    );
	my $result = $snmp_session->get_entries(-columns => [keys %cool_oid]);

	if ($blade && !defined $result) {
	    return 0;
	}
	elsif (!$blade && !defined $result) {
	    printf "SNMP [cooling]: %s.\n", $snmp_session->error;
	    $snmp_session->close;
	    exit $E_UNKNOWN;
	}

	@output = @{ get_snmp_output($result, \%cool_oid) };
    }
    else {
	@output = @{ run_omreport("$omopt_chassis fans") };
    }

    my $count_fans = 0;

  FAN:
    foreach my $out (@output) {
	if ($snmp) {
	    $index    = $out->{coolingDeviceIndex};
	    $status   = $snmp_probestatus{$out->{coolingDeviceStatus}};
	    $reading  = $out->{coolingDeviceReading};
	    $location = $out->{coolingDeviceLocationName};
	    $max_crit = exists $out->{coolingDeviceUpperCriticalThreshold}
	      ? $out->{coolingDeviceUpperCriticalThreshold} : 0;
	    $max_warn = exists $out->{coolingDeviceUpperNonCriticalThreshold}
	      ? $out->{coolingDeviceUpperNonCriticalThreshold} : 0;
	}
	else {
	    $index    = $out->{'Index'};
	    $status   = $out->{'Status'};
	    $reading  = $out->{'Reading'};
	    $location = $out->{'Probe Name'};
	    $max_crit = $out->{'Maximum Failure Threshold'} ne '[N/A]'
	      ? $out->{'Maximum Failure Threshold'} : 0;
	    $max_warn = $out->{'Maximum Warning Threshold'} ne '[N/A]'
	      ? $out->{'Maximum Warning Threshold'} : 0;
	    $reading  =~ s{\A (\d+).* \z}{$1}xms;
	    $max_warn =~ s{\A (\d+).* \z}{$1}xms;
	    $max_crit =~ s{\A (\d+).* \z}{$1}xms;
	}

	next FAN if blacklisted('fan', $index);
	$count_fans++;

	if ($status ne 'Ok') {
	    my $msg = sprintf 'Chassis fan %d (%s) needs attention: %s',
	      $index, $location, $status;
	    my $err = $snmp ? $probestatus2nagios{$status} : $status2nagios{$status};
	    report('chassis', $msg, $err, $index);
	}
	else {
	    my $msg = sprintf 'Chassis fan %d (%s): %s',
	      $index, $location, $reading;
	    report('chassis', $msg, $E_OK, $index);
	}

	# Collect performance data
	if (defined $opt{'perfdata'}) {
	    my $pname = lc $location;
	    $pname =~ s{\s}{_}gxms;
	    $pname =~ s{proc_}{cpu#}xms;
	    my $pkey = join q{_}, 'fan', $index, $pname;
	    my $pval = join q{;}, "${reading}RPM", $max_warn, $max_crit;
	    $perfdata{$pkey} = $pval;
	}
    }
    return $count_fans;
}


#-----------------------------------------
# CHASSIS: Check power supplies
#-----------------------------------------
sub check_powersupplies {
    my $index    = undef;
    my $status   = undef;
    my $type     = undef;
    my $err_type = undef;
    my $state    = undef;
    my @states   = ();
    my @output   = ();

    if ($snmp) {
	my %ps_oid
	  = (
	     '1.3.6.1.4.1.674.10892.1.600.12.1.2.1'  => 'powerSupplyIndex',
	     '1.3.6.1.4.1.674.10892.1.600.12.1.5.1'  => 'powerSupplyStatus',
	     '1.3.6.1.4.1.674.10892.1.600.12.1.7.1'  => 'powerSupplyType',
	     '1.3.6.1.4.1.674.10892.1.600.12.1.11.1' => 'powerSupplySensorState',
	     '1.3.6.1.4.1.674.10892.1.600.12.1.12.1' => 'powerSupplyConfigurationErrorType',
	    );
	my $result = $snmp_session->get_entries(-columns => [keys %ps_oid]);

	# No instrumented PSU is OK (blades, low-end servers)
	return 0 if !defined $result;

	@output = @{ get_snmp_output($result, \%ps_oid) };
    }
    else {
	@output = @{ run_omreport("$omopt_chassis pwrsupplies") };
    }

    my %ps_type
      = (
	 1  => 'Other',
	 2  => 'Unknown',
	 3  => 'Linear',
	 4  => 'Switching',
	 5  => 'Battery',
	 6  => 'Uninterruptible Power Supply',
	 7  => 'Converter',
	 8  => 'Regulator',
	 9  => 'AC',
	 10 => 'DC',
	 11 => 'VRM',
	);

    my %ps_state
      = (
	 1  => 'Presence detected',
	 2  => 'Failure detected',
	 4  => 'Predictive Failure',
	 8  => 'AC lost',
	 16 => 'AC lost or out-of-range',
	 32 => 'AC out-of-range but present',
	 64 => 'Configuration error',
	);

    my %ps_config_error_type
      = (
	 1 => 'Vendor mismatch',
	 2 => 'Revision mismatch',
	 3 => 'Processor missing',
	);

    my $count_psus = 0;

  PS:
    foreach my $out (@output) {
	if ($snmp) {
	    @states = ();  # contains states for the PS

	    $index    = $out->{powerSupplyIndex} - 1;
	    $status   = $snmp_status{$out->{powerSupplyStatus}};
	    $type     = $ps_type{$out->{powerSupplyType}};
	    $err_type = defined $out->{powerSupplyConfigurationErrorType}
	      ? $ps_config_error_type{$out->{powerSupplyConfigurationErrorType}} : undef;

	    # get the combined state from the StatusReading OID
	    foreach my $mask (sort keys %ps_state) {
		if (($out->{powerSupplySensorState} & $mask) != 0) {
		    push @states, $ps_state{$mask};
		}
	    }

	    # If configuration error, also include the error type
	    if (defined $err_type) {
		push @states, $err_type;
	    }

	    # Finally, construct the state string
	    $state = join q{, }, @states;
	}
	else {
	    $index  = $out->{'Index'};
	    $status = $out->{'Status'};
	    $type   = $out->{'Type'};
	    $state  = $out->{'Online Status'};
	}

	next PS if blacklisted('ps', $index);
	$count_psus++;

	if ($status ne 'Ok') {
	    my $msg = sprintf 'Power Supply %d (%s) needs attention: %s',
	      $index, $type, $state;
	    report('chassis', $msg, $status2nagios{$status}, $index);
	}
	else {
	    my $msg = sprintf 'Power Supply %d (%s): %s',
	      $index, $type, $state;
	    report('chassis', $msg, $E_OK, $index);
	}
    }
    return $count_psus;
}


#-----------------------------------------
# CHASSIS: Check temperatures
#-----------------------------------------
sub check_temperatures {
    my $index    = undef;
    my $status   = undef;
    my $reading  = undef;
    my $location = undef;
    my $max_crit = undef;
    my $max_warn = undef;
    my $min_warn = undef;
    my $min_crit = undef;
    my $type     = undef;
    my $discrete = undef;
    my @output = ();

    # Getting custom temperature thresholds (user option)
    my %warn_threshold = %{ custom_temperature_thresholds('w') };
    my %crit_threshold = %{ custom_temperature_thresholds('c') };

    if ($snmp) {
	my %temp_oid
	  = (
	     '1.3.6.1.4.1.674.10892.1.700.20.1.2.1'  => 'temperatureProbeIndex',
	     '1.3.6.1.4.1.674.10892.1.700.20.1.5.1'  => 'temperatureProbeStatus',
	     '1.3.6.1.4.1.674.10892.1.700.20.1.6.1'  => 'temperatureProbeReading',
	     '1.3.6.1.4.1.674.10892.1.700.20.1.7.1'  => 'temperatureProbeType',
	     '1.3.6.1.4.1.674.10892.1.700.20.1.8.1'  => 'temperatureProbeLocationName',
	     '1.3.6.1.4.1.674.10892.1.700.20.1.10.1' => 'temperatureProbeUpperCriticalThreshold',
	     '1.3.6.1.4.1.674.10892.1.700.20.1.11.1' => 'temperatureProbeUpperNonCriticalThreshold',
	     '1.3.6.1.4.1.674.10892.1.700.20.1.12.1' => 'temperatureProbeLowerNonCriticalThreshold',
	     '1.3.6.1.4.1.674.10892.1.700.20.1.13.1' => 'temperatureProbeLowerCriticalThreshold',
	     '1.3.6.1.4.1.674.10892.1.700.20.1.16.1' => 'temperatureProbeDiscreteReading',
	    );
	# this didn't work well for some reason
	#my $result = $snmp_session->get_entries(-columns => [keys %temp_oid]);

	# Getting values using the table
	my $temperatureProbeTable = '1.3.6.1.4.1.674.10892.1.700.20';
	my $result = $snmp_session->get_table(-baseoid => $temperatureProbeTable);

	if (!defined $result) {
	    printf "SNMP [temperatures]: %s.\n", $snmp_session->error;
	    $snmp_session->close;
	    exit $E_UNKNOWN;
	}

	@output = @{ get_snmp_output($result, \%temp_oid) };
    }
    else {
	@output = @{ run_omreport("$omopt_chassis temps") };
    }

    my %probe_type
      = (
	 1  => 'Other',      # type is other than following values
	 2  => 'Unknown',    # type is unknown
	 3  => 'AmbientESM', # type is Ambient Embedded Systems Management temperature probe
	 16 => 'Discrete',   # type is temperature probe with discrete reading
	);

    my $count_temps = 0;

  TEMP:
    foreach my $out (@output) {
	if ($snmp) {
	    $index    = $out->{temperatureProbeIndex} - 1;
	    $status   = $snmp_probestatus{$out->{temperatureProbeStatus}};
	    $reading  = $out->{temperatureProbeReading} / 10;
	    $location = $out->{temperatureProbeLocationName};
	    $max_crit = $out->{temperatureProbeUpperCriticalThreshold} / 10;
	    $max_warn = $out->{temperatureProbeUpperNonCriticalThreshold} / 10;
	    $min_crit = exists $out->{temperatureProbeLowerCriticalThreshold}
	      ? $out->{temperatureProbeLowerCriticalThreshold} / 10 : '[N/A]';
	    $min_warn = exists $out->{temperatureProbeLowerNonCriticalThreshold}
	      ? $out->{temperatureProbeLowerNonCriticalThreshold} / 10 : '[N/A]';
	    $type     = $probe_type{$out->{temperatureProbeType}};
	    $discrete = exists $out->{temperatureProbeDiscreteReading}
	      ? $out->{temperatureProbeDiscreteReading} : undef;
	}
	else {
	    $index    = $out->{'Index'};
	    $status   = $out->{'Status'};
	    $reading  = $out->{'Reading'}; $reading =~ s{\.0\s+C}{}xms;
	    $location = $out->{'Probe Name'};
	    $max_crit = $out->{'Maximum Failure Threshold'}; $max_crit =~ s{\.0\s+C}{}xms;
	    $max_warn = $out->{'Maximum Warning Threshold'}; $max_warn =~ s{\.0\s+C}{}xms;
	    $min_crit = $out->{'Minimum Failure Threshold'}; $min_crit =~ s{\.0\s+C}{}xms;
	    $min_warn = $out->{'Minimum Warning Threshold'}; $min_warn =~ s{\.0\s+C}{}xms;
	    $type     = $reading =~ m{\A\d+\z}xms ? 'AmbientESM' : 'Discrete';
	    $discrete = $reading;
	}

	next TEMP if blacklisted('temp', $index);
	$count_temps++;

	if ($type eq 'Discrete') {
	    my $msg = sprintf 'Temperature probe %d (%s): is %s',
	      $index, $location, $discrete;
	    my $err = $snmp ? $probestatus2nagios{$status} : $status2nagios{$status};
	    report('chassis', $msg, $err, $index);
	}
	else {
	    # First check according to custom thresholds
	    if (exists $crit_threshold{$index}{max} and $reading > $crit_threshold{$index}{max}) {
		# Custom critical MAX
		my $msg = sprintf 'Temperature Probe %d (%s) reads %d C (custom max=%d)',
		  $index, $location, $reading, $crit_threshold{$index}{max};
		report('chassis', $msg, $E_CRITICAL, $index);
	    }
	    elsif (exists $warn_threshold{$index}{max} and $reading > $warn_threshold{$index}{max}) {
		# Custom warning MAX
		my $msg = sprintf 'Temperature Probe %d (%s) reads %d C (custom max=%d)',
		  $index, $location, $reading, $warn_threshold{$index}{max};
		report('chassis', $msg, $E_WARNING, $index);
	    }
	    elsif (exists $crit_threshold{$index}{min} and $reading < $crit_threshold{$index}{min}) {
		# Custom critical MIN
		my $msg = sprintf 'Temperature Probe %d (%s) reads %d C (custom min=%d)',
		  $index, $location, $reading, $crit_threshold{$index}{min};
		report('chassis', $msg, $E_CRITICAL, $index);
	    }
	    elsif (exists $warn_threshold{$index}{min} and $reading < $warn_threshold{$index}{min}) {
		# Custom warning MIN
		my $msg = sprintf 'Temperature Probe %d (%s) reads %d C (custom min=%d)',
		  $index, $location, $reading, $warn_threshold{$index}{min};
		report('chassis', $msg, $E_WARNING, $index);
	    }
	    elsif ($status ne 'Ok' and $max_crit ne '[N/A]' and $reading > $max_crit) {
		my $msg = sprintf 'Temperature Probe %d (%s) is critically high at %d C',
		  $index, $location, $reading;
		my $err = $snmp ? $probestatus2nagios{$status} : $status2nagios{$status};
		report('chassis', $msg, $err, $index);
	    }
	    elsif ($status ne 'Ok' and $max_warn ne '[N/A]' and $reading > $max_warn) {
		my $msg = sprintf 'Temperature Probe %d (%s) is too high at %d C',
		  $index, $location, $reading;
		my $err = $snmp ? $probestatus2nagios{$status} : $status2nagios{$status};
		report('chassis', $msg, $err, $index);
	    }
	    elsif ($status ne 'Ok' and $min_crit ne '[N/A]' and $reading < $min_crit) {
		my $msg = sprintf 'Temperature Probe %d (%s) is critically low at %d C',
		  $index, $location, $reading;
		my $err = $snmp ? $probestatus2nagios{$status} : $status2nagios{$status};
		report('chassis', $msg, $err, $index);
	    }
	    elsif ($status ne 'Ok' and $min_warn ne '[N/A]' and $reading < $min_warn) {
		my $msg = sprintf 'Temperature Probe %d (%s) is too low at %d C',
		  $index, $location, $reading;
		my $err = $snmp ? $probestatus2nagios{$status} : $status2nagios{$status};
		report('chassis', $msg, $err, $index);
	    }
	    # Ok
	    else {
		my $msg = sprintf 'Temperature Probe %d (%s) reads %d C (min=%s/%s, max=%s/%s)',
		  $index, $location, $reading, $min_warn, $min_crit, $max_warn, $max_crit;
		my $err = $snmp ? $probestatus2nagios{$status} : $status2nagios{$status};
		report('chassis', $msg, $err, $index);
	    }

	    # Collect performance data
	    if (defined $opt{'perfdata'}) {
		my $pname = lc $location;
		$pname =~ s{\s}{_}gxms;
		$pname =~ s{_temp\z}{}xms;
		$pname =~ s{proc_}{cpu#}xms;
		my $pkey = join q{_}, 'temp', $index, $pname;
		my $pval = join q{;}, "${reading}C", $max_warn, $max_crit;
		$perfdata{$pkey} = $pval;
	    }
	}
    }
    return $count_temps;
}


#-----------------------------------------
# CHASSIS: Check processors
#-----------------------------------------
sub check_processors {
    my $index   = undef;
    my $status  = undef;
    my $state   = undef;
    my $oid_ver = 'new';
    my @output  = ();

    if ($snmp) {

	# NOTE: For some reason, older models don't have the
	# "Processor Device Status" OIDs. We first check the newer
	# (preferred) OIDs, and if that doesn't work, check the "old"
	# OIDs.

	my %cpu_oid_new  # for newer models
	  = (
	     '1.3.6.1.4.1.674.10892.1.1100.32.1.2.1' => 'processorDeviceStatusIndex',
	     '1.3.6.1.4.1.674.10892.1.1100.32.1.5.1' => 'processorDeviceStatusStatus',
	     '1.3.6.1.4.1.674.10892.1.1100.32.1.6.1' => 'processorDeviceStatusReading',
	    );

	my %cpu_oid_old  # for older models
	  = (
             '1.3.6.1.4.1.674.10892.1.1100.30.1.2.1' => 'processorDeviceIndex',
             '1.3.6.1.4.1.674.10892.1.1100.30.1.5.1' => 'processorDeviceStatus',
             '1.3.6.1.4.1.674.10892.1.1100.30.1.9.1' => 'processorDeviceStatusState',
	    );

	my $result = $snmp_session->get_entries(-columns => [keys %cpu_oid_new]);

	if (!defined $result) {
	    $oid_ver = 'old';
	    $result = $snmp_session->get_entries(-columns => [keys %cpu_oid_old]);
	}

	if (!defined $result) {
	    printf "SNMP [processors]: %s.\n", $snmp_session->error;
	    $snmp_session->close;
	    exit $E_UNKNOWN;
	}

	if ($oid_ver eq 'new') {
	    @output = @{ get_snmp_output($result, \%cpu_oid_new) };
	}
	else {
	    @output = @{ get_snmp_output($result, \%cpu_oid_old) };
	}
    }
    else {
	@output = @{ run_omreport("$omopt_chassis processors") };
    }

    my %cpu_state
      = (
         1 => 'Other',         # other than following values
         2 => 'Unknown',       # unknown
         3 => 'Enabled',       # enabled
         4 => 'User Disabled', # disabled by user via BIOS setup
         5 => 'BIOS Disabled', # disabled by BIOS (POST error)
         6 => 'Idle',          # idle
        );

    my %cpu_reading
      = (
	 1    => 'Internal Error',      # Internal Error
	 2    => 'Thermal Trip',        # Thermal Trip
	 32   => 'Configuration Error', # Configuration Error
	 128  => 'Present',             # Processor Present
	 256  => 'Disabled',            # Processor Disabled
	 512  => 'Terminator Present',  # Terminator Present
	 1024 => 'Throttled',           # Processor Throttled
	);


    my $count_cpus = 0;

  CPU:
    foreach my $out (@output) {
	if ($snmp) {
	    if ($oid_ver eq 'new') {
		my @states  = ();  # contains states for the CPU
		$index  = $out->{processorDeviceStatusIndex} - 1;
		$status = $snmp_status{$out->{processorDeviceStatusStatus}};

		# get the combined state from the StatusReading OID
		foreach my $mask (sort keys %cpu_reading) {
		    if (($out->{processorDeviceStatusReading} & $mask) != 0) {
			push @states, $cpu_reading{$mask};
		    }
		}

		# Finally, create the state string
		$state = join q{, }, @states;
	    }
	    else {
		$index  = $out->{processorDeviceIndex} - 1;
		$status = $snmp_status{$out->{processorDeviceStatus}};
		$state  = $cpu_state{$out->{processorDeviceStatusState}};
	    }
	}
	else {
	    $index  = $out->{'Index'};
	    $status = $out->{'Status'};
	    $state  = $out->{'State'};
	}

	next CPU if blacklisted('cpu', $index);

	# Ignore unoccupied CPU slots (omreport)
	next CPU if (defined $out->{'Processor Manufacturer'}
		     and $out->{'Processor Manufacturer'} eq '[Not Occupied]')
	  or (defined $out->{'Processor Brand'} and $out->{'Processor Brand'} eq '[Not Occupied]');

	# Ignore unoccupied CPU slots (snmp)
	if ($snmp and exists $out->{processorDeviceStatusReading}
	    and $out->{processorDeviceStatusReading} == 0) {
	    next CPU;
	}

	$count_cpus++;

	# Default
	if ($status ne 'Ok') {
	    my $msg = sprintf 'CPU %d needs attention: %s',
	      $index, $state;
	    report('chassis', $msg, $status2nagios{$status}, $index);
	}
	# Ok
	else {
	    my $msg = sprintf 'CPU %d is %s',
	      $index, $state;
	    report('chassis', $msg, $E_OK, $index);
	}
    }
    return $count_cpus;
}


#-----------------------------------------
# CHASSIS: Check voltage probes
#-----------------------------------------
sub check_volts {
    my $index    = undef;
    my $status   = undef;
    my $reading  = undef;
    my $location = undef;
    my @output = ();

    if ($snmp) {
	my %volt_oid
	  = (
	     '1.3.6.1.4.1.674.10892.1.600.20.1.2.1'  => 'voltageProbeIndex',
	     '1.3.6.1.4.1.674.10892.1.600.20.1.5.1'  => 'voltageProbeStatus',
	     '1.3.6.1.4.1.674.10892.1.600.20.1.6.1'  => 'voltageProbeReading',
	     '1.3.6.1.4.1.674.10892.1.600.20.1.8.1'  => 'voltageProbeLocationName',
	     '1.3.6.1.4.1.674.10892.1.600.20.1.16.1' => 'voltageProbeDiscreteReading',
	    );

	my $voltageProbeTable = '1.3.6.1.4.1.674.10892.1.600.20.1';
        my $result = $snmp_session->get_table(-baseoid => $voltageProbeTable);

	if (!defined $result) {
	    printf "SNMP [voltage probes]: %s.\n", $snmp_session->error;
	    $snmp_session->close;
	    exit $E_UNKNOWN;
	}

	@output = @{ get_snmp_output($result, \%volt_oid) };
    }
    else {
	@output = @{ run_omreport("$omopt_chassis volts") };
    }

    my %volt_discrete_reading
      = (
	 1 => 'Good',
	 2 => 'Bad',
	);

    my $count_volts = 0;

  VOLT:
    foreach my $out (@output) {
	if ($snmp) {
	    $index    = $out->{voltageProbeIndex} - 1;
	    $status   = $snmp_status{$out->{voltageProbeStatus}};
	    $reading  = exists $out->{voltageProbeReading}
	      ? sprintf('%.3f V', $out->{voltageProbeReading}/1000)
	      : $volt_discrete_reading{$out->{voltageProbeDiscreteReading}};
	    $location = $out->{voltageProbeLocationName};
	}
	else {
	    $index    = $out->{'Index'};
	    $status   = $out->{'Status'};
	    $reading  = $out->{'Reading'};
	    $location = $out->{'Probe Name'};
	}

	next VOLT if blacklisted('volt', $index);
	$count_volts++;

	my $msg = sprintf 'Voltage sensor %d (%s) is %s',
	  $index, $location, $reading;
	my $err = $snmp ? $probestatus2nagios{$status} : $status2nagios{$status};
	report('chassis', $msg, $err, $index);
    }
    return $count_volts;
}


#-----------------------------------------
# CHASSIS: Check batteries
#-----------------------------------------
sub check_batteries {
    my $index    = undef;
    my $status   = undef;
    my $reading  = undef;
    my $location = undef;
    my @output = ();

    if ($snmp) {
	my %bat_oid
	  = (
	     '1.3.6.1.4.1.674.10892.1.600.50.1.2.1' => 'batteryIndex',
	     '1.3.6.1.4.1.674.10892.1.600.50.1.5.1' => 'batteryStatus',
	     '1.3.6.1.4.1.674.10892.1.600.50.1.6.1' => 'batteryReading',
	     '1.3.6.1.4.1.674.10892.1.600.50.1.7.1' => 'batteryLocationName',
	    );
	my $result = $snmp_session->get_entries(-columns => [keys %bat_oid]);

	# No batteries is OK
	return 0 if !defined $result;

	@output = @{ get_snmp_output($result, \%bat_oid) };
    }
    else {
	@output = @{ run_omreport("$omopt_chassis batteries") };
    }

    my %bat_reading
      = (
	 1 => 'Predictive Failure',
	 2 => 'Failed',
	 4 => 'Presence Detected',
	);

    my $count_bats = 0;

  BATTERY:
    foreach my $out (@output) {
	if ($snmp) {
	    $index    = $out->{batteryIndex} - 1;
	    $status   = $snmp_status{$out->{batteryStatus}};
	    $reading  = $bat_reading{$out->{batteryReading}};
	    $location = $out->{batteryLocationName};
	}
	else {
	    $index    = $out->{'Index'};
	    $status   = $out->{'Status'};
	    $reading  = $out->{'Reading'};
	    $location = $out->{'Probe Name'};
	}

	next BATTERY if blacklisted('bp', $index);
	$count_bats++;

	my $msg = sprintf 'Battery probe %d (%s) is %s',
	  $index, $location, $reading;
	report('chassis', $msg, $status2nagios{$status}, $index);
    }
    return $count_bats;
}


#-----------------------------------------
# CHASSIS: Check amperage probes (power monitoring)
#-----------------------------------------
sub check_pwrmonitoring {
    my $index    = undef;
    my $status   = undef;
    my $reading  = undef;
    my $location = undef;
    my $max_crit = undef;
    my $max_warn = undef;
    my $unit     = undef;
    my @output = ();

    if ($snmp) {
	my %amp_oid
	  = (
	     '1.3.6.1.4.1.674.10892.1.600.30.1.2.1'  => 'amperageProbeIndex',
	     '1.3.6.1.4.1.674.10892.1.600.30.1.5.1'  => 'amperageProbeStatus',
	     '1.3.6.1.4.1.674.10892.1.600.30.1.6.1'  => 'amperageProbeReading',
	     '1.3.6.1.4.1.674.10892.1.600.30.1.7.1'  => 'amperageProbeType',
	     '1.3.6.1.4.1.674.10892.1.600.30.1.8.1'  => 'amperageProbeLocationName',
	     '1.3.6.1.4.1.674.10892.1.600.30.1.10.1' => 'amperageProbeUpperCriticalThreshold',
	     '1.3.6.1.4.1.674.10892.1.600.30.1.11.1' => 'amperageProbeUpperNonCriticalThreshold',
	     '1.3.6.1.4.1.674.10892.1.600.30.1.16.1' => 'amperageProbeDiscreteReading',
	    );
	my $result = $snmp_session->get_entries(-columns => [keys %amp_oid]);

	# No pwrmonitoring is OK
	return 0 if !defined $result;

	@output = @{ get_snmp_output($result, \%amp_oid) };
    }
    else {
	@output = @{ run_omreport("$omopt_chassis pwrmonitoring") };
    }

    my %amp_type   # Amperage probe types
      = (
	 1  => 'amperageProbeTypeIsOther',            # other than following values
	 2  => 'amperageProbeTypeIsUnknown',          # unknown
	 3  => 'amperageProbeTypeIs1Point5Volt',      # 1.5 amperage probe
	 4  => 'amperageProbeTypeIs3Point3volt',      # 3.3 amperage probe
	 5  => 'amperageProbeTypeIs5Volt',            # 5 amperage probe
	 6  => 'amperageProbeTypeIsMinus5Volt',       # -5 amperage probe
	 7  => 'amperageProbeTypeIs12Volt',           # 12 amperage probe
	 8  => 'amperageProbeTypeIsMinus12Volt',      # -12 amperage probe
	 9  => 'amperageProbeTypeIsIO',               # I/O probe
	 10 => 'amperageProbeTypeIsCore',             # Core probe
	 11 => 'amperageProbeTypeIsFLEA',             # FLEA (standby) probe
	 12 => 'amperageProbeTypeIsBattery',          # Battery probe
	 13 => 'amperageProbeTypeIsTerminator',       # SCSI Termination probe
	 14 => 'amperageProbeTypeIs2Point5Volt',      # 2.5 amperage probe
	 15 => 'amperageProbeTypeIsGTL',              # GTL (ground termination logic) probe
	 16 => 'amperageProbeTypeIsDiscrete',         # amperage probe with discrete reading
	 23 => 'amperageProbeTypeIsPowerSupplyAmps',  # Power Supply probe with reading in Amps
	 24 => 'amperageProbeTypeIsPowerSupplyWatts', # Power Supply probe with reading in Watts
	 25 => 'amperageProbeTypeIsSystemAmps',       # System probe with reading in Amps
	 26 => 'amperageProbeTypeIsSystemWatts',      # System probe with reading in Watts
	);

    my %amp_discrete
      = (
	 1 => 'Good',
	 2 => 'Bad',
	);

    my %amp_unit
      = (
	 'amperageProbeTypeIsPowerSupplyAmps'  => 'hA',  # tenths of Amps
	 'amperageProbeTypeIsSystemAmps'       => 'hA',  # tenths of Amps
	 'amperageProbeTypeIsPowerSupplyWatts' => 'W',   # Watts
	 'amperageProbeTypeIsSystemWatts'      => 'W',   # Watts
	 'amperageProbeTypeIsDiscrete'         => q{},   # discrete reading, no unit
	);

    my $count_pwr = 0;

  AMP:
    foreach my $out (@output) {
	if ($snmp) {
	    $index    = $out->{amperageProbeIndex} - 1;
	    $status   = $snmp_status{$out->{amperageProbeStatus}};
	    $reading  = $amp_type{$out->{amperageProbeType}} eq 'amperageProbeTypeIsDiscrete'
	      ? $amp_discrete{$out->{amperageProbeDiscreteReading}}
		: $out->{amperageProbeReading};
	    $location = $out->{amperageProbeLocationName};
	    $max_crit = exists $out->{amperageProbeUpperCriticalThreshold}
	      ? $out->{amperageProbeUpperCriticalThreshold} : 0;
	    $max_warn = exists $out->{amperageProbeUpperNonCriticalThreshold}
	      ? $out->{amperageProbeUpperNonCriticalThreshold} : 0;
	    $unit     = exists $amp_unit{$amp_type{$out->{amperageProbeType}}}
	      ? $amp_unit{$amp_type{$out->{amperageProbeType}}} : 'mA';
	    if ($unit eq 'hA') {
		$reading  /= 10;
		$max_crit /= 10;
		$max_warn /= 10;
		$unit      = 'A';
	    }
	}
	else {
	    $index    = $out->{'Index'};
	    next if $index !~ m/^\d+$/x;
	    $status   = $out->{'Status'};
	    $reading  = $out->{'Reading'};
	    $location = $out->{'Probe Name'};
	    $max_crit = $out->{'Failure Threshold'} ne '[N/A]'
	      ? $out->{'Failure Threshold'} : 0;
	    $max_warn = $out->{'Warning Threshold'} ne '[N/A]'
	      ? $out->{'Warning Threshold'} : 0;
	    $reading  =~ s{\A (\d+.*?)\s+([a-zA-Z]+) \s*\z}{$1}xms;
	    $unit     = $2;
	    $max_warn =~ s{\A (\d+.*?)\s+[a-zA-Z]+ \s*\z}{$1}xms;
	    $max_crit =~ s{\A (\d+.*?)\s+[a-zA-Z]+ \s*\z}{$1}xms;
	}

	next AMP if blacklisted('pm', $index);
	next AMP if $index !~ m{\A \d+ \z}xms;
	$count_pwr++;

	my $msg = sprintf 'Amperage probe %d (%s) reads %s %s',
	  $index, $location, $reading, $unit, $status;
	report('chassis', $msg, $status2nagios{$status}, $index);

	# Collect performance data
	if (defined $opt{'perfdata'}) {
	    next AMP if $reading !~ m{\A \d+(\.\d+)? \z}xms; # discrete reading (not number)
	    my $pname = lc $location;
	    $pname =~ s{\s}{_}gxms;
	    my $pkey = join q{_}, 'pwr_mon', $index, $pname;
	    my $pval = join q{;}, "$reading$unit", $max_warn, $max_crit;
	    $perfdata{$pkey} = $pval;
	}
    }

    # Collect EXTRA performance data not found at first run. This is a
    # rather ugly hack
    if (defined $opt{'perfdata'} && !$snmp) {
	my $found = 0;
	my $index = 0;
	my %used  = ();
	
	# find used indexes
	foreach (keys %perfdata) {
	    if (m/\A pwr_mon_(\d+)/xms) {
		$used{$1} = 1;
	    }
	}

      AMP2:
	foreach my $line (@{ run_command("$omreport $omopt_chassis pwrmonitoring -fmt ssv") }) {
	    chop $line;
	    if ($line eq 'Location;Reading') {
		$found = 1;
		next AMP2;
	    }
	    if ($line eq q{}) {
		$found = 0;
		next AMP2;
	    }
	    if ($found and $line =~ m/\A ([^;]+?) ; (\d*\.\d+) \s ([AW]) \z/xms) {
		my $aname = lc $1;
		my $aval = $2;
		my $aunit = $3;
		$aname =~ s{\s}{_}gxms;

		# don't use an existing index
		while (exists $used{$index}) { ++$index; }

		$perfdata{"pwr_mon_${index}_${aname}"} = "$aval$aunit;0;0";
		++$index;
	    }
	}
    }

    return $count_pwr;
}


#-----------------------------------------
# CHASSIS: Check intrusion
#-----------------------------------------
sub check_intrusion {
    my $index    = undef;
    my $status   = undef;
    my $reading  = undef;
    my @output = ();

    if ($snmp) {
	my %int_oid
	  = (
	     '1.3.6.1.4.1.674.10892.1.300.70.1.2.1' => 'intrusionIndex',
	     '1.3.6.1.4.1.674.10892.1.300.70.1.5.1' => 'intrusionStatus',
	     '1.3.6.1.4.1.674.10892.1.300.70.1.6.1' => 'intrusionReading',
	    );
	my $result = $snmp_session->get_entries(-columns => [keys %int_oid]);

	# No intrusion is OK
	return 0 if !defined $result;

	@output = @{ get_snmp_output($result, \%int_oid) };
    }
    else {
	@output = @{ run_omreport("$omopt_chassis intrusion") };
    }

    my %int_reading
      = (
	 1 => 'Not Breached',          # chassis not breached and no uncleared breaches
	 2 => 'Breached',              # chassis currently breached
	 3 => 'Breached Prior',        # chassis breached prior to boot and has not been cleared
	 4 => 'Breach Sensor Failure', # intrusion sensor has failed
	);

    my $count_intr = 0;

  INTRUSION:
    foreach my $out (@output) {
	if ($snmp) {
	    $index    = $out->{intrusionIndex} - 1;
	    $status   = $snmp_status{$out->{intrusionStatus}};
	    $reading  = $int_reading{$out->{intrusionReading}};
	}
	else {
	    $index    = $out->{'Index'};
	    $status   = $out->{'Status'};
	    $reading  = $out->{'State'};
	}

	next INTRUSION if blacklisted('intr', $index);
	$count_intr++;

	if ($status ne 'Ok') {
	    my $msg = sprintf 'Chassis intrusion %d detected: %s',
	      $index, $reading;
	    report('chassis', $msg, $E_WARNING, $index);
	}
	# Ok
	else {
	    my $msg = sprintf 'Chassis intrusion %d detection: %s (%s)',
	      $index, $status, $reading;
	    report('chassis', $msg, $E_OK, $index);
	}
    }
    return $count_intr;
}


#-----------------------------------------
# CHASSIS: Check alert log
#-----------------------------------------
sub check_alertlog {
    my %count = (
		 'Ok'           => 0,
		 'Non-Critical' => 0,
		 'Critical'     => 0,
		);

    return \%count if $snmp; # Not supported with SNMP

    my @output = @{ run_omreport("$omopt_system alertlog") };
    foreach my $out (@output) {
	++$count{$out->{Severity}};
    }

    # Create error messages and set exit value if appropriate
    my $err = 0;
    if ($count{'Critical'} > 0)        { $err = $E_CRITICAL; }
    elsif ($count{'Non-Critical'} > 0) { $err = $E_WARNING;  }

    my $msg = sprintf 'Alert log content: %d critical, %d non-critical, %d ok',
      $count{'Critical'}, $count{'Non-Critical'}, $count{'Ok'};
    report('other', $msg, $err);

    return \%count;
}

#-----------------------------------------
# CHASSIS: Check ESM log overall health
#-----------------------------------------
sub check_esmlog_health {
    my $health = 'Ok';

    if ($snmp) {
	my $systemStateEventLogStatus = '1.3.6.1.4.1.674.10892.1.200.10.1.41.1';
	my $result = $snmp_session->get_request(-varbindlist => [$systemStateEventLogStatus]);
	if (!defined $result) {
	    my $msg = sprintf 'SNMP ERROR getting systemStateEventLogStatus OID: %s',
	      $snmp_session->error;
	    report('other', $msg, $E_UNKNOWN);
	}
	$health = $snmp_status{$result->{$systemStateEventLogStatus}};
    }
    else {
	foreach (@{ run_command("$omreport $omopt_system esmlog -fmt ssv") }) {
	    if (m/\A Health;(.+) \z/xms) {
		$health = $1;
		chop $health;
		last;
	    }
	}
    }

    # If the overall health of the ESM log is other than "Ok", the
    # fill grade of the log is more than 80% and the log should be
    # cleared
    if ($health eq 'Ok') {
	my $msg = sprintf 'ESM log is health is OK (less than 80%% full)';
	report('other', $msg, $E_OK);
    }
    elsif ($health eq 'Critical') {
	my $msg = sprintf 'ESM log is 100%% full!';
	report('other', $msg, $status2nagios{$health});
    }
    else {
	my $msg = sprintf 'ESM log is more than 80%% full';
	report('other', $msg, $status2nagios{$health});
    }

    return;
}

#-----------------------------------------
# CHASSIS: Check ESM log
#-----------------------------------------
sub check_esmlog {
    my %count = (
		 'Ok'           => 0,
		 'Non-Critical' => 0,
		 'Critical'     => 0,
		);
    my @output = ();

    if ($snmp) {
	my %esm_oid
	  = (
	     '1.3.6.1.4.1.674.10892.1.300.40.1.7.1'  => 'eventLogSeverityStatus',
	    );
	my $result = $snmp_session->get_entries(-columns => [keys %esm_oid]);

	# No entries is OK
	return 0 if !defined $result;

	@output = @{ get_snmp_output($result, \%esm_oid) };
	foreach my $out (@output) {
	    ++$count{$snmp_status{$out->{eventLogSeverityStatus}}};
	}
    }
    else {
	@output = @{ run_omreport("$omopt_system esmlog") };
	foreach my $out (@output) {
	    ++$count{$out->{Severity}};
	}
    }

    # Create error messages and set exit value if appropriate
    my $err = 0;
    if ($count{'Critical'} > 0)        { $err = $E_CRITICAL; }
    elsif ($count{'Non-Critical'} > 0) { $err = $E_WARNING;  }

    my $msg = sprintf 'ESM log content: %d critical, %d non-critical, %d ok',
      $count{'Critical'}, $count{'Non-Critical'}, $count{'Ok'};
    report('other', $msg, $err);

    return \%count;
}

#
# Handy function for checking all storage components
#
sub check_storage {
    check_controllers();
    check_physical_disks();
    check_virtual_disks();
    check_cache_battery();
    check_connectors();
    check_enclosures();
    check_enclosure_fans();
    check_enclosure_pwr();
    check_enclosure_temp();
    check_enclosure_emms();
    return;
}



#---------------------------------------------------------------------
# Info functions
#---------------------------------------------------------------------

#
# Fetch output from 'omreport chassis info', put in sysinfo hash
#
sub get_omreport_chassis_info {
    if (open my $INFO, '-|', "$omreport $omopt_chassis info -fmt ssv") {
	my @lines = <$INFO>;
	close $INFO;
	foreach (@lines) {
	    next if !m/\A (Chassis\sModel|Chassis\sService\sTag|Model|Service\sTag)/xms;
	    my ($key, $val) = split /;/xms;
	    $key =~ s{\s+\z}{}xms; # remove trailing whitespace
	    $val =~ s{\s+\z}{}xms; # remove trailing whitespace
	    if ($key eq 'Chassis Model' or $key eq 'Model') {
		$sysinfo{model}  = $val;
	    }
	    if ($key eq 'Chassis Service Tag' or $key eq 'Service Tag') {
		$sysinfo{serial} = $val;
	    }
	}
    }
    return;
}

#
# Fetch output from 'omreport chassis bios', put in sysinfo hash
#
sub get_omreport_chassis_bios {
    if (open my $BIOS, '-|', "$omreport $omopt_chassis bios -fmt ssv") {
	my @lines = <$BIOS>;
	close $BIOS;
	foreach (@lines) {
	    next if !m/;/xms;
	    my ($key, $val) = split /;/xms;
	    $key =~ s{\s+\z}{}xms; # remove trailing whitespace
	    $val =~ s{\s+\z}{}xms; # remove trailing whitespace
	    $sysinfo{bios}     = $val if $key eq 'Version';
	    $sysinfo{biosdate} = $val if $key eq 'Release Date';
	}
    }
    return;
}

#
# Fetch output from 'omreport system operatingsystem', put in sysinfo hash
#
sub get_omreport_system_operatingsystem {
    if (open my $VER, '-|', "$omreport $omopt_system operatingsystem -fmt ssv") {
	my @lines = <$VER>;
	close $VER;
	foreach (@lines) {
	    next if !m/;/xms;
	    my ($key, $val) = split /;/xms;
	    $key =~ s{\s+\z}{}xms; # remove trailing whitespace
	    $val =~ s{\s+\z}{}xms; # remove trailing whitespace
	    if ($key eq 'Operating System') {
		$sysinfo{osname} = $val;
	    }
	    elsif ($key eq 'Operating System Version') {
		$sysinfo{osver}  = $val;
	    }
	}
    }
    return;
}

#
# Fetch output from 'omreport about', put in sysinfo hash
#
sub get_omreport_about {
    if (open my $OM, '-|', "$omreport about -fmt ssv") {
	my @lines = <$OM>;
	close $OM;
	foreach (@lines) {
	    if (m/\A Version;(.+) \z/xms) {
		$sysinfo{om} = $1;
		chomp $sysinfo{om};
	    }
	}
    }
    return;
}

#
# Fetch chassis info via SNMP, put in sysinfo hash
#
sub get_snmp_chassis_info {
    my %chassis_oid
      = (
	 '1.3.6.1.4.1.674.10892.1.300.10.1.9.1'  => 'chassisModelName',
	 '1.3.6.1.4.1.674.10892.1.300.10.1.11.1' => 'chassisServiceTagName',
	);

    my $chassisInformationTable = '1.3.6.1.4.1.674.10892.1.300.10.1';
    my $result = $snmp_session->get_table(-baseoid => $chassisInformationTable);

    if (defined $result) {
	foreach my $oid (keys %{ $result }) {
	    if (exists $chassis_oid{$oid} and $chassis_oid{$oid} eq 'chassisModelName') {
		$sysinfo{model} = $result->{$oid};
		$sysinfo{model} =~ s{\s+\z}{}xms; # remove trailing whitespace
	    }
	    elsif (exists $chassis_oid{$oid} and $chassis_oid{$oid} eq 'chassisServiceTagName') {
		$sysinfo{serial} = $result->{$oid};
	    }
	}
    }
    else {
	my $msg = sprintf 'SNMP ERROR getting chassis info: %s',
	  $snmp_session->error;
	report('other', $msg, $E_UNKNOWN);
    }
    return;
}

#
# Fetch BIOS info via SNMP, put in sysinfo hash
#
sub get_snmp_chassis_bios {
    my %bios_oid
      = (
	 '1.3.6.1.4.1.674.10892.1.300.50.1.7.1.1' => 'systemBIOSReleaseDateName',
	 '1.3.6.1.4.1.674.10892.1.300.50.1.8.1.1' => 'systemBIOSVersionName',
	);

    my $systemBIOSTable = '1.3.6.1.4.1.674.10892.1.300.50.1';
    my $result = $snmp_session->get_table(-baseoid => $systemBIOSTable);

    if (defined $result) {
	foreach my $oid (keys %{ $result }) {
	    if (exists $bios_oid{$oid} and $bios_oid{$oid} eq 'systemBIOSReleaseDateName') {
		$sysinfo{biosdate} = $result->{$oid};
		$sysinfo{biosdate} =~ s{\A (\d{4})(\d{2})(\d{2}).*}{$2/$3/$1}xms;
	    }
	    elsif (exists $bios_oid{$oid} and $bios_oid{$oid} eq 'systemBIOSVersionName') {
		$sysinfo{bios} = $result->{$oid};
	    }
	}
    }
    else {
	my $msg = sprintf 'SNMP ERROR getting BIOS info: %s',
	  $snmp_session->error;
	report('other', $msg, $E_UNKNOWN);
    }
    return;
}

#
# Fetch OS info via SNMP, put in sysinfo hash
#
sub get_snmp_system_operatingsystem {
    my %os_oid
      = (
	 '1.3.6.1.4.1.674.10892.1.400.10.1.6.1' => 'operatingSystemOperatingSystemName',
	 '1.3.6.1.4.1.674.10892.1.400.10.1.7.1' => 'operatingSystemOperatingSystemVersionName',
	);

    my $operatingSystemTable = '1.3.6.1.4.1.674.10892.1.400.10.1';
    my $result = $snmp_session->get_table(-baseoid => $operatingSystemTable);

    if (defined $result) {
	foreach my $oid (keys %{ $result }) {
	    if (exists $os_oid{$oid} and $os_oid{$oid} eq 'operatingSystemOperatingSystemName') {
		$sysinfo{osname} = ($result->{$oid});
	    }
	    elsif (exists $os_oid{$oid} and $os_oid{$oid} eq 'operatingSystemOperatingSystemVersionName') {
		$sysinfo{osver} = $result->{$oid};
	    }
	}
    }
    else {
	my $msg = sprintf 'SNMP ERROR getting OS info: %s',
	  $snmp_session->error;
	report('other', $msg, $E_UNKNOWN);
    }
    return;
}

#
# Fetch OMSA version via SNMP, put in sysinfo hash
#
sub get_snmp_about {
    my %omsa_oid
      = (
	 '1.3.6.1.4.1.674.10892.1.100.10.0' => 'systemManagementSoftwareGlobalVersionName',
	);
    my $systemManagementSoftwareGroup = '1.3.6.1.4.1.674.10892.1.100';
    my $result = $snmp_session->get_table(-baseoid => $systemManagementSoftwareGroup);
    if (defined $result) {
	foreach my $oid (keys %{ $result }) {
	    if (exists $omsa_oid{$oid} and $omsa_oid{$oid} eq 'systemManagementSoftwareGlobalVersionName') {
		$sysinfo{om} = ($result->{$oid});
	    }
	}
    }
    else {
	my $msg = sprintf 'SNMP ERROR getting OMSA info: %s',
	  $snmp_session->error;
	report('other', $msg, $E_UNKNOWN);
    }
    return;
}

#
# Collects some information about the system
#
sub get_sysinfo
{
    # Get system model and serial number
    $snmp ? get_snmp_chassis_info() : get_omreport_chassis_info();

    # Get BIOS information. Only if needed
    if ( $opt{okinfo} >= 1
	 or $opt{verbose}
	 or (defined $opt{postmsg} and $opt{postmsg} =~ m/[%][bd]/xms) ) {
	$snmp ? get_snmp_chassis_bios() : get_omreport_chassis_bios();
    }

    # Return now if verbose
    return if $opt{verbose};

    # Get OS information. Only if needed
    if (defined $opt{postmsg} and $opt{postmsg} =~ m/[%][or]/xms) {
	$snmp ? get_snmp_system_operatingsystem() : get_omreport_system_operatingsystem();
    }

    # Get OMSA information. Only if needed
    if ($opt{okinfo} >= 3) {
	$snmp ? get_snmp_about() : get_omreport_about();
    }

    return;
}


# Helper function for running omreport when the results are strictly
# name=value pairs.
sub run_omreport_info {
    my $command = shift;
    my %output  = ();
    my @keys    = ();

    # Run omreport and fetch output
    my $rawtext = slurp_command("$omreport $command -fmt ssv 2>&1");

    # Parse output, store in array
    for ((split /\n/xms, $rawtext)) {
	if (m/\A Error/xms) {
	    my $msg = "Problem running 'omreport $command': $_";
	    report('other', $msg, $E_UNKNOWN);
	}
	next if !m/;/xms;  # ignore lines with less than two fields
	my @vals = split m/;/xms;
	$output{$vals[0]} = $vals[1];
    }

    # Finally, return the collected information
    return \%output;
}

# Get various firmware information (BMC, RAC)
sub get_firmware_info {
    my @snmp_output = ();
    my %nrpe_output = ();

    if ($snmp) {
	my %fw_oid
	  = (
	     '1.3.6.1.4.1.674.10892.1.300.60.1.7.1'  => 'firmwareType',
	     '1.3.6.1.4.1.674.10892.1.300.60.1.8.1'  => 'firmwareTypeName',
	     '1.3.6.1.4.1.674.10892.1.300.60.1.11.1' => 'firmwareVersionName',
	    );

	my $firmwareTable = '1.3.6.1.4.1.674.10892.1.300.60.1';
	my $result = $snmp_session->get_table(-baseoid => $firmwareTable);

	# Some don't have this OID, this is ok
	if (!defined $result) {
	    return;
	}

	@snmp_output = @{ get_snmp_output($result, \%fw_oid) };
    }
    else {
	%nrpe_output = %{ run_omreport_info("$omopt_chassis info") };
    }

    my %fw_type  # Firmware types
      = (
	 1  => 'other',                              # other than following values
	 2  => 'unknown',                            # unknown
	 3  => 'systemBIOS',                         # System BIOS
	 4  => 'embeddedSystemManagementController', # Embedded System Management Controller
	 5  => 'powerSupplyParallelingBoard',        # Power Supply Paralleling Board
	 6  => 'systemBackPlane',                    # System (Primary) Backplane
	 7  => 'powerVault2XXSKernel',               # PowerVault 2XXS Kernel
	 8  => 'powerVault2XXSApplication',          # PowerVault 2XXS Application
	 9  => 'frontPanel',                         # Front Panel Controller
	 10 => 'baseboardManagementController',      # Baseboard Management Controller
	 11 => 'hotPlugPCI',                         # Hot Plug PCI Controller
	 12 => 'sensorData',                         # Sensor Data Records
	 13 => 'peripheralBay',                      # Peripheral Bay Backplane
	 14 => 'secondaryBackPlane',                 # Secondary Backplane for ESM 2 systems
	 15 => 'secondaryBackPlaneESM3And4',         # Secondary Backplane for ESM 3 and 4 systems
	 16 => 'rac',                                # Remote Access Controller
	 17 => 'imc'                                 # Integrated Management Controller
	);


    if ($snmp) {
	foreach my $out (@snmp_output) {
	    if ($fw_type{$out->{firmwareType}} eq 'baseboardManagementController') {
		$sysinfo{'bmc'} = 1;
		$sysinfo{'bmc_fw'} = $out->{firmwareVersionName};
	    }
	    elsif ($fw_type{$out->{firmwareType}} =~ m{\A rac|imc \z}xms) {
		my $name = $out->{firmwareTypeName}; $name =~ s/\s//gxms;
		$sysinfo{'rac'} = 1;
		$sysinfo{'rac_name'} = $name;
		$sysinfo{'rac_fw'} = $out->{firmwareVersionName};
	    }
	}
    }
    else {
	foreach my $key (keys %nrpe_output) {
	    next if !defined $nrpe_output{$key};
	    if ($key eq 'BMC Version' or $key eq 'Baseboard Management Controller Version') {
		$sysinfo{'bmc'} = 1;
		$sysinfo{'bmc_fw'} = $nrpe_output{$key};
	    }
	    elsif ($key =~ m{\A (i?DRAC)\s*(\d?)\s+Version}xms) {
		my $name = "$1$2";
		$sysinfo{'rac'} = 1;
		$sysinfo{'rac_fw'} = $nrpe_output{$key};
		$sysinfo{'rac_name'} = $name;
	    }
	}
    }

    return;
}



#=====================================================================
# Main program
#=====================================================================

# Counters
$i_count = 0;
%h_count = ('Ok' => 0, 'Non-Critical' => 0, 'Critical' => 0);

# Here we do the actual checking of components
if (defined $component) {
    # Do single selected check
    if ($component eq 'storage')        { check_storage();                  }
    elsif ($component eq 'fans')        { $i_count = check_fans();          }
    elsif ($component eq 'temperature') { $i_count = check_temperatures();  }
    elsif ($component eq 'memory')      { $i_count = check_memory();        }
    elsif ($component eq 'power')       { $i_count = check_powersupplies(); }
    elsif ($component eq 'cpu')         { $i_count = check_processors();    }
    elsif ($component eq 'voltage')     { $i_count = check_volts();         }
    elsif ($component eq 'batteries')   { $i_count = check_batteries();     }
    elsif ($component eq 'pwrmonitor')  { $i_count = check_pwrmonitoring(); }
    elsif ($component eq 'intrusion')   { $i_count = check_intrusion();     }
    elsif ($component eq 'alertlog')    { %h_count = %{ check_alertlog() }; }
    elsif ($component eq 'esmlog')      { %h_count = %{ check_esmlog() };   }
    elsif ($component eq 'esmhealth')   { check_esmlog_health();            }
}
else {
    # Check global status if applicable
    if ($opt{global}) {
	$globalstatus = check_global();
    }

    # Do multiple selected checks
    if ($check{storage})     { check_storage();       }
    if ($check{memory})      { check_memory();        }
    if ($check{fans})        { check_fans();          }
    if ($check{power})       { check_powersupplies(); }
    if ($check{temperature}) { check_temperatures();  }
    if ($check{cpu})         { check_processors();    }
    if ($check{voltage})     { check_volts();         }
    if ($check{batteries})   { check_batteries();     }
    if ($check{pwrmonitor})  { check_pwrmonitoring(); }
    if ($check{intrusion})   { check_intrusion();     }
    if ($check{alertlog})    { check_alertlog();      }
    if ($check{esmlog})      { check_esmlog();        }
    if ($check{esmhealth})   { check_esmlog_health(); }
}


#---------------------------------------------------------------------
# Finish up
#---------------------------------------------------------------------

# Counter variable
%nagios_alert_count
  = (
     'OK'       => 0,
     'WARNING'  => 0,
     'CRITICAL' => 0,
     'UNKNOWN'  => 0,
    );

# Get system information
get_sysinfo();

# Get firmware info if requested via option
if ($opt{okinfo} >= 1) {
    get_firmware_info();
}

# Close SNMP session
if ($snmp) {
    $snmp_session->close;
}

# Print messages
if ($opt{verbose}) {
    print "   System:      $sysinfo{model}\n";
    print "   ServiceTag:  $sysinfo{serial}\n";
    print "   BIOS/date:   $sysinfo{bios} $sysinfo{biosdate}\n";
    if ($#report_storage >= 0) {
	print "-----------------------------------------------------------------------------\n";
	print "   Storage Components                                                        \n";
	print "=============================================================================\n";
	print "  STATE  |    ID    |  MESSAGE TEXT                                          \n";
	print "---------+----------+--------------------------------------------------------\n";
	foreach (@report_storage) {
	    my ($msg, $level, $nexus) = @{$_};
	    print q{ } x (8 - length $reverse_exitcode{$level}) . "$reverse_exitcode{$level} | "
	      . q{ } x (8 - length $nexus) . "$nexus | $msg\n";
	    $nagios_alert_count{$reverse_exitcode{$level}}++;
	}
    }
    if ($#report_chassis >= 0) {
	print "-----------------------------------------------------------------------------\n";
	print "   Chassis Components                                                        \n";
	print "=============================================================================\n";
	print "  STATE  |  ID  |  MESSAGE TEXT                                          \n";
	print "---------+------+------------------------------------------------------------\n";
	foreach (@report_chassis) {
	    my ($msg, $level, $nexus) = @{$_};
	    print q{ } x (8 - length $reverse_exitcode{$level}) . "$reverse_exitcode{$level} | "
	      . q{ } x (4 - length $nexus) . "$nexus | $msg\n";
	    $nagios_alert_count{$reverse_exitcode{$level}}++;
	}
    }
    if ($#report_other >= 0) {
	print "-----------------------------------------------------------------------------\n";
	print "   Other messages                                                            \n";
	print "=============================================================================\n";
	print "  STATE  |  MESSAGE TEXT                                                     \n";
	print "---------+-------------------------------------------------------------------\n";
	foreach (@report_other) {
	    my ($msg, $level, $nexus) = @{$_};
	    print q{ } x (8 - length $reverse_exitcode{$level}) . "$reverse_exitcode{$level} | $msg\n";
	    $nagios_alert_count{$reverse_exitcode{$level}}++;
	}
    }
}
else {
    my $c = 0;  # counter to determine linebreaks

    # Run through each message, sorted by severity level
  ALERT:
    foreach (sort {$a->[1] < $b->[1]} (@report_storage, @report_chassis, @report_other)) {
	my ($msg, $level, $nexus) = @{ $_ };
	next ALERT if $level == $E_OK;

	# If user wants only critical alerts
	next ALERT if ($opt{only_critical} and $level == $E_WARNING);

	# If user wants only warning alerts
	next ALERT if ($opt{only_warning} and $level == $E_CRITICAL);

	# Prefix with service tag if specified with option '-i|--info'
	if ($opt{info}) {
	    if (defined $opt{htmlinfo}) {
		$msg = '[<a href="' . warranty_url($sysinfo{serial})
		  . "\">$sysinfo{serial}</a>] " . $msg;
	    }
	    else {
		$msg = "[$sysinfo{serial}] " . $msg;
	    }
	}

	# Prefix with nagios level if specified with option '--state'
	$msg = $reverse_exitcode{$level} . ": $msg" if $opt{state};

	# Prefix with one-letter nagios level if specified with option '--short-state'
	$msg = (substr $reverse_exitcode{$level}, 0, 1) . ": $msg" if $opt{shortstate};

	($c++ == 0) ? print $msg : print $linebreak, $msg;

	$nagios_alert_count{$reverse_exitcode{$level}}++;
    }
}

# Determine our exit code
$exit_code = $E_OK;
$exit_code = $E_UNKNOWN  if $nagios_alert_count{'UNKNOWN'} > 0;
$exit_code = $E_WARNING  if $nagios_alert_count{'WARNING'} > 0;
$exit_code = $E_CRITICAL if $nagios_alert_count{'CRITICAL'} > 0;

# Global status via SNMP.. extra safety check
if ($globalstatus != $E_OK && $exit_code == $E_OK && !$opt{only_critical} && !$opt{only_warning}) {
    print "OOPS! Something is wrong with this server, but I don't know what. ";
    print "The global system health status is $reverse_exitcode{$globalstatus}, ";
    print "but every component check is OK. This may be a bug in the Nagios plugin, ";
    print "please file a bug report.\n";
    exit $E_UNKNOWN;
}

# Print OK message
if ($exit_code == $E_OK && defined $component && !$opt{verbose}) {
    my %okmsg
      = ( 'storage'     => "all storage components ok, $no_of_pdisks physical drives, $no_of_vdisks logical drives",
	  'fans'        => $i_count == 0 && $blade ? 'blade system with no fan probes' : "all $i_count fans ok",
	  'temperature' => "all $i_count temperatures ok",
	  'memory'      => "all $i_count memory modules ok",
	  'power'       => $i_count == 0 ? 'no instrumented power supplies found' : "all $i_count power supplies ok",
	  'cpu'         => "all $i_count processors ok",
	  'voltage'     => "all $i_count voltage probes ok",
	  'batteries'   => $i_count == 0 ? 'no batteries found' : "all $i_count batteries ok",
	  'pwrmonitor'  => $i_count == 0 ? 'no power monitoring probes found' : "all $i_count power monitoring probes ok",
	  'intrusion'   => $i_count == 0 ? 'no intrusion detection probes found' : "all $i_count intrusion detection probes ok",
	  'alertlog'    => $snmp ? 'not supported via snmp' : "all alerts: $h_count{Ok} ok, $h_count{'Non-Critical'} warning and $h_count{Critical} critical",
	  'esmlog'      => "all esm log entries: $h_count{Ok} ok, $h_count{'Non-Critical'} warning and $h_count{Critical} critical",
	  'esmhealth'   => "ESM log health ok",
	);

    print 'OK - ' . $okmsg{$component};
}
elsif ($exit_code == $E_OK && !$opt{verbose}) {
    if (defined $opt{htmlinfo}) {
	printf q{OK - System: '<a href="%s">%s</a>', SN: '<a href="%s">%s</a>', hardware working fine},
	  documentation_url($sysinfo{model}), $sysinfo{model},
	    warranty_url($sysinfo{serial}), $sysinfo{serial};
    }
    else {
	printf q{OK - System: '%s', SN: '%s', hardware working fine},
	  $sysinfo{model}, $sysinfo{serial};
    }


    if ($check{storage}) {
	printf ', %d logical drives, %d physical drives',
	  $no_of_vdisks, $no_of_pdisks;
    }
    else {
	print ', not checking storage';
    }

    if ($opt{okinfo} >= 1) {
	print $linebreak;
	printf q{----- BIOS='%s %s'}, $sysinfo{bios}, $sysinfo{biosdate};

	if ($sysinfo{rac}) {
	    printf q{, %s='%s'}, $sysinfo{rac_name}, $sysinfo{rac_fw};
	}
	if ($sysinfo{bmc}) {
	    printf q{, BMC='%s'}, $sysinfo{bmc_fw};
	}
    }

    if ($opt{okinfo} >= 2) {
	if ($check{storage}) {
	    my @storageprint = ();
	    foreach my $id (sort keys %{ $sysinfo{controller} }) {
		chomp $sysinfo{controller}{$id}{driver};
		push @storageprint, sprintf q{----- CTRL %s (%s): FW='%s', DR='%s'},
		  $sysinfo{controller}{$id}{id}, $sysinfo{controller}{$id}{name},
		    $sysinfo{controller}{$id}{firmware}, $sysinfo{controller}{$id}{driver};
	    }
	    foreach my $id (sort keys %{ $sysinfo{enclosure} }) {
		push @storageprint, sprintf q{----- ENCL %s (%s): FW='%s'},
		  $sysinfo{enclosure}{$id}->{id}, $sysinfo{enclosure}{$id}->{name},
		    $sysinfo{enclosure}{$id}->{firmware};
	    }

	    # print stuff
	    foreach my $line (@storageprint) {
		print $linebreak, $line;
	    }
	}
    }

    if ($opt{okinfo} >= 3) {
	print "$linebreak----- OpenManage Server Administrator (OMSA) version: '$sysinfo{om}'";
    }

}
else {
    if ($opt{extinfo}) {
	print $linebreak;
	if (defined $opt{htmlinfo}) {
	    printf '------ SYSTEM: <a href="%s">%s</a>, SN: <a href="%s">%s</a>',
	      documentation_url($sysinfo{model}), $sysinfo{model},
		warranty_url($sysinfo{serial}), $sysinfo{serial};
	}
	else {
	    printf '------ SYSTEM: %s, SN: %s',
	      $sysinfo{model}, $sysinfo{serial};
	}
    }
    if (defined $opt{postmsg}) {
	my $post = undef;
	if (-f $opt{postmsg}) {
	    open my $POST, '<', $opt{postmsg}
	      or ( print $linebreak
		   and print "ERROR: Couldn't open post message file $opt{postmsg}: $!\n"
		   and exit $E_UNKNOWN );
	    $post = <$POST>;
	    close $POST;
	    chomp $post;
	}
	else {
	    $post = $opt{postmsg};
	}
	if (defined $post) {
	    print $linebreak;
	    $post =~ s{[%]s}{$sysinfo{serial}}gxms;
	    $post =~ s{[%]m}{$sysinfo{model}}gxms;
	    $post =~ s{[%]b}{$sysinfo{bios}}gxms;
	    $post =~ s{[%]d}{$sysinfo{biosdate}}gxms;
	    $post =~ s{[%]o}{$sysinfo{osname}}gxms;
	    $post =~ s{[%]r}{$sysinfo{osver}}gxms;
	    $post =~ s{[%]p}{$no_of_pdisks}gxms;
	    $post =~ s{[%]l}{$no_of_vdisks}gxms;
	    $post =~ s{[%]n}{$linebreak}gxms;
	    $post =~ s{[%]{2}}{%}gxms;
	    print $post;
	}
    }
}

# Print performance data
if (defined $opt{perfdata} && !$opt{verbose} && %perfdata) {
    my $lb = $opt{perfdata} eq 'multiline' ? "\n" : q{ };  # line break for perfdata
    print q{|};

    sub perfdata {
	my %order
	  = (
	     fan       => 0,
	     pwr       => 1,
	     temp      => 2,
	     enclosure => 3,
	    );
	return ($order{(split /_/, $a, 2)[0]} cmp $order{(split /_/, $b, 2)[0]}) || $a cmp $b;
    }

    print join $lb, map { "'$_'=$perfdata{$_}" } sort perfdata keys %perfdata;
}
print "\n" if !$opt{verbose};

# Exit with proper exit code
exit $exit_code;


# Man page created with:
#
#  pod2man -s 3pm -r "`./check_openmanage -V | head -n 1`" -c 'Nagios plugin' check_openmanage check_openmanage.3pm
#

__END__

=head1 NAME

check_openmanage - Nagios plugin for checking the hardware status on
                   Dell servers running OpenManage

=head1 SYNOPSIS

check_openmanage [I<OPTION>]...

=head1 DESCRIPTION

check_openmanage is a plugin for Nagios which checks the hardware
health of Dell PowerEdge and PowerVault servers. It uses the Dell
OpenManage Server Administrator (OMSA) software to accomplish this
task. check_openmanage can be used with SNMP or NRPE, whichever suits
your needs and particular taste. The plugin checks the health of the
storage subsystem, power supplies, memory modules, temperature probes
etc., and gives an alert if any of the components are faulty or
operate outside normal parameters.

check_openmanage is designed to be used by either locally (using NRPE)
or remotely (using SNMP). In either mode, the output is (nearly) the
same. Note that checking the alert log is not supported in SNMP mode.

=head2 Alternate Basename

=over 4

The normal basename is C<check_openmanage>. With this every component
in the server is checked (modifiable via the B<--check> option). You
can create symbolic links C<check_openmanage_COMPONENT> which changes
the behaviour of the plugin. The C<COMPONENT> part may be one of

=over 4

=item B<storage>

Only check storage

=item B<memory>

Only check memory modules

=item B<fans>

Only check fans

=item B<power>

Only check power supplies

=item B<temperature>

Only check temperatures

=item B<cpu>

Only check processors

=item B<voltage>

Only check voltage probes

=item B<batteries>

Only check batteries

=item B<pwrmonitor>

Only check power usage

=item B<intrusion>

Only check chassis intrusion

=item B<esmhealth>

Only check ESM log overall health, i.e. fill grade

=item B<esmlog>

Only check the event log (ESM) content

=item B<alertlog>

Only check the alert log content

=back

=back

=head1 OPTIONS

=head2 General Options

=over 4

=over 4

=item -t, --timeout I<SECONDS>

The number of seconds after which the plugin will abort. Default
timeout is 30 seconds if the option is not present.

=item -g, --global

Check everything except logs. By default log content and chassis
intrusion sensor are skipped. With this option everything will be
checked except for log contents.

If used with SNMP, the global system health OID is also probed. This
gives an added security against bugs in the plugin. The plugin will
produce an special error message in cases where 1) the global status
is not OK, and 2) a hardware error has not been detected by the rest
of the plugin.

If used with omreport, i.e. via NRPE or similar, the output from
C<omreport system> is used to find the global chassis health. Note
that storage health is excluded. Not as good as with SNMP, but it
still means added security against plugin bugs.

This option negates the C<--check> option described below, for all
checks but the esmlog and alertlog. If used with alternate basenames,
the option has no effect.

NOTE: If blacklisting is used, this option is negated.

=item --only-critical

Print only critical alerts. With this option any warning alerts are
suppressed. Using this option negates the B<--global> option.

=item --only-warning

Print only warning alerts. With this option any critical alerts are
suppressed. Using this option negates the B<--global> option.

=item -p, --perfdata [I<multline>]

Collect performance data. Performance data collected include
temperatures (in Celcius) and fan speeds (in rpm). On systems that
support it, power consumption is also collected (in Watts).

If given the argument C<multiline>, the plugin will output the
performance data on multiple lines, for Nagios 3.x and above.

=item -w, --warning I<STRING> or I<FILE>

Override the machine-default temperature warning thresholds. Syntax is
C<id1=max[/min],id2=max[/min],...>. The following example sets warning
limits to max 50C for probe 0, and max 45C and min 10C for probe 1:

check_openmanage -w 0=50,1=45/10

The minimum limit can be omitted, if desired. Most often, you are only
interested in setting the maximum thresholds.

This parameter can be either a string with the limits, or a file
containing the limits string. The option can be specified multiple
times.

=item -c, --critical I<STRING> or I<FILE>

Override the machine-default temperature critical thresholds. Syntax
and behaviour is the same as for warning thresholds described above.

=item -o, --ok-info I<NUMBER>

This option lets you define how much output you want the plugin to
give when everything is OK, i.e. the verbosity level. The default
value is 0 (one line of output). The output levels are cumulative.

=over 4

=item B<0>

- Only one line (default)

=item B<1>

- BIOS and firmware info on a separate line

=item B<2>

- Storage controller and enclosure info on separate lines

=item B<3>

- OMSA version on separate line

=back

The reason that OMSA version is separated from the rest is that
finding it requires running a really slow omreport command, when the
plugin is run locally via NRPE.

=item -i, --info

Prefix any alerts with the service tag.

=item -e, --extinfo

Display a short summary of system information (model and service tag)
in case of an alert.

=item --htmlinfo [I<CODE>]

Using this option will make the servicetag and model name into
clickable HTML links in the output. The model name link will point to
the official Dell documentation for that model, while the servicetag
link will point to a website containing support info for that
particular server.

This option takes an optional argument, which should be your country
code or C<me> for the middle east. If the country code is omitted the
servicetag link will still work, but it will not be speficic for your
country or area. Example for Germany:

  check_openmanage --htmlinfo de

If this option is used together with either the I<--extinfo> or
I<--info> options, it is particularly useful. Only the most common
country codes is supported at this time.

=item --postmsg I<STRING> or I<FILE>

User specified post message. Useful for displaying arbitrary or
various system information at the end of alerts. The argument is
either a string with the message, or a file containing that
string. You can control the format with the following interpreted
sequences:

=over 4

=item B<%m>

System model

=item B<%s>

Service tag

=item B<%b>

BIOS version

=item B<%d>

BIOS release date

=item B<%o>

Operating system name

=item B<%r>

Operating system release

=item B<%p>

Number of physical drives

=item B<%l>

Number of logical drives

=item B<%n>

Line break. Will be a regular line break if run from a TTY, else an
HTML line break.

=item B<%%>

A literal C<%>

=back

=item --state

Prefix each alert with its corresponding service state (i.e. warning,
critical etc.). This is useful in case of several alerts from the same
monitored system.

=item --short-state

Same as the B<--state> option above, except that the state is
abbreviated to a single letter (W=warning, C=critical etc.).

=item --linebreak=I<STRING>

check_openmanage will sometimes report more than one line, e.g. if
there are several alerts. If the script has a TTY, it will use regular
linebreaks. If not (which is the case with NRPE) it will use HTML
linebreaks. Sometimes it can be useful to control what the plugin uses
as a line separator, and this option provides that control.

The argument is the exact string to be used as the line
separator. There are two exceptions, i.e. two keywords that translates
to the following:

=over 4

=item B<REG>

Regular linebreaks, i.e. "\n".

=item B<HTML>

HTML linebreaks, i.e. "<br/>".

=back

This is a rather special option that is normally not needed. The
default behaviour should be sufficient for most users.

=item -v, --verbose

Verbose output. Will report status on everything, even if status is
ok. Blacklisted or unchecked components are ignored (i.e. no output).

NOTE: This option is intended for diagnostics and debugging purposes
only. Do not use this option from within Nagios, i.e. in the Nagios
config.

=item -h, --help

Display help text.

=item -m, --man

Display man page.

=item -V, --version

Display version info.

=back

=back

=head2 SNMP Options

=over 4

=over 4

=item -H, --hostname I<HOSTNAME>

The transport address of the destination SNMP device. Using this
option triggers SNMP mode.

=item -P, --protocol I<PROTOCOL>

SNMP protocol version. This option is optional and expects a digit
(i.e.  C<1>, C<2> or C<3>) to define the SNMP version. The default is
C<2>, i.e. SNMP version 2c.

=item -C, --community I<COMMUNITY>

This option expects a string that is to be used as the SNMP community
name when using SNMP version 1 or 2c.  By default the community name
is set to C<public> if the option is not present.

=item --port I<PORT>

SNMP port of the remote (monitored) system. Defaults to the well-known
SNMP port 161.

=item -s, --snmp

This option does nothing.

=item -U, --username I<SECURITYNAME>

[SNMPv3] The User-based Security Model (USM) used by SNMPv3 requires
that a securityName be specified. This option is required when using
SNMP version 3, and expects a string 1 to 32 octets in lenght.

=item --authpassword I<PASSWORD>, --authkey I<KEY>

[SNMPv3] By default a securityLevel of C<noAuthNoPriv> is assumed.  If
the --authpassword option is specified, the securityLevel becomes
C<authNoPriv>.  The --authpassword option expects a string which is at
least 1 octet in length as argument.

Optionally, instead of the --authpassword option, the --authkey option
can be used so that a plain text password does not have to be
specified in a script.  The --authkey option expects a hexadecimal
string produced by localizing the password with the
authoritativeEngineID for the specific destination device.  The
C<snmpkey> utility included with the Net::SNMP distribution can be
used to create the hexadecimal string (see L<snmpkey>).

=item --authprotocol I<ALGORITHM>

[SNMPv3] Two different hash algorithms are defined by SNMPv3 which can
be used by the Security Model for authentication. These algorithms are
HMAC-MD5-96 C<MD5> (RFC 1321) and HMAC-SHA-96 C<SHA-1> (NIST FIPS PUB
180-1). The default algorithm used by the plugin is HMAC-MD5-96.  This
behavior can be changed by using this option. The option expects
either the string C<md5> or C<sha> to be passed as argument to modify
the hash algorithm.

=item --privpassword I<PASSWORD>, --privkey I<KEY>

[SNMPv3] By specifying the options --privkey or --privpassword, the
securityLevel associated with the object becomes
C<authPriv>. According to SNMPv3, privacy requires the use of
authentication. Therefore, if either of these two options are present
and the --authkey or --authpassword arguments are missing, the
creation of the object fails.  The --privkey and --privpassword
options expect the same input as the --authkey and --authpassword
options respectively.

=item --privprotocol I<ALGORITHM>

[SNMPv3] The User-based Security Model described in RFC 3414 defines a
single encryption protocol to be used for privacy.  This protocol,
CBC-DES C<DES> (NIST FIPS PUB 46-1), is used by default or if the
string C<des> is passed to the --privprotocol option. The Net::SNMP
module also supports RFC 3826 which describes the use of
CFB128-AES-128 C<AES> (NIST FIPS PUB 197) in the USM.  The AES
encryption protocol can be selected by passing C<aes> or C<aes128> to
the --privprotocol option.

One of the following arguments are required: des, aes, aes128, 3des,
3desde

=back

=back

=head2 Blacklisting

=over 4

=over 4

=item -b, --blacklist I<STRING> or I<FILE>

Blacklist missing and/or failed components, if you do not plan to fix
them. The parameter is either the blacklist string, or a file (that
may or may not exist) containing the string. The blacklist string
contains component names with component IDs separated by slash
(/). Blacklisted components are left unchecked.

TIP: Use the option C<-v> (or C<--verbose>) to get the blacklist ID for
devices. The ID is listed in a separate column in the verbose output.

NOTE: If blacklisting is in effect, the use of the I<--global> option
is negated.

=over 9

=item B<Syntax:>

component1=id1[,id2,...]/component2=id1[,id2,...]/...

=item B<Example:>

check_openmanage -b ps=0/fan=3,5/pdisk=1:0:0:1

=back

In the example we blacklist powersupply 0, fans 3 and 5, and
physical disk 1:0:0:1. Legal component names include:

=over 8

=item B<ctrl>

Controller

=item B<ctrl_fw>

Suppress the special warning message about old controller
firmware. Use this if you can not or will not upgrade the firmware.

=item B<ctrl_driver>

Suppress the special warning message about old controller driver.
Particularly useful on systems where you can not upgrade the driver.

=item B<pdisk>

Physical disk.

=item B<vdisk>

Logical drive (virtual disk)

=item B<bat>

Controller cache battery

=item B<conn>

Connector (channel)

=item B<encl>

Enclosure

=item B<encl_fan>

Enclosure fan

=item B<encl_ps>

Enclosure power supply

=item B<encl_temp>

Enclosure temperature probe

=item B<encl_emm>

Enclosure management module (EMM)

=item B<dimm>

Memory module

=item B<fan>

Fan

=item B<ps>

Powersupply

=item B<temp>

Temperature sensor

=item B<cpu>

Processor (CPU)

=item B<volt>

Voltage probe

=item B<bp>

System battery

=item B<pm>

Amperage probe (power consumption monitoring)

=item B<intr>

Intrusion sensor

=back

=back

=back

=head2 Check Control

=over 4

=over 4

=item --check I<STRING> or I<FILE>

This parameter allows you to adjust which components that should be
checked at all. This is a rougher approach than blacklisting, which
require that you specify component id or index. The parameter should
be either a string containing the adjustments, or a file containing
the string. No errors are raised if the file does not exist.

Note: This option is ignored with alternate basenames.

=over 9

=item B<Example:>

check_openmanage --check storage=0,intrusion=1

=back

Legal values are described below, along with the default value.

=over 4

=item B<storage>

Check storage subsystem (controllers, disks etc.). Default: ON

=item B<memory>

Check memory (dimms). Default: ON

=item B<fans>

Check chassis fans. Default: ON

=item B<power>

Check power supplies. Default: ON

=item B<temperature>

Check temperature sensors. Default: ON

=item B<cpu>

Check CPUs. Default: ON

=item B<voltage>

Check voltage sensors. Default: ON

=item B<batteries>

Check system batteries. Default: ON

=item B<pwrmonitor>

Check power consumption monitoring. Default: ON

=item B<intrusion>

Check chassis intrusion. Default: OFF

=item B<esmhealth>

Check the ESM log health, i.e. fill grade. Default: ON

=item B<esmlog>

Check the ESM log content. Default: OFF

=item B<alertlog>

Check the alert log content. Default: OFF

=back

=back

=back

=head1 DIAGNOSTICS

The option C<--verbose> (or C<-v>) can be specified to display all
monitored components.

=head1 DEPENDENCIES

If SNMP is requested, the perl module Net::SNMP is
required. Otherwise, only a regular perl distribution is required to
run the script. On the target (monitored) system, Dell Openmanage
Server Administrator (OMSA) must be installed and running.

=head1 EXIT STATUS

If no errors are discovered, a value of 0 (OK) is returned. An exit
value of 1 (WARNING) signifies one or more non-critical errors, while
2 (CRITICAL) signifies one or more critical errors.

The exit value 3 (UNKNOWN) is reserved for errors within the script,
or errors getting values from Dell OMSA.

=head1 AUTHOR

Written by Trond H. Amundsen <t.h.amundsen@usit.uio.no>

=head1 BUGS AND LIMITATIONS

Storage info is not collected or checked on very old PowerEdge models
and/or old OMSA versions, due to limitations in OMSA. The overall
support on those models/versions by this plugin is not well tested.

=head1 INCOMPATIBILITIES

The plugin should work with the Nagios embedded perl interpreter
(ePN). However, this is not thoroughly tested, and the script
specifies C<nagios: -epn> among the first 10 lines to disable ePN as
to avoid any C<accidents>. If you want to use this script with ePN,
remove this line from the script.

=head1 REPORTING BUGS

Report bugs to <t.h.amundsen@usit.uio.no>

=head1 LICENSE AND COPYRIGHT

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or (at
your option) any later version.

This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see L<http://www.gnu.org/licenses/>.

=head1 SEE ALSO

L<http://folk.uio.no/trondham/software/check_openmanage.html>

=cut
