#!/usr/bin/perl -w
# nagios: -epn
#
# check_multi - nagios plugin
#
# Copyright (c) 2007-8 Matthias Flacke (matthias.flacke at gmx.de)
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
#
# $Id: check_multi 186 2009-02-11 06:07:51Z flackem $
#
# EPN capability only without strictness :-/
#use strict; 
use Getopt::Long qw(:config no_ignore_case bundling);
BEGIN { eval("use Time::HiRes qw(time)") }
use lib "/usr/local/nagios/libexec";
use vars qw(
$MYSELF %cmds $returncode %def %rc $no $VERSION $tmp_stdout $tmp_stderr $xml
$OK $WARNING $CRITICAL $UNKNOWN
$DETAIL_LIST $DETAIL_LIST_FULL $DETAIL_HTML $DETAIL_STDERR $DETAIL_PERFORMANCE
$DETAIL_PERFORMANCE_CLASSIC $DETAIL_STATUS $DETAIL_PERFORMANCE_LINK $DETAIL_XML $DETAIL_NAGIOS2
$DETAIL_NOTES_LINK $DETAIL_SERVICE_DEFINITION
*DEBUG1 *DEBUG2 *DEBUG3
);
#-------------------------------------------------------------------------------
#--- vars ----------------------------------------------------------------------
#-------------------------------------------------------------------------------
$MYSELF="check_multi";
$VERSION='$Revision: 186 $ $Date: 2009-02-11 07:07:51 +0100 (Wed, 11 Feb 2009) $ $Author: flackem $';
#
#--- RC defines
$OK=0;
$WARNING=1;
$CRITICAL=2;
$UNKNOWN=3;
#
#--- report defines
$DETAIL_LIST=1;
$DETAIL_HTML=2;
$DETAIL_STDERR=4;
$DETAIL_PERFORMANCE=8;
$DETAIL_LIST_FULL=16;
$DETAIL_PERFORMANCE_CLASSIC=32;
$DETAIL_STATUS=64;
$DETAIL_PERFORMANCE_LINK=128;
$DETAIL_XML=256;
$DETAIL_NAGIOS2=512;
$DETAIL_NOTES_LINK=1024;
$DETAIL_SERVICE_DEFINITION=2048;
#
#--- vars
$no=0;
$returncode=0;
$tmp_stdout="";
$tmp_stderr="";
%def=(
	label	=> { $OK  => "OK", $WARNING  => "WARNING", $CRITICAL  => "CRITICAL", $UNKNOWN  => "UNKNOWN", },
	code	=> { "OK" => $OK,  "WARNING" => $WARNING,  "CRITICAL" => $CRITICAL,  "UNKNOWN" => $UNKNOWN,  },
	s2r	=> { 0    => $OK,  2         => $WARNING,  3          => $CRITICAL,  1         => $UNKNOWN,  },
	r2s	=> { $OK  => 0,    $WARNING  => 2,         $CRITICAL  => 3,          $UNKNOWN  => 1,         },
);
my %opt=(
	"execute"	=> [],
	"filename"	=> [],
	"set"		=> {
		action_url	=> "",
		collapse	=> 1,
		exec_open3	=> 0,
		file_extension	=> "cmd",
		ignore_missing_cmd_file => 0,
		illegal_chars	=> "\r",
		indent		=> " ",
		indent_label	=> 1,
		libexec		=> "/usr/local/nagios/libexec",
		name		=> "",
		report		=> 13,
		no_checks_rc	=> $UNKNOWN,
		notes_url	=> "",
		persistent	=> 0,
		run_as_user	=> "nagios",
		service_definition_template => "/tmp/service_definition.tpl",
		tag_notes_link	=> 0,
		target		=> "_self",
		test		=> 0,
		timeout		=> 10,
		TIMEOUT		=> 60,
		verbose		=> 0,
		tmp_dir		=> "/tmp/check_multi",
		tmp_dir_permissions => 01777,
		xml_elements	=> "name,rc,output,error,plugin,command,performance,runtime",
	},

	"ok"		=> undef,
	"warning"	=> undef,
	"critical"	=> undef,
	"unknown"	=> undef,
);
my %cmds=(
	"0" => {
		name	=> "",
		error	=> [ ],		# stderr and other errors
		starttime => 0.0,	# start timestamp
		timeouttime => 0.0,	# timeout timestamp
		endtime => 0.0,		# end timestamp
		runtime => 0.0,		# runtime in seconds
		nchecks => 0,		# number of child checks
	}
);
my %rc=(
	count	=> { $OK => 0, $WARNING => 0, $CRITICAL => 0, $UNKNOWN => 0, },
	list	=> { $OK => [],$WARNING => [],$CRITICAL => [],$UNKNOWN => [], },
	expr	=> { $OK => "1", $WARNING => "COUNT(WARNING)>0", $CRITICAL => "COUNT(CRITICAL)>0", $UNKNOWN => "COUNT(UNKNOWN)>0" },
	match	=> { $OK => 0, $WARNING => 0, $CRITICAL => 0, $UNKNOWN => 0, },
	top	=> $OK,
);

	
#-------------------------------------------------------------------------------
#--- subs ----------------------------------------------------------------------
#-------------------------------------------------------------------------------

sub process_parameters {

	#--- check version of modules
	if ($Getopt::Long::VERSION < 2.27) {
		print "Error: module Getopt::Long version $Getopt::Long::VERSION is too old, minimum version is 2.27\n";
		return $UNKNOWN;
	}

	if (! GetOptions(
		"f|filename=s"	=> \@{$opt{filename}},
		"h|help"	=> \$opt{help},
		"l|libexec=s"	=> \$opt{set}{libexec},
		"n|name=s"	=> \$opt{set}{name},
		"r|report:i"	=> \$opt{set}{report},
		"s|set=s"	=> \%{$opt{set}},
		"t|timeout=i"	=> \$opt{set}{timeout},
		"T|TIMEOUT=i"	=> \$opt{set}{TIMEOUT},
		"v|verbose:+"	=> \$opt{set}{verbose},
		"V|version"	=> \$opt{version},
		"x|execute=s"	=> \@{$opt{execute}},
		"o|O|ok=s"	=> \$opt{ok},
		"w|W|warning=s"	=> \$opt{warning},
		"c|C|critical=s"=> \$opt{critical},
		"u|U|unknown=s"	=> \$opt{unknown},)
	) {
		short_usage();
		return $UNKNOWN;
	}
	*DEBUG1=($opt{set}{verbose}>=1) ? \&debug_message : sub {};
	*DEBUG2=($opt{set}{verbose}>=2) ? \&debug_message : sub {};
	*DEBUG3=($opt{set}{verbose}>=3) ? \&debug_message : sub {};

	if ($opt{version}) {
		print "$MYSELF: v$VERSION\n";
		return $UNKNOWN;
	}
	if ($opt{help}) {
		short_usage();
		long_usage();
		return $UNKNOWN;
	}
	if (! $opt{filename}[0] && ! $opt{execute}[0]) {
		print "$MYSELF error: no config file(s) or command parameters specified\n";
		short_usage();
		return $UNKNOWN;
	} else {
		#--- check if filenames are directories and replace with '*cmd' files
		for (my $i=0;$i<@{$opt{filename}}; $i++) {
			splice(@{$opt{filename}},$i,1,glob "$opt{filename}[$i]/*$opt{set}->{file_extension}") if (-d $opt{filename}[$i]);
		}
	}
	if ($opt{set}{report} & $DETAIL_PERFORMANCE_LINK && !$opt{set}{name}) {
		$opt{set}{name}=$MYSELF;
		DEBUG2("process_parameters: performance report option set and no name defined: taking $MYSELF as name");
	}
	if ($opt{set}{report} & $DETAIL_NOTES_LINK && !$opt{set}{notes_url}) {
		$opt{set}{notes_url}=$ENV{NAGIOS_SERVICENOTESURL};
		DEBUG2("process_parameters: notes report option set - taking Nagios SERVICENOTESURL $ENV{NAGIOS_SERVICENOTESURL}");
	}
	#--- if no module HTML::Entities available - set var
	if ($opt{set}{report} & $DETAIL_HTML) {
		$opt{set}{use_html_entities} = 1;
		unless (eval "require HTML::Entities;1") {
			$opt{set}{use_html_entities} = 0;
			DEBUG2("process_parameters: HTML::Entities not available");
		}
	}
	#--- if no modules IPC::Open3 or IO::Select, use old backquote exec and temp files
	if ($opt{set}{exec_open3}) {
		unless (eval "use IPC::Open3;1") {
			$opt{set}{exec_open3}=0;
			DEBUG2("process_parameters: IPC::Open3 not available");
		}
	}
	if ($opt{set}{exec_open3}) {
		unless (eval "use IO::Select;1") {
			$opt{set}{exec_open3}=0;
			DEBUG2("process_parameters: IO::Select not available");
		}
		DEBUG2("process_parameters: IPC::Open3 and IO::Select modules loaded");
	}
	#--- if no module XML::Simple available - set var
	if ($opt{set}{persistent}) {
		$opt{set}{use_xml_simple} = 1;
		unless (eval "require XML::Simple;1") {
			$opt{set}{use_xml_simple} = 0;
			DEBUG1("process_parameters: XML::Simple not available:$@");
		}
	}
	#--- set libexec-dir at the beginning of PATH var
	if ($opt{set}{libexec}) {
		$ENV{PATH}="$opt{set}{libexec}:$ENV{PATH}";
	}
	if ($opt{ok}) {
		$rc{expr}{$OK}=$opt{ok};
	}
	if ($opt{warning}) {
		$rc{expr}{$WARNING}=$opt{warning};
	}
	if ($opt{critical}) {
		$rc{expr}{$CRITICAL}=$opt{critical};
	}
	if ($opt{unknown}) {
		$rc{expr}{$UNKNOWN}=$opt{unknown};
	}
	if ($opt{set}{timeout} && $opt{set}{TIMEOUT} && $opt{set}{timeout} > $opt{set}{TIMEOUT}) {
		print "$MYSELF: error - child timeout $opt{set}{timeout}s must not be greater than parent timeout $opt{set}{TIMEOUT}s\n";
		return $UNKNOWN;
	}
	foreach my $option (sort keys(%{$opt{set}})) {
		$ENV{"MULTI_".$option}=$opt{set}{$option};
	}
	if (!defined($ENV{"MULTI_PPID"})) {
		$ENV{"MULTI_PPID"}=$$;
	}
	foreach my $option (sort keys(%opt)) {
		DEBUG3("process_parameters: \$opt{$option}=$opt{$option}") if (defined($opt{$option}));
	}
	foreach my $option (sort keys(%{$opt{set}})) {
		DEBUG3("process_parameters: \$opt{set}{$option}=$opt{set}{$option}") if (defined($opt{set}{$option}));
	}
	#mkdir "$opt{set}{tmp_dir}", $opt{set}{tmp_dir_permissions} if (! -d "$opt{set}{tmp_dir}"); 
	if (! -d "$opt{set}{tmp_dir}") {
		mkdir "$opt{set}{tmp_dir}";
		#chmod oct($opt{set}{tmp_dir_permissions}), "$opt{set}{tmp_dir}";
	}
	if (! -d "$opt{set}{tmp_dir}") {
		print "Error: could not create tmp directory $opt{set}{tmp_dir} as user " . getpwuid($<) . "\n";
		return $UNKNOWN;
	} elsif (! -w "$opt{set}{tmp_dir}") {
		print "Error: cannot write to tmp directory $opt{set}{tmp_dir} as user " . getpwuid($<) . "\n";
		return $UNKNOWN;
	}
	if (@ARGV) {
		print "Error: orphaned parameters found on command line:";
		for (my $i=1; $#ARGV>-1; $i++) {
			print " ARG$i:",shift(@ARGV);
		}
		print "\n";
		return $UNKNOWN;
	}
	return $OK;
}

sub short_usage {
print <<SHORTEOF;
$MYSELF -f <config file> [-n name] [-t timeout] [-T TIMEOUT] 
	[-r level] [-l libexec_path] [-s option=value]
	[-x command] [-w <expr>] [-c <expr>] [-u <expr>] [-o <expr>]
$MYSELF [-h | --help]
$MYSELF [-v | --verbose]
$MYSELF [-V | --version]

[ more infos on http://my-plugin.de/check_multi ]

SHORTEOF
}

sub long_usage {
print <<LONGEOF;
Options:
-f, --filename
   config file which contains commands to be executed
   multiple files can be specified serially
   if filename is a directory, all '.cmd' files will be taken
   (file format follows nrpe style: command[tag]=plugin command line)
-n, --name
   multi plugin name (shown in output), default: $opt{set}{name}
-t, --timeout
   timeout for one command, default: $opt{set}{timeout}
-T, --TIMEOUT
   TIMEOUT for all commands, default: $opt{set}{TIMEOUT}
-r, --report <level>
   specify level of details in output (level is binary coded, just sum up all options)
   default: $opt{set}{report}
        1: mention service names in plugin_output, e.g.
           "24 plugins checked, 1 critical (http), 0 warning, 0 unknown, 23 ok"
        2: add HTML coloring of output for extinfo
        4: show STDERR (if any) in each line of plugin output
        8: show performance data (with check_multi_style)
       16: show full list of states, normally '0 warning' is omitted
       32: show old type of performance data (without check_multi style)
       64: add explicit status (OK,WARNING,CRITICAL,UNKNOWN) in front of output
      128: add action link if performance data available
      256: XML: print structured XML output
      512: Nagios 2 compatibility: one summary line of output only
     1024: show notes_url
-s, --set <option>=<value>
   action_url=<URL> - see Nagios action_url
   ignore_missing_cmd_file=(0|1) (don't complain if config file missing)	
   notes_url=<URL> - see Nagios notes_url
   suppress_perfdata=<tag1>[,<tag2>][,...] (don't provide perfdata)
   target=<target> - frame target for action_url and notes_url
   tmp_dir=<directory> - path for temporary files
-l, --libexec
   path to plugins, default: $opt{set}{libexec}
-h, --help
   print detailed help screen
-v, --verbose
   prints debug output (multiple -v extend debug level) 
-V, --version
   print version information

Extended mode - specify extra settings for threshold interpretation:
-x, --execute "command [ tag ] = check_xyz"
-w, --warning  <expression>
-c, --critical <expression>
-u, --unknown  <expression>
-o, --ok       <expression>
LONGEOF

}

#---
#--- numerical sort
#---
sub numerically { $a <=> $b }

#---
#--- trim input string if found any chars from trim string
#---
sub mytrim {
	my ($src, $trim)=@_;
	DEBUG3("mytrim: src:\'$src\' trim:\'$trim\'");
	return ($src=~/[$trim]*(.*)[$trim]*/) ? $1 : $src;
}

#---
#--- substitute macros a la $HOSTNAME$ from environment
#---
sub substitute_macros {
	my ($input)=@_;
	DEBUG3("substitute_macros: checking $input");
	while ((my $var)=($input=~/\$([A-Za-z0-9_^\$]+)\$/)) {
		#--- 1. check for MULTI var
		if (defined($ENV{"MULTI_${var}"})) {
			DEBUG3("substitute_macros: replacing env var MULTI_${var} with $ENV{\"MULTI_${var}\"}");
			$input=~s/\$$var\$/$ENV{"MULTI_${var}"}/g;
		#--- 2. check for NAGIOS var
		} elsif (defined($ENV{"NAGIOS_${var}"})) {
			DEBUG3("substitute_macros: replacing env var NAGIOS_${var} with $ENV{\"NAGIOS_${var}\"}");
			$input=~s/\$$var\$/$ENV{"NAGIOS_${var}"}/g;
		#--- 3. nothing to substitute
		} else {
			$input=~s/\$$var\$//g;
			DEBUG3("substitute_macros: var NAGIOS_${var}/MULTI_${var} not defined: removed \$$var\$");
		}
	}
	return $input;
}

#---
#--- print debug message (see Macro DEBUG)
#---
sub debug_message {
	foreach (@_) {
		print "$_\n";
	}	
}

#---
#--- install signal handlers
#--- 
sub install_signal_handler {
	my ($handler, @signals)=@_;
	foreach my $signal (@signals) {
		$SIG{$signal} = \&signal_handler;
	}
}

#---
#--- got signal? report what we have and terminate savely
#--- 
sub signal_handler {
	my $signal=$_[0];

	#--- reinstall signal_handler (just paranoid ;)
	install_signal_handler(\install_signal_handler, $signal);

	#--- do reports before quitting
	add_error("Got signal $signal");
	&result_rating;
	&report_all;

	#--- cleanup end exit
	unlink $tmp_stdout, $tmp_stderr if (!$opt{set}{exec_open3});
	exit $rc{top};
}

#---
#--- add error(s) to global error list
#---
sub add_error {
	push @{$cmds{0}{error}}, @_;
	DEBUG2("add_error: added error message \'@_\'");
}

#---
#--- create unique tmpfile and try to create it
#---
sub get_tmpfile {
	my ($path,$prefix)=@_;
	my $attempt=0;
	my $tmpfile="";
	#--- check existance of path and create it if necessary
	if (! -d $path && ! mkdir($path,0700)) {
		add_error("get_tmpfile: error creating tmp_path $path:$!");
		return "";
	}
	#--- do 5 attempts to create tmpfile
	while ($attempt++ < 5) {
		my $suffix=int(rand(89999))+10000;
		$tmpfile="$path/$prefix.$suffix";
		next if (-f $tmpfile);
		if (open(TMP,">$tmpfile")) {
			close TMP;
			DEBUG3("get_tmpfile: created $tmpfile");
			return $tmpfile;
		}
	}
	add_error("get_tmpfile: giving up opening $tmpfile after $attempt attempts:$!");
	return "";
}

#---
#--- read file and return its contents
#---
sub readfile {
	my ($filename)=@_;
	open(FILE,$filename) || add_error("readfile: error opening $filename:$!") && return "";
	my @lines=<FILE>;
	close(FILE);
	return join("", @lines);
}

#---
#--- writes file and return its contents
#---
sub writefile {
	my ($filename,@lines)=@_;
	open(FILE,"$filename") || add_error("writefile: error opening $filename:$!") && return "";
	print FILE @lines;
	close(FILE);
	return join("", @lines);
}

#---
#--- parse command file and call line parser
#---
sub parse_files {
	my ($filenames)=@_;	# allow multiple filenames (array reference)
	my (@lines)=();

	#--- loop over filenames
	foreach my $filename (@{$filenames}) {

		if (!open(FILE, $filename)) {
			next if ($opt{set}{ignore_missing_cmd_file});
			add_error("parse_files: cannot open config file $filename: $!");
			$rc{top}=$UNKNOWN;		# top RC to UNKNOWN
			$rc{expr}{$OK}="0==1";	# OK: false
			next;
		} else {
			@lines=<FILE>;
			push @lines, ""; # add empty line to avoid last line eval problem
			close FILE;
		}
		DEBUG2("-" x 80);
		DEBUG2("parse_files: parsing file $filename with $#lines lines");
		DEBUG2("-" x 80);
	
		parse_lines(@lines);
	}
	return scalar(keys(%cmds));
}

#---
#--- parse command array and fill %cmds structure
#---
sub parse_lines {
	my @lines=@_;
	my ($line,$cmd,$type,$name,$plugin,$pplugin,$expr,$lineno,$i);

	$lineno=0;	# count sourcefile lines
	while ($line=shift(@lines)) {
		chomp($line);
		#--- remove unwanted characters from input line
		$line=~s/$opt{set}{illegal_chars}//g;
		$lineno++;		

		#--- check for continuation line with trailing '\'
		if ($line =~ s/\\\s*$//) { 
			$line .= shift(@lines); # add next line
			redo unless ($#lines<=0);
		}
		next if ($line=~/^\s*$/);	# skip empty lines
		next if ($line=~/^\s*#/);	# skip comments
	
		#--- format: 'command [ tag[::plugin] ] = plugin command line'
		#--- format: 'eval    [ tag[::plugin] ] = expression'
		#--- format: 'eeval   [ tag[::plugin] ] = expression'
		if ($line=~/\s*(command|eval|eeval)\s*\[\s*([^:\s]+)[:]*(\S*)\s*\]\s*=\s*(.*)\s*/i) {
			$type=lc($1);
			$name=$2;
			$pplugin=$3;
			$cmd=$4;
			$plugin=($type eq "command") ? (split(/\s+/,"$cmd"))[0] : "eval";

			#--- invalid characters found?
			$name=substitute_macros($name);
			if ($name!~/^[\w:-]+$/) {
				if ($name=~/\$([^\$]+)\$/) {
					add_error("parse_lines: tag $name invalid - macro \$$1\$ not found");
				} else {
					add_error("parse_lines: tag $name invalid - using non allowed characters");
				}
				next;
			}

			#--- overloading: find plugin with same name
			if ($type eq "command" || $type=~/eval$/) {
				#MF printf STDERR "before %d\n", scalar(keys(%cmds));
				for ($i=1; $i<scalar(keys(%cmds));$i++) {
					last if ($cmds{$i}{name} eq $name);
				}
				#MF printf STDERR "after %d\n", scalar(keys(%cmds));
				if ($cmds{$i}{command}) {
					DEBUG2("parse_lines: overloaded $name with $type \'$cmd\'");
				} else {
					DEBUG2("parse_lines: added $name with $type \'$cmd\'");
				}
			}
			#--- store vars into cmds structure
			$cmds{$i}{command}=$cmd;
			$cmds{$i}{type}=$type;
			$cmds{$i}{plugin}=$plugin;
			$cmds{$i}{pplugin}=$pplugin;
			$cmds{$i}{name}=$name;
			$cmds{$i}{rc}=$UNKNOWN;
			$cmds{$i}{output}="";
			$cmds{$i}{error}="";
			$cmds{$i}{runtime}=0;
			if ($opt{set}{suppress_perfdata} && 
			    $opt{set}{suppress_perfdata}=~/\b$name\b/i) {
				$cmds{$i}{process_perfdata}=0;
				DEBUG2("parse_lines: perfdata of [ $name ] will be suppressed");
			} else {
				$cmds{$i}{process_perfdata}=1;
			}
			DEBUG2("parse_lines: parsed $type [ $name ] = $cmd");

		#--- format: 'state [ tag ] = (EXPR)
		} elsif ($line=~/\s*state\s*\[\s*(\S+)\s*\]\s*=\s*(.*)\s*/i) {
			$name=uc($1);
			$expr=$2;
			if (!defined($def{code}{$name})) {
				add_error("parse_lines: invalid state code specified in line $lineno: $name");
				next;
			}
			# store state expression only if NOT set via commandline
			if (!defined($opt{lc($name)})) {
				DEBUG2("parse_lines: added state{$def{code}{$name}} expression: $expr");
				$rc{expr}{$def{code}{$name}}=$expr; 
			} else {
				DEBUG2("parse_lines: cmdline precedence over state expression. Using \$opt{$name}: $opt{$name}");
			}
		#--- format: 'output = (EXPR)
		} elsif ($line=~/\s*output\s*\[\s*(\S+)\s*\]\s*=\s*(.*)\s*/i) {
			$name=$1;
			$expr=$2;
			my $found=0;
			for ($i=0; $found==0 && $i<=scalar(keys(%cmds));$i++) {
				if ($cmds{$i}{name} eq $name) {
					$cmds{$i}{fmtstr}=$expr;
					$found=1;
				}
			}
			if (!$found) {
				add_error("parse_lines: invalid tag name specified in line $lineno: $name");
				next;
			}
		#--- not yet matched? -> invalid format
		} else {
			add_error("parse_lines: invalid format in line $lineno: $line");
			next;
		}
	}
}

#---
#--- execute command number $no from %cmds
#---
sub exec_command {
	my ($no)=@_;

	#--- start with proper RC
	$?=0;

	#--- at runtime: substitute $MACRO$ macros and states
	$cmds{$no}{command}=substitute_macros($cmds{$no}{command});

	#--- measure command runtime;
	my $cmd_start=time;
	$ENV{"MULTI_${no}_NAME"}=$cmds{$no}{name};

	if ($cmds{$no}{type} eq "command") {

		eval {
			alarm($opt{set}{timeout});
	
			#--- classic exec with temporary files
			if (!$opt{set}{exec_open3}) {

				#--- prepare tmpfiles for stdout and stderr
				$tmp_stdout=&get_tmpfile("$opt{set}{tmp_dir}", "${MYSELF}_stdout_$$");
				$tmp_stderr=&get_tmpfile("$opt{set}{tmp_dir}", "${MYSELF}_stderr_$$");
	
				#--- execute command and store stdout/stderr/return code
				if (!defined(my $child=fork())) {
    					die "cannot fork: $!";
				} elsif ($child==0) {
					exec("$cmds{$no}{command} 1>$tmp_stdout 2>$tmp_stderr");
				} else { 
					$SIG{'ALRM'} = sub { 
						add_error("$cmds{$no}{name}: timeout after $opt{set}{timeout}s"); 
						kill -15,$child; 
						sleep 1; 
						kill -9,$child; 
					};
					alarm($opt{set}{timeout});
					waitpid($child,0);
					alarm(0);
				}
				$cmds{$no}{rc}=$? >> 8;
				$ENV{"MULTI_${no}_RC"}=$cmds{$no}{rc};
	
				#--- store stdout/stderr and cleanup tmpfiles
				$cmds{$no}{output}=readfile($tmp_stdout);
				DEBUG2("exec_command: raw output >$cmds{$no}{output}<");
				$cmds{$no}{error}=readfile($tmp_stderr);
				DEBUG2("exec_command: raw stderr >$cmds{$no}{error}<");
				unlink $tmp_stdout, $tmp_stderr;
				$cmds{$no}{output}=~s/[$opt{set}{illegal_chars}]*//g; chomp $cmds{$no}{output};	
				$cmds{$no}{error}=~s/[$opt{set}{illegal_chars}]*//g; chomp $cmds{$no}{error};

			#--- new open3 exec (to be tested carefully before getting standard ;-))
			} else {
				my $pid=open3(*CIN,*COUT,*CERR,$cmds{$no}{command});
				close(CIN); # not needed: close STDIN filehandle
				$SIG{CHLD}=sub {
					$cmds{$no}{rc}=$?>>8 if waitpid($pid, 0) > 0;
					DEBUG2("REAPER: status $cmds{$no}{rc} on $pid");
				};

				my $sel=IO::Select->new();
				$sel->add(*CERR,*COUT);
				while (my @ready = $sel->can_read) {
					foreach my $fh (@ready) {
						if (fileno($fh) == fileno(CERR)) {
							$cmds{$no}{error}.=scalar <CERR>; 
						} else {
							$cmds{$no}{output}.=scalar <COUT>; 
						}
						$sel->remove($fh) if eof($fh);
					}
				}
				close(COUT);
				close(CERR);
				DEBUG2("exec_command: open3 raw output >$cmds{$no}{output}<");
				DEBUG2("exec_command: open3 raw stderr >$cmds{$no}{error}<");
				chomp $cmds{$no}{output};	
				chomp $cmds{$no}{error};
			}	
			#--- unknown return code? change it explicitly to UNKNOWN
			if (! defined($def{r2s}{$cmds{$no}{rc}})) {
				$cmds{$no}{error}.=" RC was $cmds{$no}{rc}!";
				$cmds{$no}{rc}=$UNKNOWN;
			}
			
			#--- remove white chars from output
			#$cmds{$no}{error}=~s/[\s]+/ /g;
			#DEBUG2("exec_command: raw stderr >$cmds{$no}{error}<");
			$cmds{$no}{stdout}=HTML::Entities::encode_entities($cmds{$no}{stdout}) if ($opt{set}{use_html_entities} && $cmds{$no}{stdout});
			$cmds{$no}{error}=HTML::Entities::encode_entities($cmds{$no}{error}) if ($opt{set}{use_html_entities} && $cmds{$no}{error});
	
			#--- split performance data from standard output
			if ($cmds{$no}{output}=~/\|/) {
				DEBUG3("exec_command(1): output contains \|");
				if ($cmds{$no}{output}=~/([^\n]*)\n(.*)/s) {
					DEBUG3("exec_command(2): output contains \\n");
					my $rest=$2;
					DEBUG3("exec_command(3): \$1:$1");
					DEBUG3("exec_command(4): \$2:$2");
					($cmds{$no}{output},$cmds{$no}{performance})=split(/\|/,$1);
					DEBUG3("exec_command(5): rest:>$rest<");
					if ($rest=~/(.*)\|(.*)/s) {
						$cmds{$no}{output}.="\n$1";
						$cmds{$no}{performance}.=" " . join(' ', split(/\n/,$2));
					} else {
						$cmds{$no}{output}.="\n$rest";
					}
					DEBUG3("exec_command(6): output:$cmds{$no}{output} performance:$cmds{$no}{performance}");
				} else {
					($cmds{$no}{output},$cmds{$no}{performance})=split(/\|/,$cmds{$no}{output});
					DEBUG3("exec_command performance string: $cmds{$no}{performance}");
					$cmds{$no}{performance}=mytrim($cmds{$no}{performance},"\\s");
				}

				#--- check performance data and suppress if found errors
				if (($opt{set}{report} & $DETAIL_PERFORMANCE) && $cmds{$no}{process_perfdata}) {
					if (my $errstr=error_in_perfdata($cmds{$no}{performance})) {
						add_error("$cmds{$no}{name} perfdata discarded for $errstr");
						$cmds{$no}{performance}=undef;
					}
				}
			}
	
			alarm(0);
		};
	} elsif ($cmds{$no}{type} eq "eval" || $cmds{$no}{type} eq "eeval") {
		alarm($opt{set}{timeout});
		$cmds{$no}{output}=eval($cmds{$no}{command});
		if ($cmds{$no}{type} eq "eeval") {
			$cmds{$no}{rc}=$?>>8;
         		#--- unknown return code? change it explicitly to UNKNOWN
         		if (! defined($def{r2s}{$cmds{$no}{rc}})) {
            			$cmds{$no}{error}.=" RC was $cmds{$no}{rc}!";
            			$cmds{$no}{rc}=$UNKNOWN;
         		}
		} elsif ($cmds{$no}{type} eq "eval") {
			$cmds{$no}{rc}=$OK;
		}
		if ($@) {
			$cmds{$no}{output}.="[$@]";
			$cmds{$no}{rc}=$WARNING;
		} else {
			chomp($cmds{$no}{output});
			$ENV{"MULTI_".$cmds{$no}{name}}="$cmds{$no}{output}";
			#--- split performance data from standard output
			if ($cmds{$no}{output}=~/\|/) {
				($cmds{$no}{output},$cmds{$no}{performance})=split(/\|/,$cmds{$no}{output});
				$cmds{$no}{performance}=mytrim($cmds{$no}{performance},"\\s");

				#--- check performance data and suppress if found errors
				if (($opt{set}{report} & $DETAIL_PERFORMANCE) && $cmds{$no}{process_perfdata}) {
					if (my $errstr=error_in_perfdata($cmds{$no}{performance})) {
						add_error("$cmds{$no}{name} perfdata discarded for $errstr");
						$cmds{$no}{performance}=undef;
					}
				}
			}
			DEBUG2("exec_command: environment var MULTI_$cmds{$no}{name}=$cmds{$no}{output}");
		}
		$cmds{$no}{runtime}=time-$cmd_start;
		alarm(0);
		return $cmds{$no}{rc};
	}
	$cmds{$no}{runtime}=time-$cmd_start;
	
	#--- any oddities during command execution?
	if ($@) {
		#--- timeout encountered: store status
		if ($@ =~ /timeout/) {
			$cmds{$no}{output}="UNKNOWN - $cmds{$no}{plugin} cancelled after timeout ($opt{set}{timeout}s)";
			$cmds{$no}{rc}=$UNKNOWN;
			$cmds{$no}{output}.=readfile($tmp_stdout);
			$cmds{$no}{error}.=readfile($tmp_stderr);
		#--- catchall for unknown errors
		} else {
			alarm(0);
			$cmds{$no}{rc}=$UNKNOWN;
			add_error("unexpected exception encountered:$@");
		}
		unlink $tmp_stdout, $tmp_stderr;
	} else {
		$ENV{"MULTI_".$cmds{$no}{name}}="$cmds{$no}{output}";
		DEBUG2("exec_command: environment var MULTI_$cmds{$no}{name}=$cmds{$no}{output}");
		$ENV{"MULTI_STATE_".$cmds{$no}{name}}="$cmds{$no}{rc}";
		DEBUG2("exec_command: environment var MULTI_STATE_$cmds{$no}{name}=$cmds{$no}{rc}");
	}
	return $cmds{$no}{rc};
}

sub eval_result {
	my ($input)=@_;
	my $input_org=$input;
	my $message="";

 	#--- at runtime: substitute $MACRO$ macros and $STATES$
	$input=substitute_macros($input);
	$input=substitute_states($input);

	#--- evaluate expression
	my $result=eval "($input)";

	#--- catch error
	if ($@) {
		$message="Evaluation error in \'$input_org\': $@\n";
		$message=~s/\n/ /g;
		return (-1,$message);
	#--- return result
	} else {
		$message="eval_result: input:\'$input_org\' parsed:\'$input\' result:\'$result\'\n";
		$message=~s/\n/ /g;
		DEBUG2($message);
		return ($result,$message);
	}
}

sub substitute_states {
	my ($input)=@_;
	#--- 1. replace COUNT(WARNING)
	$input=~s/\bCOUNT\s*\(\s*(OK)\s*\)/$rc{count}{$OK}/ig;
	$input=~s/\bCOUNT\s*\(\s*(WARNING)\s*\)/$rc{count}{$WARNING}/ig;
	$input=~s/\bCOUNT\s*\(\s*(CRITICAL)\s*\)/$rc{count}{$CRITICAL}/ig;
	$input=~s/\bCOUNT\s*\(\s*(UNKNOWN)\s*\)/$rc{count}{$UNKNOWN}/ig;

	#--- 2. replace all STATES (OK)
	$input=~s/\b(OK)\b/$OK/ig;
	$input=~s/\b(WARNING)\b/$WARNING/ig;
	$input=~s/\b(CRITICAL)\b/$CRITICAL/ig;
	$input=~s/\b(UNKNOWN)\b/$UNKNOWN/ig;

	#--- 3. replace all vars with RC
	for ($no=1;$no<keys(%cmds);$no++) {
		$input=~s/\b($cmds{$no}{name})\b/$cmds{$no}{rc}/ig;
	}
	#--- 4. replace IGNORE
	$input=~s/\b(IGNORE)\b/(0==1)/ig;
	return $input;
}

#---
#--- calculate sums from %cmds and %rc
#---
sub result_rating {

	#--- measure runtime without reporting ;-)
	$cmds{0}{runtime}=time - $cmds{0}{starttime};
	$cmds{0}{nchecks}=scalar(keys(%cmds))-1;

	#--- count return codes
	for ($no=1;$no<keys(%cmds);$no++) {
		$rc{count}{$cmds{$no}{rc}}++;	# count return codes
		push @{$rc{list}{$cmds{$no}{rc}}},$cmds{$no}{name}; # add plugin to list
	}
	
	foreach my $s (sort numerically keys %{$def{s2r}}) {
		$ENV{"MULTI_$def{label}{$s}_COUNT"}=$rc{count}{$s};
		$ENV{"MULTI_$def{label}{$s}_LIST"}=join(',',@{$rc{list}{$s}});
		
		my $state=$def{s2r}{$s};
		
		my ($result,$message)=eval_result($rc{expr}{$state});
		if (! defined($result) || $result eq "") {
			; # do nothing
		} elsif ($result < 0) {
			add_error("result_rating: parsing error ($message)");
		} else {
			$rc{match}{$state}=1;
			$rc{top}=$state;
		}
	}
}

#---
#--- start different report routines
#---
sub report_all {

	#--- some debugging first
	DEBUG3("MULTI Environment (sorted):\n" . `env | grep '^MULTI' | sort | sed 's/^/   /g'`);
	DEBUG3("NAGIOS Environment (sorted):\n" . `env | grep '^NAGIOS' | sort | sed 's/^/   /g'`);

	#--- construction site for persistence
	if ($opt{set}{test} && $opt{set}{persistent}) {
                unless (eval "use Data::Dumper;1") {
                        $opt{set}{test}=0;
                        DEBUG2("report_all: Data::Dumper not available");
                } else {
                	DEBUG2("report_all: Data::Dumper module loaded");
			DEBUG2("report_all:" . Dumper(\%cmds));
		}

		writefile(">$opt{set}{tmp_dir}/test$$.xml", XML::Simple::XMLout(
			\%cmds,
			KeyAttr=>"no",
			NoAttr=>1,
			RootName=>"cmds",
		));
		writefile(">>$opt{set}{tmp_dir}/test$$.xml", XML::Simple::XMLout(
			\%rc,
			KeyAttr=>"no",
			NoAttr=>1,
			KeepRoot=>1,
			RootName=>"rc",
		));
	}

	#--- print service definition
	if ($opt{set}{report} & $DETAIL_SERVICE_DEFINITION) {
		&report_service_definition;
		return; # 	no normal output here
	}

	#--- classical report
	if (! ($opt{set}{report} & $DETAIL_HTML) &&
	    ! ($opt{set}{report} & $DETAIL_XML)) {
		&report;
	}
	#--- report HTML output
	if ($opt{set}{report} & $DETAIL_HTML) {
		&report_html;
	} 
	#--- report XML output 
	if ($opt{set}{report} & $DETAIL_XML) {
		&report_xml;
	}
	
	
	#--- at last: perfdata
	&report_perfdata;
	
	#--- final '\n' - dedicated to Wolfgang Barth ;-)
	if (	!($opt{set}{report} & $DETAIL_NAGIOS2) && 
		!($opt{set}{report} & $DETAIL_HTML) &&
		!($opt{set}{report} & $DETAIL_XML)) {
		print "\n";
	}
}

#---
#--- report results stored in %cmds (ASCII report)
#---
sub report {

	DEBUG1("\n","-" x 80);
	DEBUG1("Plugin output");
	DEBUG1("-" x 80);
	
	if ($opt{set}{report} & $DETAIL_NAGIOS2) {
		print "$opt{set}{name} " if $opt{set}{name};
		print "$def{label}{$rc{top}}";
	} else {
		#--- print header line (1): name, state, number of plugins
		print "$opt{set}{name} " if $opt{set}{name};
		print "$def{label}{$rc{top}} - $cmds{0}{nchecks} plugins checked, ";
	
		#--- print header line (2): summary for particular states 
		if ($opt{set}{report} & $DETAIL_LIST_FULL) {
			print	"$rc{count}{$CRITICAL} critical" . ((@{$rc{list}{$CRITICAL}}) ? " (" . join(', ',@{$rc{list}{$CRITICAL}}) . ')' : "") . ", " .
				"$rc{count}{$WARNING} warning"   . ((@{$rc{list}{$WARNING}})  ? " (" . join(', ',@{$rc{list}{$WARNING}})  . ')' : "") . ", " .
				"$rc{count}{$UNKNOWN} unknown"   . ((@{$rc{list}{$UNKNOWN}})  ? " (" . join(', ',@{$rc{list}{$UNKNOWN}})  . ')' : "") . ", " .
				"$rc{count}{$OK} ok";
		} elsif ($opt{set}{report} & $DETAIL_LIST) {
			my @r=();
			push @r, "$rc{count}{$CRITICAL} critical (" . join(', ',@{$rc{list}{$CRITICAL}}) . ")" if (@{$rc{list}{$CRITICAL}});
			push @r, "$rc{count}{$WARNING} warning ("   . join(', ',@{$rc{list}{$WARNING}})  . ")" if (@{$rc{list}{$WARNING}});
			push @r, "$rc{count}{$UNKNOWN} unknown ("   . join(', ',@{$rc{list}{$UNKNOWN}})  . ")" if (@{$rc{list}{$UNKNOWN}});
			push @r, "$rc{count}{$OK} ok" if (@{$rc{list}{$OK}});
			print join(", ", @r);
		} else {
			print	"$rc{count}{$CRITICAL} critical, " .
				"$rc{count}{$WARNING} warning, " .
				"$rc{count}{$UNKNOWN} unknown, " .
				"$rc{count}{$OK} ok";
		}
	}
	#--- print general errors if any occured
	print " [" . join(", ",@{$cmds{0}{error}}) . "]" if (defined($cmds{0}{error}[0]));
	
	#--- loop over commands: report particular results for long plugin output
	for ($no=1;$no<keys(%cmds);$no++) {

		#--- skip eval
		next if ($cmds{$no}{type} eq "eval");

		#--- if NAGIOS2 output: skip $OK results
		if ($opt{set}{report} & $DETAIL_NAGIOS2) {
			next if ($cmds{$no}{rc} == $OK);
			$cmds{$no}{output}=~s/\n//g;
			$cmds{$no}{error}=~s/\n//g;
			printf ", %s %s%s",
				$cmds{$no}{name},
				$cmds{$no}{output},
				(defined($cmds{$no}{error}) && $cmds{$no}{error} ne "" && ($opt{set}{report} & $DETAIL_STDERR)) ?
			 	" [ STDERR: " . $cmds{$no}{error} . ']' : "";
		} else {
			$cmds{$no}{output}=~s/\n/\n$opt{set}{indent}/g;
			printf "%s[%2.d] %s %s%s%s",
				($opt{set}{report} & $DETAIL_NAGIOS2) ? ", " : "\n",
				$no,
				$cmds{$no}{name},
				($opt{set}{report} & $DETAIL_STATUS) ? "$def{label}{$cmds{$no}{rc}} " : "",
				($cmds{$no}{output}=~/\<!--(.*?)--\>/sg) ? $1 : $cmds{$no}{output},
				(defined($cmds{$no}{error}) && $cmds{$no}{error} ne "" && ($opt{set}{report} & $DETAIL_STDERR)) ?
			 	" [ STDERR: " . $cmds{$no}{error} . ']' : "";
		}
	}

	my $maxcmdlen=0;
	for ($no=1;$no<keys(%cmds);$no++) {
		$maxcmdlen=length($cmds{$no}{name}) if (length($cmds{$no}{name})>$maxcmdlen);
	}

	#--- print results
	DEBUG1("\n","-" x 80);
	DEBUG1(sprintf("No   Name%sRuntime  RC Output", ' ' x ($maxcmdlen-3)));
	DEBUG1("-" x 80);
	for ($no=1;$no<keys(%cmds);$no++) {
		DEBUG1(sprintf "[%2.d] %-${maxcmdlen}s %6.4fs %3d %s", $no, $cmds{$no}{name}, $cmds{$no}{runtime}, $cmds{$no}{rc}, $cmds{$no}{output});
		DEBUG2(sprintf "%s%-12s%s", ' ' x ($maxcmdlen+6), $cmds{$no}{type}, $cmds{$no}{command});
	}

	#--- print state settings and RC evaluation result
	DEBUG1("\n","-" x 80);
	DEBUG1(sprintf "%-8s %-55s %s", "State","Expression","Evaluates to");
	DEBUG1("-" x 80);
	foreach my $s (sort numerically keys %{$def{s2r}}) {
		DEBUG1(sprintf "%-8s %-55s %s", $def{label}{$def{s2r}{$s}}, $rc{expr}{$def{s2r}{$s}}, ($rc{match}{$def{s2r}{$s}}) ? "TRUE" : "FALSE");
	}
	DEBUG1("-" x 80);
	DEBUG1(sprintf "%8s %55s %s", "", "Overall state =>", $def{label}{$rc{top}});
	DEBUG1("-" x 80);
}

sub report_html {
	my $output="";
	
	#--- print header line (1): name, state, number of plugins
	$output.="$opt{set}{name} " if $opt{set}{name};
	$output.="$def{label}{$rc{top}} - $cmds{0}{nchecks} plugins checked, ";

	#--- print header line (2): summary for particular states 
	if ($opt{set}{report} & $DETAIL_LIST_FULL) {
		$output.="$rc{count}{$CRITICAL} critical" . ((@{$rc{list}{$CRITICAL}}) ? " (" . join(', ',@{$rc{list}{$CRITICAL}}) . ')' : "") . ", " .
			"$rc{count}{$WARNING} warning"   . ((@{$rc{list}{$WARNING}})  ? " (" . join(', ',@{$rc{list}{$WARNING}})  . ')' : "") . ", " .
			"$rc{count}{$UNKNOWN} unknown"   . ((@{$rc{list}{$UNKNOWN}})  ? " (" . join(', ',@{$rc{list}{$UNKNOWN}})  . ')' : "") . ", " .
			"$rc{count}{$OK} ok";
	} elsif ($opt{set}{report} & $DETAIL_LIST) {
		my @r=();
		push @r, "$rc{count}{$CRITICAL} critical (" . join(', ',@{$rc{list}{$CRITICAL}}) . ")" if (@{$rc{list}{$CRITICAL}});
		push @r, "$rc{count}{$WARNING} warning ("   . join(', ',@{$rc{list}{$WARNING}})  . ")" if (@{$rc{list}{$WARNING}});
		push @r, "$rc{count}{$UNKNOWN} unknown ("   . join(', ',@{$rc{list}{$UNKNOWN}})  . ")" if (@{$rc{list}{$UNKNOWN}});
		push @r, "$rc{count}{$OK} ok" if (@{$rc{list}{$OK}});
		$output.=join(", ", @r);
	} else {
		$output.="$rc{count}{$CRITICAL} critical, " .
			"$rc{count}{$WARNING} warning, " .
			"$rc{count}{$UNKNOWN} unknown, " .
			"$rc{count}{$OK} ok";
	}

	#--- print general errors if any occured
	$output.=" [" . join(", ",@{$cmds{0}{error}}) . "]" if (defined($cmds{0}{error}[0]));
	$output.="\n";


	#--- loop over commands: report particular results for long plugin output
	if ($opt{set}{collapse} == 1) {
		$output.="<SCRIPT LANGUAGE='JavaScript'> function Toggle(node) { if (node.nextSibling.style.display == 'none') { if (node.childNodes.length > 0) { node.childNodes.item(0).replaceData(0,1,'-'); } node.nextSibling.style.display = 'block'; } else { if (node.childNodes.length > 0) { node.childNodes.item(0).replaceData(0,1,'+'); } node.nextSibling.style.display = 'none'; } } </SCRIPT>";
		if ($rc{top} == $OK && $ENV{"MULTI_PPID"} != $$) {
			$output.="<A onClick='Toggle(this)' style='line-height:0.1em;color:#4444FF;font-size:1.5em'>+</A>";
			$output.="<DIV style='display:none'>";
		} else {
			$output.="<A onClick='Toggle(this)' style='line-height:0.1em;color:#4444FF;font-size:1.5em'>-</A>";
			$output.="<DIV style='display:block'>";
		}
	}

	$output.="<table style='border-left-width:1px; border-left-style:dotted; border-right-width:0px;' id=multi_table>";

	for ($no=1;$no<keys(%cmds);$no++) {
		#--- skip eval
		next if ($cmds{$no}{type} eq "eval");

		#--- allow commands to get tag name
		$ENV{"MULTI_TAG"}=$cmds{$no}{name};

		$output.="<tr valign='top'>";
		$output.=sprintf "<td nowrap>%s&nbsp;%2.d&nbsp;%s</td>", 
			($opt{set}{report} & $DETAIL_HTML) ? "<DIV CLASS=\'service$def{label}{$cmds{$no}{rc}}\' style='font-size:7pt'>" : "",
			$no,
			($opt{set}{report} & $DETAIL_HTML) ? "</DIV>" : "";
		#--- Action url
		if (	($opt{set}{report} & $DETAIL_PERFORMANCE) &&
			($opt{set}{report} & $DETAIL_PERFORMANCE_LINK) && 
			defined($cmds{$no}{performance}) &&
			$cmds{$no}{process_perfdata}) {

			my $hostname=($ENV{NAGIOS_HOSTNAME}) ? $ENV{NAGIOS_HOSTNAME} : `uname -n`; chomp($hostname);
			DEBUG2("report_html: hostname is $hostname");
			$output.=sprintf "<td nowrap>%s</td>", 
				"<A HREF='/nagios/pnp/index.php?host=${hostname}&srv=$cmds{$no}{name}' TARGET=\'$opt{set}{target}\'>" . 
				"<img src='/nagios/images/action.gif' width=20 height=20 border=0 align=top alt='Show performance chart for $hostname / $cmds{$no}{plugin}'></A>";
		} else {
			$output.="<td></td>";
		}
		#--- Notes url
		if (	($opt{set}{report} & $DETAIL_NOTES_LINK) &&
			defined($opt{set}{notes_url}) &&
			!$opt{set}{tag_notes_link}) {
			my $notes_url=substitute_macros($opt{set}{notes_url});
			$output.=sprintf "<td nowrap>%s</td>", 
				"<A HREF=\'$notes_url\' TARGET=\'$opt{set}{target}\'>" . 
				"<img src='/nagios/images/notes.gif' width=20 height=20 border=0 align=top alt='Show notes for $cmds{$no}{plugin}'></A>";
		} else {
			$output.="<td></td>";
		}
		#--- and the rest...
		if (!$opt{set}{tag_notes_link}) {
			$output.=sprintf "<td CLASS='dataVal'>&nbsp;&nbsp;%s</td>", ($cmds{$no}{name}) ? $cmds{$no}{name} : "";
		} else {
			my $notes_url=substitute_macros($opt{set}{notes_url});
			$output.=sprintf "<td CLASS='dataVal'>&nbsp;&nbsp;<A HREF=\'$notes_url\' TARGET=\'$opt{set}{target}\'>%s</A></td>", ($cmds{$no}{name}) ? $cmds{$no}{name} : "";
		}
		DEBUG2("report_html output:$cmds{$no}{output}");
		DEBUG3("HTML check_multi recursive levels: " . scalar ($cmds{$no}{output}=~s/multi_table/multi_table/g));
		$output.=sprintf "<td CLASS='dataVal'>%s</td>", 
				($cmds{$no}{output}=~/^([^\n]+)\n(.*)/is) 
				? (($opt{set}{indent_label}) 
					? "$1</td></tr><tr><td colspan='4'></td><td colspan='1' CLASS='dataVal'>$2"
					: "$1</td></tr><tr><td></td><td colspan='5' CLASS='dataVal'>$2")
				: $cmds{$no}{output};
		if (	defined($cmds{$no}{error}) && 
			$cmds{$no}{error} ne "" && 
			($opt{set}{report} & $DETAIL_STDERR)) {

			$output.=" [ STDERR: " . join(' ',$cmds{$no}{error}) . "]";
		}
		$output.="</tr>";
	}
	$output.="</table>";
	$output.="</div>" if ($opt{set}{collapse} == 1);

	#--- print state evaluation if verbose flag set
        if ($opt{set}{verbose} >= 1) {
		if ($ENV{"MULTI_PPID"} != $$) {
			$output.="<A onClick='Toggle(this)' style='line-height:0.1em;color:#4444FF;font-size:1.5em'>+</A>";
			$output.="<DIV style='display:none'><h4>Child check overview</h4>";
		}
		my @colors=("silver","lightgrey");
		my $flipflop=0;
		$output.="<table><tr style='text-align:left' bgcolor='" . $colors[$flipflop] . "'><th>No</th><th>Name</th><th>Runtime</th><th>RC</th><th>Output</th></tr>";
		for ($no=1;$no<keys(%cmds);$no++) {
			$flipflop=!$flipflop;
			$output.=sprintf "<tr bgcolor='" . $colors[$flipflop] . "'><td>%d</td><td>%s</td><td>%7.5f</td><td>%d</td><td>%s</td></tr>", $no, $cmds{$no}{name}, $cmds{$no}{runtime}, $cmds{$no}{rc}, $cmds{$no}{output};
			$output.=sprintf "<tr bgcolor='" . $colors[$flipflop] . "'><td></td><td></td><td>%s</td><td></td><td>%s</td></tr>", $cmds{$no}{type}, $cmds{$no}{command};
		}
		$output.="</table></div>";

		#--- print state settings and RC evaluation result
		if ($ENV{"MULTI_PPID"} != $$) {
			$output.="<A onClick='Toggle(this)' style='line-height:0.1em;color:#4444FF;font-size:1.5em'>+</A>";
			$output.="<DIV style='display:none'><h4>State evaluation</h4>";
		}
		$flipflop=0;
		$output.="<table><tr style='text-align:left' bgcolor='" . $colors[$flipflop] . "'><th>State</th><th>Expression</th><th>Evaluates to</th></tr>";
		foreach my $s (sort numerically keys %{$def{s2r}}) {
			$flipflop=!$flipflop;
			$output.=sprintf "<tr bgcolor='" . $colors[$flipflop] . "'><td>%s</td><td>%s</td><td>%s</td></tr>", $def{label}{$def{s2r}{$s}}, $rc{expr}{$def{s2r}{$s}}, ($rc{match}{$def{s2r}{$s}}) ? "TRUE" : "FALSE";
		}
		$flipflop=!$flipflop;
		$output.="<tr bgcolor='" . $colors[$flipflop] . "'><td></td><td style='text-align:right'>Overall state</td><td style='font-weight:bold'>$def{label}{$rc{top}}</td></tr>";
		$output.="</table></div>";
	}
	if ($opt{set}{verbose} >=1) {

		if ($ENV{"MULTI_PPID"} != $$) {
			$output.="<A onClick='Toggle(this)' style='line-height:0.1em;color:#4444FF;font-size:1.5em'>+</A>";
			$output.="<DIV style='display:none'><h4>Child check details</h4>";
		}
		my @colors=("silver","lightgrey");
		my $flipflop=0;
		for ($no=1;$no<keys(%cmds);$no++) {
			$output.="<h5>$no - $cmds{$no}{name}</h5>";
			$output.="<table><tr style='text-align:left' bgcolor='" . $colors[$flipflop] . "'><th></th><th>Attribute</th><th>Value</th></tr>";
			my %vars=(
				1 => {
					att => "\$no",	
					val => $no, 
				},
				2 => {
					att => "tag",	
					val => $cmds{$no}{name},	 
				},
				3 => {
					att => "\$STATE_$cmds{$no}{name}\$",
					val => $cmds{$no}{rc},
				},
				4 => {
					att => "\$$cmds{$no}{name}\$",
					val => $cmds{$no}{output},	
				},
				5 => {
					att => "type",			
					val => $cmds{$no}{type},	
				},
				6 => {
					att => "command",		
					val => $cmds{$no}{command},	 
				},
				7 => {
					att => "runtime",		
					val => $cmds{$no}{runtime},	 
				},
			);

			foreach my $i (sort keys %vars) {
				$flipflop=!$flipflop;
				$output.=sprintf "<tr bgcolor='" . $colors[$flipflop] . "'><td>%d</td><td>%s</td><td>%s</td></tr>", $i, $vars{$i}{att}, $vars{$i}{val};
			}
			$output.="</table>";
		}
		$output.="</div>";
	}
	
	#--- last not least: output ;-)
	print $output;
}

sub report_xml {

	my $xmlstr="<div id=\"check_multi_xml\" style='display:none'>";
	#$xmlstr.="<meta http-equiv=\"Content-Style-Type\" content=\"application/xml\">\n";
	$xmlstr.="<?xml version=\"1.0\"?>\n";
	$xmlstr.="<?xml-stylesheet type=\"text/xsl\" href=\"extinfo.xsl\"?>\n";
	$xmlstr.="<PARENT>\n";
	$xmlstr.="\t<name>$cmds{0}{name}</name>\n";
	$xmlstr.="\t<plugins>$cmds{0}{nchecks}</plugins>\n";
	$xmlstr.="\t<time>$cmds{0}{runtime}</time>\n";
	$xmlstr.="\t<output>";
	#--- print header line (2): summary for particular states 
	if ($opt{set}{report} & $DETAIL_LIST_FULL) {
		$xmlstr.="$rc{count}{$CRITICAL} critical" . ((@{$rc{list}{$CRITICAL}}) ? " (" . join(', ',@{$rc{list}{$CRITICAL}}) . ')' : "") . ", " .
			"$rc{count}{$WARNING} warning"   . ((@{$rc{list}{$WARNING}})  ? " (" . join(', ',@{$rc{list}{$WARNING}})  . ')' : "") . ", " .
			"$rc{count}{$UNKNOWN} unknown"   . ((@{$rc{list}{$UNKNOWN}})  ? " (" . join(', ',@{$rc{list}{$UNKNOWN}})  . ')' : "") . ", " .
			"$rc{count}{$OK} ok";
	} elsif ($opt{set}{report} & $DETAIL_LIST) {
		my @r=();
		push @r, "$rc{count}{$CRITICAL} critical (" . join(', ',@{$rc{list}{$CRITICAL}}) . ")" if (@{$rc{list}{$CRITICAL}});
		push @r, "$rc{count}{$WARNING} warning ("   . join(', ',@{$rc{list}{$WARNING}})  . ")" if (@{$rc{list}{$WARNING}});
		push @r, "$rc{count}{$UNKNOWN} unknown ("   . join(', ',@{$rc{list}{$UNKNOWN}})  . ")" if (@{$rc{list}{$UNKNOWN}});
		push @r, "$rc{count}{$OK} ok" if (@{$rc{list}{$OK}});
		$xmlstr.=join(", ", @r);
	} else {
		$xmlstr.="$rc{count}{$CRITICAL} critical, " .
			"$rc{count}{$WARNING} warning, " .
			"$rc{count}{$UNKNOWN} unknown, " .
			"$rc{count}{$OK} ok";
	}
	$xmlstr.="</output>\n";
	$xmlstr.="\t<error>" . join(", ",@{$cmds{0}{error}}) . "</error>\n" if (defined($cmds{0}{error}[0]));

	for ($no=1;$no<keys(%cmds);$no++) {
		$xmlstr.="\t<CHILD>\n";
		$xmlstr.="\t\t<no>$no</no>\n";
		foreach my $token (split(/,/,$opt{set}{xml_elements})) {
			if (defined($cmds{$no}{$token})) {
				$xmlstr.=sprintf "\t\t<%s>%s</%s>\n",$token,$cmds{$no}{$token},$token;
			} else {
				add_error("report_xml: XML element $token not found");
			}
		}
		$xmlstr.="\t</CHILD>\n";
	}
	$xmlstr.="</PARENT>\n</div>\n";
	print $xmlstr;
}

sub report_perfdata {
	if (!$opt{set}{report} & $DETAIL_HTML) {
		DEBUG1("\n","-" x 80);
		DEBUG1("Plugin performance data");
		DEBUG1("-" x 80);
	}
	#--- report performance data?
	if ($opt{set}{report} & $DETAIL_PERFORMANCE) {
		my $perftmp="";
		DEBUG2("report_perfdata (DETAIL_PERFORMANCE): name=" . ($opt{set}{name}) ? $opt{set}{name} : $MYSELF);
		print "\|";
		printf "%s::%s::plugins=%d time=%f ", 
			($opt{set}{name}) ? $opt{set}{name} : $MYSELF, $MYSELF,
			scalar(keys(%cmds)),
			time - $cmds{0}{starttime};
		#--- one line per command, format: tag=output
		for ($no=1;$no<keys(%cmds);$no++) {

			#--- suppress_perfdata set? ignore perfdata
			next if (! $cmds{$no}{process_perfdata});

			if (defined($cmds{$no}{performance})) {
				#--- prevent concatenation of multi-labels if called recursively
				if ($cmds{$no}{performance}=~/check_multi::check_multi::(.*)/) {
					$perftmp="$cmds{$no}{name}::check_multi::$1 ";
				} elsif ($cmds{$no}{performance}=~/::check_multi::/) {
					$perftmp="$cmds{$no}{performance} ";
				} else {
					#--- do we have an explicit plugin specification? take it
					my $plugin=($cmds{$no}{pplugin} ne "") ? $cmds{$no}{pplugin} : $cmds{$no}{plugin};
					if ($cmds{$no}{performance}=~/^\s*\'([^']*)\'=(\S+)(.*)/) {
						$perftmp="\'$cmds{$no}{name}::${plugin}::$1\'=$2$3 ";
					} else {
						$perftmp="$cmds{$no}{name}::${plugin}::$cmds{$no}{performance} ";
					}
				}
				$cmds{$no}{performance}="";
				DEBUG3("report_perfdata before splitting: $perftmp");
				#--- preserve '::' as multi delimiter - replace all '::' with '_'
				while ($perftmp=~/\s*([^=]+)=(\S+)\s*(.*)/) {
					my $label=$1;
					my $data=$2;
					$perftmp=$3;

					my @tmparr=split(/::/,$label);
					DEBUG3("report_perfdata token before splitting: $label [0-$#tmparr]:" . join("|",@tmparr));
					if ($#tmparr > 1) {
						$cmds{$no}{performance}.=shift(@tmparr)."::".shift(@tmparr)."::";
						$cmds{$no}{performance}.=join("_",@tmparr);
					} else {
						$cmds{$no}{performance}.=$label;
					}
					$cmds{$no}{performance}.="=$data ";
					DEBUG3("report_perfdata remaining perftmp: $perftmp");
				}
				DEBUG3("report_perfdata complete after splitting: $cmds{$no}{performance}");
				print $cmds{$no}{performance};
			}
		}
	} elsif ($opt{set}{report} & $DETAIL_PERFORMANCE_CLASSIC) {
		print "\|";

		#--- one line per command, format: tag=output
		for ($no=1;$no<keys(%cmds);$no++) {

			#--- suppress_perfdata set? ignore perfdata
			next if (! $cmds{$no}{process_perfdata});

			if (defined($cmds{$no}{performance})) {
				if (my $errstr=error_in_perfdata($cmds{$no}{performance})) {
					add_error("$cmds{$no}{name} perfdata discarded for $errstr");
				} else {
					print "$cmds{$no}{performance} ";
				}
			}
		}
	}
}

sub error_in_perfdata {
	my $perfdata=shift;
	my $label="";
	my $data="";
	my $error="";
	my $uom="";


	#---
	#--- 'label'=value[UOM];[warn];[crit];[min];[max]
	#---
	#--- loop over perfdata
	while ($perfdata) {
		DEBUG3("error_in_perfdata: parsing perfdata:$perfdata");	
		#--- label w/ \'
		if ($perfdata=~/^\s*(\'[^']*\')=([^ ]+)(.*)/) {
			$label=$1;
			$data=$2;
			$perfdata=$3;
			DEBUG3("error_in_perfdata: parsed perfdata -'label':$1 data:$2 rest:$3");	
		#--- label w/o \'
		} elsif ($perfdata=~/^\s*([^=]+)=([^ ]+)(.*)/) {
			$label=$1;
			$data=$2;
			$perfdata=$3;
			DEBUG3("error_in_perfdata: parsed perfdata - label:$1 data:$2 rest:$3");	
		#--- invalid
		} else {
			DEBUG3("error_in_perfdata: general parsing error - invalid perfdata");	
			return "general error in '$perfdata'";
		}
		$perfdata=~s/^\s+//; $perfdata=~s/\s+$//;

		#--- perfdata has label and data, now do detailed checks
		my ($value,$warning,$critical,$min,$max)=split(/;/,$data);
		return "$label: no value in data \'$data\'" if ($value eq "");
		if ($value=~/([-0-9.]+)([^0-9-.]{0,2})/) {
			$value=$1;
			$uom=$2;
		}
		$error.= "$label: bad value \'$value\' in data \'$data\' " if ($value && $value !~/^[-0-9.]+$/);
		$error.= "$label: bad UOM \'$uom\' in data \'$data\' " if ($uom ne "" && ($uom!~/s$/ && $uom!~/%$/ && $uom!~/b$/i && $uom!~/c$/));
		$error.= "$label: bad warning \'$warning\' in data \'$data\' " if ($warning && $warning !~/^[-0-9.:]+$/);
		$error.= "$label: bad critical \'$critical\' in data \'$data\' " if ($critical && $critical !~/^[-0-9.:]+$/);
		$error.= "$label: bad min \'$min\' in data \'$data\' " if ($min && $min !~/^[-0-9.]+$/);
		$error.= "$label: bad max \'$max\' in data \'$data\' " if ($max && $max !~/^[-0-9.]+$/);
		return $error if ($error);

		#--- done: perfdata is ok
	}
	DEBUG3("error_in_perfdata: no errors found");	
	return undef;
}

sub report_service_definition {
	my @tpl="";
	if (! -f "$opt{set}{service_definition_template}") {
		add_error("service definition template file \'$opt{set}{service_definition_template}\' not found");
		return 1;
	} elsif (!open(DEF, $opt{set}{service_definition_template})) {
			add_error("Cannot open $opt{set}{service_definition_template}:$?");
			return 2;
	} else {
		@tpl=<DEF>;
		close DEF;
	}
#        foreach my $no (sort numerically keys %cmds) {
#		my $line="$opt{set}{service_description}";
#	print ">>>opt{set}{service_description}:$opt{set}{service_description}\n";
#		print ">>>line1:$line\n";
#		$line=substitute_macros($line);
#		print ">>>line2:$line\n";
#		print $line;
#	}
#	return;
	my $output="";
        foreach my $no (sort numerically keys %cmds) {
		next if ($cmds{$no}{type} eq "eval");
		my @svc=@tpl;
		foreach my $line (@svc) {
			DEBUG3("report_service_definition: before substitution: $line");
			$line=~s/THIS/$no/g;
			$line=substitute_macros($line);
			DEBUG3("report_service_definition: after substitution: $line");
			$output.=$line;
		}
        }
	print $output;
}

#-------------------------------------------------------------------------------
#--- main ----------------------------------------------------------------------
#-------------------------------------------------------------------------------

#--- take care against signals
install_signal_handler(\install_signal_handler, "INT", "TERM", "QUIT", "HUP");

#--- parse command line options
exit $UNKNOWN if (&process_parameters != $OK);

#--- don't run this as root ;-)
DEBUG3("main: userid=$>");
add_error("please don't run plugins as root!") if ($> == 0);

#--- parse command file (nrpe format)
&parse_files($opt{filename});
#--- parse single command lines
&parse_lines(@{$opt{execute}});
#--- no child checks defined yet? Throw UNKNOWN
if (!%cmds) {
	add_error("no checks defined");
	$rc{top}=$opt{set}{no_checks_rc};
	$rc{expr}{$OK}="0==1" if ($opt{set}{no_checks_rc} != $OK);
}

#--- initialize timer for overall timeout
$cmds{0}{starttime}=time;
$cmds{0}{timeouttime}=$cmds{0}{starttime} + $opt{set}{TIMEOUT};

#--- loop over commands in order of config file
$no=1;
while ($no < scalar(keys %cmds)) {
	
	#--- if total timeout is going to be exceeded, cancel next commands
	if (time + $opt{set}{timeout} > $cmds{0}{timeouttime}) {
		$cmds{$no}{output}="UNKNOWN - execution cancelled due to global timeout ($opt{set}{TIMEOUT}s)";
		$cmds{$no}{rc}=$UNKNOWN;
		$cmds{$no}{runtime}=0;
	} else {
		#--- execute command
		&exec_command($no);
	}

	$no++;
}

#--- prepare output
&result_rating;

#--- report
&report_all;

#--- return rc with highest severity
exit $rc{top};
