efebd135eb
* Fixed a bug where servers protected by DR hosts aren't deleted when the server itself is deleted. * Updated DRBD->delete_resource() to remove the server's XML file if the host is a DR host. * Updated anvil-version-change and anvil.sql to enable update_audits and the audits table. Signed-off-by: digimer <mkelly@alteeve.ca>
2188 lines
114 KiB
Perl
Executable File
2188 lines
114 KiB
Perl
Executable File
#!/usr/bin/perl
|
|
#
|
|
# This software was created by Alteeve's Niche! Inc. and has been released under the terms of the GNU GPL
|
|
# version 2.
|
|
#
|
|
# ScanCore Scan Agent for 'ipmitool' to query IPMI data.
|
|
#
|
|
# https://alteeve.com
|
|
#
|
|
# Exit Codes:
|
|
# 0 - Success
|
|
# 1 - Passed in host name was not found in the database.
|
|
# 2 - Bad number of scan_ipmitool_value_uuid's returned for the given
|
|
#
|
|
# 255 - The host's UUID isn't in the hosts table yet, ScanCore itself hasn't been run.
|
|
#
|
|
# TODO:
|
|
# - Set a health score for over/under-temp events.
|
|
# - Don't bother scanning other hosts.... ScanCore does direct calls to decide if/when to reboot an offline
|
|
# node.
|
|
# - Decide if we should parse 'ipmitool sel list'
|
|
# - Detect a hung BMC by trying to talk to ourselves and, if that fails, send 'ipmitool bmc reset cold'.
|
|
# Possibly try pinging the IPMI from the peer as it is not always possible to ping our own interface when
|
|
# the IPMI and eth device share the same physical connector.
|
|
# - Update the string processing from the '!!x!name=a,value=b,units=c!!' method to the new method here:
|
|
# https://alteeve.com/w/ScanCore#Unit_Parsing
|
|
# - Change 'scan_ipmitool_value_uuid' to 'scan_ipmitool_value_uuid'.
|
|
# - Monitor / Alert on RPM, wattage, voltage, etc
|
|
#
|
|
# - A PSU wattage dropping to '0 Watts', in conjunction with it's fan dropping to '0' RPMs, is the sign that
|
|
# input power was lost and should set a health score of ~5.
|
|
#
|
|
# NOTE:
|
|
# - Health values
|
|
# - IPMI BMC - Hung = 10
|
|
# - Temperature - Critical = 2
|
|
# - Temperature - Warning = 1
|
|
# - Fan - Failed = 5
|
|
# - PSU - Lost Input = 5
|
|
=pod
|
|
|
|
A Hung/crashed BMC will fail to reset with this:
|
|
====
|
|
[root@nr-c03n01 ~]# time ipmitool bmc reset cold
|
|
Get Device ID command failed: 0xff Unspecified error
|
|
Sent cold reset command to MC
|
|
|
|
real 3m52.860s
|
|
user 0m0.000s
|
|
sys 0m0.002s
|
|
====
|
|
|
|
Seeing this, we need to ask the peer to power-cycle us (/shared/status/.node.helpme -> 'task = power_cycle')
|
|
and set the health to 'warning'. We will call anvil-safe-stop.
|
|
|
|
The peer, on seeing (/shared/status/.node.helpme -> 'task = power_cycle'), will start pinging the peer on the
|
|
BCN and SN. Once there is no response, we will start a 60 second counter, then cut power to the PDUs for 60
|
|
seconds, then restore power. We will then start pinging the IPMI interface. Once it responds, we will wait 60
|
|
seconds and then try to power it back on.
|
|
|
|
---------
|
|
|
|
Change/set thresholds:
|
|
|
|
From: https://forums.freenas.org/index.php?threads/how-to-change-sensor-thresholds-with-ipmi-using-ipmitool.23571/
|
|
|
|
Lower Non-Recoverable
|
|
Lower Critical
|
|
Lower Non-Critical
|
|
|
|
Upper Non-Critical
|
|
Upper Critical
|
|
Upper Non-Recoverable
|
|
|
|
ipmitool sensor thresh "*sensor name*" lower *lnr* *lcr* *lnc*
|
|
|
|
ipmitool sensor thresh "*sensor name*" upper *unc* *ucr* *unr*
|
|
=cut
|
|
|
|
use strict;
|
|
use warnings;
|
|
use Anvil::Tools;
|
|
use Data::Dumper;
|
|
use Socket;
|
|
no warnings 'recursion';
|
|
|
|
# Disable buffering
|
|
$| = 1;
|
|
|
|
# Prevent a discrepency between UID/GID and EUID/EGID from throwing an error.
|
|
$< = $>;
|
|
$( = $);
|
|
|
|
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
|
|
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
|
|
if (($running_directory =~ /^\./) && ($ENV{PWD}))
|
|
{
|
|
$running_directory =~ s/^\./$ENV{PWD}/;
|
|
}
|
|
|
|
my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
|
|
$anvil->Log->level({set => 2});
|
|
$anvil->Log->secure({set => 1});
|
|
|
|
# Make sure we're running as 'root'
|
|
# $< == real UID, $> == effective UID
|
|
if (($< != 0) && ($> != 0))
|
|
{
|
|
# Not root
|
|
print $anvil->Words->string({key => "error_0005"})."\n";
|
|
$anvil->nice_exit({exit_code => 1});
|
|
}
|
|
|
|
# Here we store data and variables for this agent.
|
|
$anvil->data->{'scan-ipmitool'} = {
|
|
disable => 0,
|
|
# It will be marked as 'clear' when the temperature drops this many °C below the
|
|
# critical temperature.
|
|
host_name => {},
|
|
alert_sort => 2,
|
|
# These are used when no other limits are set for a given sensor.
|
|
thresholds => {
|
|
'default' => {
|
|
high_warning => 50,
|
|
high_critical => 55,
|
|
low_warning => 5,
|
|
low_critical => 0,
|
|
jump => 10,
|
|
### TODO: Some sensors define their hysteresis which we can read using:
|
|
### ipmitool ... sensor get "Ambient"
|
|
buffer => 2,
|
|
weight => 1,
|
|
},
|
|
### TODO: Support wild-card sensor names.
|
|
# If the user wants to assign manual values for a given sensor, they can do
|
|
# so by creating an entry hear where the key is the IPMI-returned sensor
|
|
# name.
|
|
#ie:
|
|
# ===========================================================================
|
|
#'Ambient' => {
|
|
# high_warning => 50,
|
|
# high_critical => 55,
|
|
# low_warning => 5,
|
|
# low_critical => 0,
|
|
# jump => 5,
|
|
# buffer => 2,
|
|
# weight => 1,
|
|
#},
|
|
# ===========================================================================
|
|
# CPUs tend to jump around wildly under sudden load, so we extend their jump
|
|
# range so we don't get spurious warnings all the time.
|
|
'CPU' => {
|
|
jump => 25,
|
|
},
|
|
'CPU1' => {
|
|
jump => 25,
|
|
},
|
|
'CPU2' => {
|
|
jump => 25,
|
|
},
|
|
'CPU3' => {
|
|
jump => 25,
|
|
},
|
|
'CPU4' => {
|
|
jump => 25,
|
|
},
|
|
'CPU5' => {
|
|
jump => 25,
|
|
},
|
|
'CPU6' => {
|
|
jump => 25,
|
|
},
|
|
'CPU7' => {
|
|
jump => 25,
|
|
},
|
|
'CPU8' => {
|
|
jump => 25,
|
|
},
|
|
# The ROC on LSI controllers can just dramatically
|
|
'RAID Controller' => {
|
|
jump => 15,
|
|
},
|
|
# On Dells, 'Temp (xxh)' and 'Exhaust Temp' change a lot, so we bump the jump.
|
|
'Temp' => {
|
|
jump => 30,
|
|
},
|
|
'Exhaust Temp' => {
|
|
jump => 30,
|
|
},
|
|
},
|
|
# TODO: Remove this and have Striker pull the list of thermal sensors read in the
|
|
# last $timestamp.
|
|
offline_sensor_list => "Ambient,Systemboard",
|
|
local_lock_active => 0,
|
|
scanning_myself => 0,
|
|
queries => [],
|
|
health => {
|
|
old => {},
|
|
new => {},
|
|
},
|
|
};
|
|
|
|
$anvil->Storage->read_config();
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1 , key => "log_0115", variables => { program => $THIS_FILE }});
|
|
|
|
# Read switches
|
|
$anvil->data->{switches}{force} = 0;
|
|
$anvil->data->{switches}{purge} = 0;
|
|
$anvil->Get->switches;
|
|
|
|
# Handle start-up tasks
|
|
my $problem = $anvil->ScanCore->agent_startup({agent => $THIS_FILE});
|
|
if ($problem)
|
|
{
|
|
$anvil->nice_exit({exit_code => 1});
|
|
}
|
|
|
|
if ($anvil->data->{switches}{purge})
|
|
{
|
|
# This can be called when doing bulk-database purges.
|
|
my $schema_file = $anvil->data->{path}{directories}{scan_agents}."/".$THIS_FILE."/".$THIS_FILE.".sql";
|
|
$anvil->Database->purge_data({
|
|
debug => 2,
|
|
tables => $anvil->Database->get_tables_from_schema({schema_file => $schema_file}),
|
|
});
|
|
$anvil->nice_exit({exit_code => 0});
|
|
}
|
|
|
|
# This calls anvil-report-ipmi-details to find IPMI devices to scan. It also handles any user-defined IPMI
|
|
# devices set in striker.conf.
|
|
if (not find_ipmi_targets($anvil))
|
|
{
|
|
# No targets found.
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_ipmitool_message_0001"});
|
|
$anvil->nice_exit({exit_code => 1});
|
|
}
|
|
|
|
# Query IPMI targets. Unreachable targets will simply be ignored.
|
|
query_ipmi_targets($anvil);
|
|
|
|
# Look for changes.
|
|
find_changes($anvil);
|
|
|
|
# Finally, process health weights.
|
|
process_health($anvil);
|
|
|
|
# Shut down.
|
|
$anvil->ScanCore->agent_shutdown({agent => $THIS_FILE});
|
|
|
|
|
|
#############################################################################################################
|
|
# Functions #
|
|
#############################################################################################################
|
|
|
|
# This reads in all health wieghts previously set, alters ones as needed, INSERTs new ones and DELETEs old
|
|
# ones.
|
|
sub process_health
|
|
{
|
|
my ($anvil) = @_;
|
|
|
|
# This will hold our updates.
|
|
$anvil->data->{'scan-ipmitool'}{queries} = [];
|
|
|
|
# Read in the new ones
|
|
foreach my $health_source_name (sort {$a cmp $b} keys %{$anvil->data->{'scan-ipmitool'}{health}{new}})
|
|
{
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"health::new::$health_source_name" => $anvil->data->{'scan-ipmitool'}{health}{new}{$health_source_name},
|
|
}});
|
|
|
|
my $health_uuid = $anvil->Database->insert_or_update_health({
|
|
debug => 2,
|
|
cache => $anvil->data->{'scan-ipmitool'}{queries},
|
|
health_host_uuid => $anvil->Get->host_uuid,
|
|
health_agent_name => $THIS_FILE,
|
|
health_source_name => $health_source_name,
|
|
health_source_weight => $anvil->data->{'scan-ipmitool'}{health}{new}{$health_source_name},
|
|
});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { health_uuid => $health_uuid }});
|
|
|
|
# Delete the new old key, if it existed
|
|
if ($anvil->data->{'scan-ipmitool'}{health}{old}{$health_source_name})
|
|
{
|
|
delete $anvil->data->{'scan-ipmitool'}{health}{old}{$health_source_name};
|
|
}
|
|
}
|
|
|
|
# Delete any old entries that are left.
|
|
foreach my $health_source_name (sort {$a cmp $b} keys %{$anvil->data->{'scan-ipmitool'}{health}{old}})
|
|
{
|
|
# Well set the source name to 'DELETED'.
|
|
my $health_uuid = $anvil->Database->insert_or_update_health({
|
|
debug => 2,
|
|
cache => $anvil->data->{'scan-ipmitool'}{queries},
|
|
health_uuid => $anvil->data->{'scan-ipmitool'}{health}{old}{$health_source_name}{uuid},
|
|
'delete' => 1,
|
|
});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { health_uuid => $health_uuid }});
|
|
}
|
|
|
|
# Now commit the changes.
|
|
$anvil->Database->write({query => $anvil->data->{'scan-ipmitool'}{queries}, source => $THIS_FILE, line => __LINE__});
|
|
|
|
return(0);
|
|
}
|
|
|
|
# This reads in the last scan data from one of the databases and compares it against the just-read data. If
|
|
# anything changed, register an alert.
|
|
sub find_changes
|
|
{
|
|
my ($anvil) = @_;
|
|
|
|
# Loop through each host_name
|
|
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{ipmi}})
|
|
{
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }});
|
|
|
|
# This returns the number of read sensors already in the DB for this host_name recorded by us
|
|
# previously.
|
|
if (read_last_scan($anvil, $host_name))
|
|
{
|
|
### Existing host_name, UPDATE or INSERT as needed.
|
|
foreach my $scan_ipmitool_sensor_name (sort {$a cmp $b} keys %{$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}})
|
|
{
|
|
# Put the new values into variables
|
|
my $new_scan_ipmitool_value_sensor_value = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_value_sensor_value};
|
|
my $new_scan_ipmitool_sensor_units = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_units};
|
|
my $new_scan_ipmitool_sensor_status = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_status};
|
|
my $new_scan_ipmitool_sensor_high_critical = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_critical};
|
|
my $new_scan_ipmitool_sensor_high_warning = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_warning};
|
|
my $new_scan_ipmitool_sensor_low_critical = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_critical};
|
|
my $new_scan_ipmitool_sensor_low_warning = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_warning};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
new_scan_ipmitool_value_sensor_value => $new_scan_ipmitool_value_sensor_value,
|
|
new_scan_ipmitool_sensor_units => $new_scan_ipmitool_sensor_units,
|
|
new_scan_ipmitool_sensor_status => $new_scan_ipmitool_sensor_status,
|
|
new_scan_ipmitool_sensor_high_critical => $new_scan_ipmitool_sensor_high_critical,
|
|
new_scan_ipmitool_sensor_high_warning => $new_scan_ipmitool_sensor_high_warning,
|
|
new_scan_ipmitool_sensor_low_critical => $new_scan_ipmitool_sensor_low_critical,
|
|
new_scan_ipmitool_sensor_low_warning => $new_scan_ipmitool_sensor_low_warning,
|
|
}});
|
|
|
|
# If the new value is 'na', we failed to read it. Skip.
|
|
if ($new_scan_ipmitool_value_sensor_value eq "na")
|
|
{
|
|
delete $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name};
|
|
next;
|
|
}
|
|
|
|
### NOTE: These were added to debug duplicate scan_ipmitool_values entries.
|
|
if (not $new_scan_ipmitool_sensor_units)
|
|
{
|
|
my $variables = {
|
|
host_name => $host_name,
|
|
sensor => $scan_ipmitool_sensor_name,
|
|
};
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_ipmitool_message_0002", variables => $variables});
|
|
$anvil->Alert->register({alert_level => "notice", message => "scan_ipmitool_message_0002", variables => $variables, set_by => $THIS_FILE, sort_position => $anvil->data->{'scan-ipmitool'}{alert_sort}++});
|
|
next;
|
|
}
|
|
if (not $new_scan_ipmitool_value_sensor_value)
|
|
{
|
|
my $variables = {
|
|
host_name => $host_name,
|
|
sensor => $scan_ipmitool_sensor_name,
|
|
};
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_ipmitool_message_0003", variables => $variables});
|
|
$anvil->Alert->register({alert_level => "notice", message => "scan_ipmitool_message_0003", variables => $variables, set_by => $THIS_FILE, sort_position => $anvil->data->{'scan-ipmitool'}{alert_sort}++});
|
|
next;
|
|
}
|
|
|
|
# If the value is "" and it's a digit-based value, switch it to '0'
|
|
if ((($new_scan_ipmitool_sensor_units eq "C") or ($new_scan_ipmitool_sensor_units eq "F") or ($new_scan_ipmitool_sensor_units eq "RPM")) && ($new_scan_ipmitool_value_sensor_value eq ""))
|
|
{
|
|
$new_scan_ipmitool_value_sensor_value = 0;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_scan_ipmitool_value_sensor_value => $new_scan_ipmitool_value_sensor_value }});
|
|
}
|
|
|
|
# Have I seen this sensor before?
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"ref(sql::${host_name}::scan_ipmitool_sensor_name::${scan_ipmitool_sensor_name})" => ref($anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}),
|
|
}});
|
|
if (ref($anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}))
|
|
{
|
|
### Existing record, update it if needed.
|
|
# Put the old values into variables
|
|
my $scan_ipmitool_uuid = $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_uuid};
|
|
my $old_scan_ipmitool_value_sensor_value = $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_value_sensor_value};
|
|
my $old_scan_ipmitool_sensor_units = $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_units};
|
|
my $old_scan_ipmitool_sensor_status = $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_status};
|
|
my $old_scan_ipmitool_sensor_high_critical = $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_critical};
|
|
my $old_scan_ipmitool_sensor_high_warning = $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_warning};
|
|
my $old_scan_ipmitool_sensor_low_critical = $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_critical};
|
|
my $old_scan_ipmitool_sensor_low_warning = $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_warning};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
scan_ipmitool_uuid => $scan_ipmitool_uuid,
|
|
old_scan_ipmitool_value_sensor_value => $old_scan_ipmitool_value_sensor_value,
|
|
old_scan_ipmitool_sensor_units => $old_scan_ipmitool_sensor_units,
|
|
old_scan_ipmitool_sensor_status => $old_scan_ipmitool_sensor_status,
|
|
old_scan_ipmitool_sensor_high_critical => $old_scan_ipmitool_sensor_high_critical,
|
|
old_scan_ipmitool_sensor_high_warning => $old_scan_ipmitool_sensor_high_warning,
|
|
old_scan_ipmitool_sensor_low_critical => $old_scan_ipmitool_sensor_low_critical,
|
|
old_scan_ipmitool_sensor_low_warning => $old_scan_ipmitool_sensor_low_warning,
|
|
}});
|
|
|
|
# If the value is "" and it's a digit-based value, switch it to '0'
|
|
if ((($old_scan_ipmitool_sensor_units eq "C") or ($old_scan_ipmitool_sensor_units eq "F") or ($old_scan_ipmitool_sensor_units eq "RPM")) && ($old_scan_ipmitool_value_sensor_value eq ""))
|
|
{
|
|
$old_scan_ipmitool_value_sensor_value = 0;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_scan_ipmitool_value_sensor_value => $old_scan_ipmitool_value_sensor_value }});
|
|
}
|
|
|
|
# In some odd cases, there is no old sensor value
|
|
$old_scan_ipmitool_value_sensor_value = "" if not defined $old_scan_ipmitool_value_sensor_value;
|
|
|
|
# These will be used in alert messages, if needed.
|
|
my $sensor_name = "name=$scan_ipmitool_sensor_name:units=$new_scan_ipmitool_sensor_units";
|
|
my $new_sensor_value = "value=$new_scan_ipmitool_value_sensor_value:units=$new_scan_ipmitool_sensor_units";
|
|
my $old_sensor_value = "value=$old_scan_ipmitool_value_sensor_value:units=$old_scan_ipmitool_sensor_units";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
scan_ipmitool_sensor_name => $scan_ipmitool_sensor_name,
|
|
sensor_name => $sensor_name,
|
|
new_sensor_value => $new_sensor_value,
|
|
old_sensor_value => $old_sensor_value,
|
|
}});
|
|
|
|
### TODO: We can remove this, I think.
|
|
### Look for changes
|
|
# NOTE: If the new sensor has a state of 'na' and the old state
|
|
# wasn't 'na', we ignore it. It happens not uncommonly and it's
|
|
# never been an indication of a real problem before.
|
|
# if ($new_scan_ipmitool_sensor_status eq "na")
|
|
# {
|
|
# # This is usually harmless, so we simple log this and move on
|
|
# $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_ipmitool_log_0004", variables => { sensor_name => $scan_ipmitool_sensor_name }});
|
|
#
|
|
# # We loop out now because the rest of the values will look
|
|
# # really bad (ie: temps down to 0c) and we don't want to
|
|
# # trigger preventative actions on bad data.
|
|
# delete $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name};
|
|
# next;
|
|
# }
|
|
# elsif (($new_scan_ipmitool_sensor_status ne "na") && ($old_scan_ipmitool_sensor_status eq "na"))
|
|
# {
|
|
# # It disappeared in a previous sweep, now it's back. We
|
|
# # continue normal processing to handle those rare occassions
|
|
# # where a sensor was lost back when we cared, and returned
|
|
# # after updating to this version when we stopped caring.
|
|
# $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_ipmitool_log_0005", variables => { sensor_name => $scan_ipmitool_sensor_name }});
|
|
# }
|
|
|
|
# Sensor value:
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
new_scan_ipmitool_value_sensor_value => $new_scan_ipmitool_value_sensor_value,
|
|
old_scan_ipmitool_value_sensor_value => $old_scan_ipmitool_value_sensor_value,
|
|
}});
|
|
if ($new_scan_ipmitool_value_sensor_value ne $old_scan_ipmitool_value_sensor_value)
|
|
{
|
|
# Update (no surprise ...)
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
host_name => $host_name,
|
|
sensor_name => $sensor_name,
|
|
new_scan_ipmitool_value_sensor_value => $new_scan_ipmitool_value_sensor_value,
|
|
old_scan_ipmitool_value_sensor_value => $old_scan_ipmitool_value_sensor_value,
|
|
}});
|
|
my $query = "
|
|
UPDATE
|
|
scan_ipmitool_values
|
|
SET
|
|
scan_ipmitool_value_sensor_value = ".$anvil->Database->quote($new_scan_ipmitool_value_sensor_value).",
|
|
modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
|
|
WHERE
|
|
scan_ipmitool_value_scan_ipmitool_uuid = ".$anvil->Database->quote($scan_ipmitool_uuid)."
|
|
;
|
|
";
|
|
$query =~ s/'NULL'/NULL/g;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
push @{$anvil->data->{'scan-ipmitool'}{queries}}, $query;
|
|
|
|
### NOTE: These are set in 'process_temperature_change()' as
|
|
### well, so change the values there if you change it
|
|
### here, too.
|
|
# This is an info-level alert, provided it is not gone above
|
|
# or below tolerances.
|
|
my $level = "info";
|
|
my $message_key = "scan_ipmitool_message_0004";
|
|
|
|
# If this is a temperature, see if we need to trigger an
|
|
# alarm and set/clear the 'temperature' table entry.
|
|
# TODO: For the rest of the sensors, we just log the changes.
|
|
# If a sensor goes bad, for now we'll catch errors on
|
|
# the sensor status change. We'll add better checks
|
|
# later.
|
|
if ($new_scan_ipmitool_sensor_units eq "C")
|
|
{
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
host_name => $host_name,
|
|
sensor_name => $sensor_name,
|
|
new_scan_ipmitool_value_sensor_value => $new_scan_ipmitool_value_sensor_value,
|
|
old_scan_ipmitool_value_sensor_value => $old_scan_ipmitool_value_sensor_value,
|
|
}});
|
|
($level, $message_key) = process_temperature_change($anvil, $host_name, $scan_ipmitool_sensor_name);
|
|
|
|
my $variables = {
|
|
sensor_name => $sensor_name,
|
|
new_sensor_value => $new_sensor_value,
|
|
new_sensor_status => $new_scan_ipmitool_sensor_status,
|
|
new_high_critical => $new_scan_ipmitool_sensor_high_critical eq "" ? "--" : $new_scan_ipmitool_sensor_high_critical,
|
|
new_low_critical => $new_scan_ipmitool_sensor_low_critical eq "" ? "--" : $new_scan_ipmitool_sensor_low_critical,
|
|
new_high_warning => $new_scan_ipmitool_sensor_high_warning eq "" ? "--" : $new_scan_ipmitool_sensor_high_warning,
|
|
new_low_warning => $new_scan_ipmitool_sensor_low_warning eq "" ? "--" : $new_scan_ipmitool_sensor_low_warning,
|
|
old_sensor_value => $old_sensor_value,
|
|
old_sensor_status => $old_scan_ipmitool_sensor_status,
|
|
old_high_critical => $old_scan_ipmitool_sensor_high_critical eq "" ? "--" : $old_scan_ipmitool_sensor_high_critical,
|
|
old_low_critical => $old_scan_ipmitool_sensor_low_critical eq "" ? "--" : $old_scan_ipmitool_sensor_low_critical,
|
|
old_high_warning => $old_scan_ipmitool_sensor_high_warning eq "" ? "--" : $old_scan_ipmitool_sensor_high_warning,
|
|
old_low_warning => $old_scan_ipmitool_sensor_low_warning eq "" ? "--" : $old_scan_ipmitool_sensor_low_warning,
|
|
};
|
|
my $sort_position = (($level eq "warning") or ($level eq "critical")) ? 1 : $anvil->data->{'scan-ipmitool'}{alert_sort}++;
|
|
my $log_level = (($level eq "warning") or ($level eq "critical")) ? 1 : 2;
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $log_level, key => $message_key, variables => $variables});
|
|
$anvil->Alert->register({alert_level => $level, message => $message_key, variables => $variables, set_by => $THIS_FILE, sort_position => $sort_position});
|
|
}
|
|
else
|
|
{
|
|
# TODO: Add the checks of variables against the
|
|
# high/low warning/critical thresholds here
|
|
# (like RPM, wattage, voltage, etc).
|
|
my $variables = {
|
|
sensor_name => $sensor_name,
|
|
new_sensor_value => $new_sensor_value,
|
|
old_sensor_value => $old_sensor_value
|
|
};
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => $message_key, variables => $variables});
|
|
$anvil->Alert->register({alert_level => $level, message => $message_key, variables => $variables, set_by => $THIS_FILE, sort_position => $anvil->data->{'scan-ipmitool'}{alert_sort}++});
|
|
}
|
|
}
|
|
else
|
|
{
|
|
# No change.
|
|
$anvil->Log->entry({log_level => 2, message_key => "scancore_log_0047", file => $THIS_FILE, line => __LINE__});
|
|
}
|
|
|
|
# Everything else
|
|
if (($new_scan_ipmitool_sensor_units ne $old_scan_ipmitool_sensor_units) or
|
|
($new_scan_ipmitool_sensor_status ne $old_scan_ipmitool_sensor_status) or
|
|
($new_scan_ipmitool_sensor_high_critical ne $old_scan_ipmitool_sensor_high_critical) or
|
|
($new_scan_ipmitool_sensor_high_warning ne $old_scan_ipmitool_sensor_high_warning) or
|
|
($new_scan_ipmitool_sensor_low_critical ne $old_scan_ipmitool_sensor_low_critical) or
|
|
($new_scan_ipmitool_sensor_low_warning ne $old_scan_ipmitool_sensor_low_warning))
|
|
{
|
|
# Huh, interesting. Well, update
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
new_scan_ipmitool_sensor_units => $new_scan_ipmitool_sensor_units,
|
|
new_scan_ipmitool_sensor_status => $new_scan_ipmitool_sensor_status,
|
|
new_scan_ipmitool_sensor_high_critical => $new_scan_ipmitool_sensor_high_critical,
|
|
new_scan_ipmitool_sensor_high_warning => $new_scan_ipmitool_sensor_high_warning,
|
|
new_scan_ipmitool_sensor_low_critical => $new_scan_ipmitool_sensor_low_critical,
|
|
new_scan_ipmitool_sensor_low_warning => $new_scan_ipmitool_sensor_low_warning,
|
|
old_scan_ipmitool_sensor_units => $old_scan_ipmitool_sensor_units,
|
|
old_scan_ipmitool_sensor_status => $old_scan_ipmitool_sensor_status,
|
|
old_scan_ipmitool_sensor_high_critical => $old_scan_ipmitool_sensor_high_critical,
|
|
old_scan_ipmitool_sensor_high_warning => $old_scan_ipmitool_sensor_high_warning,
|
|
old_scan_ipmitool_sensor_low_critical => $old_scan_ipmitool_sensor_low_critical,
|
|
old_scan_ipmitool_sensor_low_warning => $old_scan_ipmitool_sensor_low_warning,
|
|
}});
|
|
|
|
### NOTE: These were added to debug duplicate scan_ipmitool_values entries.
|
|
if (not $new_scan_ipmitool_sensor_units)
|
|
{
|
|
my $variables = {
|
|
host_name => $host_name,
|
|
sensor => $scan_ipmitool_sensor_name,
|
|
};
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_ipmitool_message_0015", variables => $variables});
|
|
$anvil->Alert->register({alert_level => "notice", message => "scan_ipmitool_message_0015", variables => $variables, set_by => $THIS_FILE, show_header => 0, sort_position => $anvil->data->{'scan-ipmitool'}{alert_sort}++});
|
|
next;
|
|
}
|
|
|
|
my $query = "
|
|
UPDATE
|
|
scan_ipmitool
|
|
SET
|
|
scan_ipmitool_sensor_units = ".$anvil->Database->quote($new_scan_ipmitool_sensor_units).",
|
|
scan_ipmitool_sensor_status = ".$anvil->Database->quote($new_scan_ipmitool_sensor_status).",
|
|
scan_ipmitool_sensor_high_critical = ".$anvil->Database->quote($new_scan_ipmitool_sensor_high_critical).",
|
|
scan_ipmitool_sensor_high_warning = ".$anvil->Database->quote($new_scan_ipmitool_sensor_high_warning).",
|
|
scan_ipmitool_sensor_low_critical = ".$anvil->Database->quote($new_scan_ipmitool_sensor_low_critical).",
|
|
scan_ipmitool_sensor_low_warning = ".$anvil->Database->quote($new_scan_ipmitool_sensor_low_warning).",
|
|
modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
|
|
WHERE
|
|
scan_ipmitool_sensor_host = ".$anvil->Database->quote($host_name)."
|
|
AND
|
|
scan_ipmitool_uuid = ".$anvil->Database->quote($scan_ipmitool_uuid)."
|
|
;";
|
|
$query =~ s/'NULL'/NULL/g;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
push @{$anvil->data->{'scan-ipmitool'}{queries}}, $query;
|
|
|
|
# If this is a status change, set a 'warning' level alert
|
|
# unless the change is to/from 'ns'.
|
|
my $level = "notice";
|
|
my $message_key = "scan_ipmitool_message_0016";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
level => $level,
|
|
message_key => $message_key,
|
|
}});
|
|
if (($new_scan_ipmitool_sensor_status ne $old_scan_ipmitool_sensor_status) &&
|
|
($new_scan_ipmitool_sensor_status ne "ns") &&
|
|
($old_scan_ipmitool_sensor_status ne "ns"))
|
|
{
|
|
$level = "warning";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { level => $level }});
|
|
}
|
|
my $variables = {
|
|
sensor_name => $sensor_name,
|
|
new_sensor_value => $new_sensor_value,
|
|
old_sensor_value => $old_sensor_value,
|
|
new_sensor_status => $new_scan_ipmitool_sensor_status,
|
|
old_sensor_status => $old_scan_ipmitool_sensor_status,
|
|
new_high_critical => $new_scan_ipmitool_sensor_high_critical eq "" ? "--" : $new_scan_ipmitool_sensor_high_critical." ".$new_scan_ipmitool_sensor_units,
|
|
new_high_warning => $new_scan_ipmitool_sensor_high_warning eq "" ? "--" : $new_scan_ipmitool_sensor_high_warning." ".$new_scan_ipmitool_sensor_units,
|
|
new_low_critical => $new_scan_ipmitool_sensor_low_critical eq "" ? "--" : $new_scan_ipmitool_sensor_low_critical." ".$new_scan_ipmitool_sensor_units,
|
|
new_low_warning => $new_scan_ipmitool_sensor_low_warning eq "" ? "--" : $new_scan_ipmitool_sensor_low_warning." ".$new_scan_ipmitool_sensor_units,
|
|
old_high_critical => $old_scan_ipmitool_sensor_high_critical eq "" ? "--" : $old_scan_ipmitool_sensor_high_critical." ".$old_scan_ipmitool_sensor_units,
|
|
old_high_warning => $old_scan_ipmitool_sensor_high_warning eq "" ? "--" : $old_scan_ipmitool_sensor_high_warning." ".$old_scan_ipmitool_sensor_units,
|
|
old_low_critical => $old_scan_ipmitool_sensor_low_critical eq "" ? "--" : $old_scan_ipmitool_sensor_low_critical." ".$old_scan_ipmitool_sensor_units,
|
|
old_low_warning => $old_scan_ipmitool_sensor_low_warning eq "" ? "--" : $old_scan_ipmitool_sensor_low_warning." ".$old_scan_ipmitool_sensor_units,
|
|
};
|
|
|
|
my $sort_position = (($level eq "warning") or ($level eq "critical")) ? 1 : $anvil->data->{'scan-ipmitool'}{alert_sort}++;
|
|
my $log_level = (($level eq "warning") or ($level eq "critical")) ? 1 : 2;
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $log_level, key => $message_key, variables => $variables});
|
|
$anvil->Alert->register({alert_level => $level, message => $message_key, variables => $variables, set_by => $THIS_FILE, sort_position => $sort_position});
|
|
}
|
|
|
|
# Delete the old key so that I can check to see what sensors
|
|
# vanished.
|
|
delete $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name};
|
|
}
|
|
else
|
|
{
|
|
### NOTE: If the new value is 'na', we ignore it as it is likely a
|
|
### sensor that doesn't actually exist.
|
|
if ($anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_value_sensor_value} eq "")
|
|
{
|
|
# Ignore it.
|
|
$anvil->Log->entry({log_level => 3, message_key => "scan_ipmitool_log_0005", variables => { sensor_name => $scan_ipmitool_sensor_name }, file => $THIS_FILE, line => __LINE__});
|
|
delete $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name};
|
|
next;
|
|
}
|
|
|
|
### New record, INSERT it and sent an 'notice' level alert.
|
|
# Generate a new UUID for this ipmi target.
|
|
my $scan_ipmitool_uuid = $anvil->Get->uuid();
|
|
|
|
# Record the new UUID
|
|
$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_uuid} = $scan_ipmitool_uuid;
|
|
|
|
### NOTE: These were added to debug duplicate scan_ipmitool_values entries.
|
|
if (not $new_scan_ipmitool_sensor_units)
|
|
{
|
|
my $variables = {
|
|
host_name => $host_name,
|
|
sensor => $scan_ipmitool_sensor_name,
|
|
};
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_ipmitool_message_0017", variables => $variables});
|
|
$anvil->Alert->register({alert_level => "notice", message => "scan_ipmitool_message_0017", variables => $variables, set_by => $THIS_FILE, sort_position => $anvil->data->{'scan-ipmitool'}{alert_sort}++});
|
|
next;
|
|
}
|
|
if (not $new_scan_ipmitool_value_sensor_value)
|
|
{
|
|
my $variables = {
|
|
host_name => $host_name,
|
|
sensor => $scan_ipmitool_sensor_name,
|
|
};
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_ipmitool_message_0018", variables => $variables});
|
|
$anvil->Alert->register({alert_level => "notice", message => "scan_ipmitool_message_0018", variables => $variables, set_by => $THIS_FILE, sort_position => $anvil->data->{'scan-ipmitool'}{alert_sort}++});
|
|
next;
|
|
}
|
|
|
|
# Save the new sensor.
|
|
my $query = "
|
|
INSERT INTO
|
|
scan_ipmitool
|
|
(
|
|
scan_ipmitool_uuid,
|
|
scan_ipmitool_host_uuid,
|
|
scan_ipmitool_sensor_host,
|
|
scan_ipmitool_sensor_name,
|
|
scan_ipmitool_sensor_units,
|
|
scan_ipmitool_sensor_status,
|
|
scan_ipmitool_sensor_high_critical,
|
|
scan_ipmitool_sensor_high_warning,
|
|
scan_ipmitool_sensor_low_critical,
|
|
scan_ipmitool_sensor_low_warning,
|
|
modified_date
|
|
) VALUES (
|
|
".$anvil->Database->quote($scan_ipmitool_uuid).",
|
|
".$anvil->Database->quote($anvil->Get->host_uuid).",
|
|
".$anvil->Database->quote($host_name).",
|
|
".$anvil->Database->quote($scan_ipmitool_sensor_name).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_sensor_units).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_sensor_status).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_sensor_high_critical).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_sensor_high_warning).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_sensor_low_critical).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_sensor_low_warning).",
|
|
".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
|
|
);";
|
|
$query =~ s/'NULL'/NULL/g;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
push @{$anvil->data->{'scan-ipmitool'}{queries}}, $query;
|
|
|
|
# Now INSERT the sensor value.
|
|
my $scan_ipmitool_value_uuid = $anvil->Get->uuid();
|
|
$query = "
|
|
INSERT INTO
|
|
scan_ipmitool_values
|
|
(
|
|
scan_ipmitool_value_uuid,
|
|
scan_ipmitool_value_host_uuid,
|
|
scan_ipmitool_value_scan_ipmitool_uuid,
|
|
scan_ipmitool_value_sensor_value,
|
|
modified_date
|
|
) VALUES (
|
|
".$anvil->Database->quote($scan_ipmitool_value_uuid).",
|
|
".$anvil->Database->quote($anvil->Get->host_uuid).",
|
|
".$anvil->Database->quote($scan_ipmitool_uuid).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_value_sensor_value).",
|
|
".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
|
|
);";
|
|
$query =~ s/'NULL'/NULL/g;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
push @{$anvil->data->{'scan-ipmitool'}{queries}}, $query;
|
|
|
|
my $sensor_name = "name=".$scan_ipmitool_sensor_name.":units=".$new_scan_ipmitool_sensor_units;
|
|
my $sensor_value = "value=".$new_scan_ipmitool_value_sensor_value.":units=".$new_scan_ipmitool_sensor_units;
|
|
my $message_key = "scan_ipmitool_message_0019";
|
|
my $level = "notice";
|
|
my $variables = {
|
|
host_name => $host_name,
|
|
sensor_name => $sensor_name,
|
|
sensor_value => $sensor_value,
|
|
sensor_status => $new_scan_ipmitool_sensor_status,
|
|
high_critical => $new_scan_ipmitool_sensor_high_critical eq "" ? "--" : $new_scan_ipmitool_sensor_high_critical." ".$new_scan_ipmitool_sensor_units,
|
|
high_warning => $new_scan_ipmitool_sensor_high_warning eq "" ? "--" : $new_scan_ipmitool_sensor_high_warning." ".$new_scan_ipmitool_sensor_units,
|
|
low_critical => $new_scan_ipmitool_sensor_low_critical eq "" ? "--" : $new_scan_ipmitool_sensor_low_critical." ".$new_scan_ipmitool_sensor_units,
|
|
low_warning => $new_scan_ipmitool_sensor_low_warning eq "" ? "--" : $new_scan_ipmitool_sensor_low_warning." ".$new_scan_ipmitool_sensor_units,
|
|
};
|
|
|
|
if ($new_scan_ipmitool_sensor_status ne "ok")
|
|
{
|
|
# Alert cleared.
|
|
$message_key = "scan_ipmitool_message_0020";
|
|
$level = "warning";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
message_key => $message_key,
|
|
level => $level,
|
|
}});
|
|
}
|
|
|
|
my $sort_position = $level eq "warning" ? 1 : $anvil->data->{'scan-ipmitool'}{alert_sort}++;
|
|
my $log_level = $level eq "warning" ? 1 : 2;
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $log_level, key => $message_key, variables => $variables});
|
|
$anvil->Alert->register({alert_level => $level, message => $message_key, variables => $variables, set_by => $THIS_FILE, sort_position => $sort_position});
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
# New host_name, INSERT everything.
|
|
foreach my $scan_ipmitool_sensor_name (sort {$a cmp $b} keys %{$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}})
|
|
{
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { scan_ipmitool_sensor_name => $scan_ipmitool_sensor_name }});
|
|
|
|
### NOTE: If the new value is 'na', we ignore it as it is likely a sensor
|
|
### that doesn't actually exist.
|
|
if ($anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_value_sensor_value} eq "")
|
|
{
|
|
# Ignore it.
|
|
$anvil->Log->entry({log_level => 3, message_key => "scan_ipmitool_log_0005", variables => { sensor_name => $scan_ipmitool_sensor_name }, file => $THIS_FILE, line => __LINE__});
|
|
delete $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name};
|
|
next;
|
|
}
|
|
|
|
# Generate the UUID.
|
|
my $scan_ipmitool_uuid = $anvil->Get->uuid() or $anvil->Alert->error({title_key => "error_title_0020", message_key => "error_message_0024", code => 2, file => $THIS_FILE, line => __LINE__});
|
|
|
|
# Record it.
|
|
$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_uuid} = $scan_ipmitool_uuid;
|
|
|
|
my $new_scan_ipmitool_value_sensor_value = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_value_sensor_value};
|
|
my $new_scan_ipmitool_sensor_units = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_units};
|
|
my $new_scan_ipmitool_sensor_status = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_status};
|
|
my $new_scan_ipmitool_sensor_high_critical = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_critical};
|
|
my $new_scan_ipmitool_sensor_high_warning = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_warning};
|
|
my $new_scan_ipmitool_sensor_low_critical = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_critical};
|
|
my $new_scan_ipmitool_sensor_low_warning = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_warning};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
scan_ipmitool_uuid => $scan_ipmitool_uuid,
|
|
new_scan_ipmitool_sensor_units => $new_scan_ipmitool_sensor_units,
|
|
new_scan_ipmitool_sensor_status => $new_scan_ipmitool_sensor_status,
|
|
new_scan_ipmitool_sensor_high_critical => $new_scan_ipmitool_sensor_high_critical,
|
|
new_scan_ipmitool_sensor_high_warning => $new_scan_ipmitool_sensor_high_warning,
|
|
new_scan_ipmitool_sensor_low_critical => $new_scan_ipmitool_sensor_low_critical,
|
|
new_scan_ipmitool_sensor_low_warning => $new_scan_ipmitool_sensor_low_warning,
|
|
}});
|
|
|
|
# If the new value is 'na', we failed to read it. Skip.
|
|
if ($new_scan_ipmitool_value_sensor_value eq "na")
|
|
{
|
|
delete $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name};
|
|
next;
|
|
}
|
|
|
|
if (not $new_scan_ipmitool_sensor_units)
|
|
{
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_ipmitool_warn_0001", variables => { sensor => $scan_ipmitool_sensor_name }});
|
|
next;
|
|
}
|
|
if (not $new_scan_ipmitool_value_sensor_value)
|
|
{
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_ipmitool_warn_0001", variables => { sensor => $new_scan_ipmitool_value_sensor_value }});
|
|
next;
|
|
}
|
|
|
|
# If the value is "" and it's a digit-based value, switch it to '0'
|
|
if ((($new_scan_ipmitool_sensor_units eq "C") or ($new_scan_ipmitool_sensor_units eq "F") or ($new_scan_ipmitool_sensor_units eq "RPM")) && ($new_scan_ipmitool_value_sensor_value eq ""))
|
|
{
|
|
$new_scan_ipmitool_value_sensor_value = 0;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_scan_ipmitool_value_sensor_value => $new_scan_ipmitool_value_sensor_value }});
|
|
}
|
|
|
|
my $query = "
|
|
INSERT INTO
|
|
scan_ipmitool
|
|
(
|
|
scan_ipmitool_uuid,
|
|
scan_ipmitool_host_uuid,
|
|
scan_ipmitool_sensor_name,
|
|
scan_ipmitool_sensor_host,
|
|
scan_ipmitool_sensor_units,
|
|
scan_ipmitool_sensor_status,
|
|
scan_ipmitool_sensor_high_critical,
|
|
scan_ipmitool_sensor_high_warning,
|
|
scan_ipmitool_sensor_low_critical,
|
|
scan_ipmitool_sensor_low_warning,
|
|
modified_date
|
|
) VALUES (
|
|
".$anvil->Database->quote($scan_ipmitool_uuid).",
|
|
".$anvil->Database->quote($anvil->Get->host_uuid).",
|
|
".$anvil->Database->quote($scan_ipmitool_sensor_name).",
|
|
".$anvil->Database->quote($host_name).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_sensor_units).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_sensor_status).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_sensor_high_critical).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_sensor_high_warning).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_sensor_low_critical).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_sensor_low_warning).",
|
|
".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
|
|
);";
|
|
$query =~ s/'NULL'/NULL/g;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
push @{$anvil->data->{'scan-ipmitool'}{queries}}, $query;
|
|
|
|
# Now INSERT the sensor value.
|
|
my $scan_ipmitool_value_uuid = $anvil->Get->uuid();
|
|
$query = "
|
|
INSERT INTO
|
|
scan_ipmitool_values
|
|
(
|
|
scan_ipmitool_value_uuid,
|
|
scan_ipmitool_value_host_uuid,
|
|
scan_ipmitool_value_scan_ipmitool_uuid,
|
|
scan_ipmitool_value_sensor_value,
|
|
modified_date
|
|
) VALUES (
|
|
".$anvil->Database->quote($scan_ipmitool_value_uuid).",
|
|
".$anvil->Database->quote($anvil->Get->host_uuid).",
|
|
".$anvil->Database->quote($scan_ipmitool_uuid).",
|
|
".$anvil->Database->quote($new_scan_ipmitool_value_sensor_value).",
|
|
".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
|
|
);";
|
|
$query =~ s/'NULL'/NULL/g;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
push @{$anvil->data->{'scan-ipmitool'}{queries}}, $query;
|
|
|
|
# We need to let ScanCore translate the sensor data into a string for each
|
|
# user and their prefered language and units. As such, we're going to use a
|
|
# couple special variable strings for this.
|
|
my $sensor_name = "name=".$scan_ipmitool_sensor_name.":units=".$new_scan_ipmitool_sensor_units;
|
|
my $sensor_value = "value=".$new_scan_ipmitool_value_sensor_value.":units=".$new_scan_ipmitool_sensor_units;
|
|
|
|
# If the sensor is not 'ok', set a warning level alert.
|
|
my $level = "notice";
|
|
my $message_key = "scan_ipmitool_message_0019";
|
|
if ($new_scan_ipmitool_sensor_status ne "ok")
|
|
{
|
|
$level = "warning";
|
|
$message_key = "scan_ipmitool_message_0020";
|
|
}
|
|
|
|
my $variables = {
|
|
host_name => $host_name,
|
|
sensor_name => $scan_ipmitool_sensor_name,
|
|
sensor_value => $sensor_value,
|
|
sensor_status => $new_scan_ipmitool_sensor_status,
|
|
high_critical => $new_scan_ipmitool_sensor_high_critical eq "" ? "--" : $new_scan_ipmitool_sensor_high_critical." ".$new_scan_ipmitool_sensor_units,
|
|
high_warning => $new_scan_ipmitool_sensor_high_warning eq "" ? "--" : $new_scan_ipmitool_sensor_high_warning." ".$new_scan_ipmitool_sensor_units,
|
|
low_critical => $new_scan_ipmitool_sensor_low_critical eq "" ? "--" : $new_scan_ipmitool_sensor_low_critical." ".$new_scan_ipmitool_sensor_units,
|
|
low_warning => $new_scan_ipmitool_sensor_low_warning eq "" ? "--" : $new_scan_ipmitool_sensor_low_warning." ".$new_scan_ipmitool_sensor_units,
|
|
};
|
|
my $sort_position = $level eq "warning" ? 1 : $anvil->data->{'scan-ipmitool'}{alert_sort}++;
|
|
my $log_level = $level eq "warning" ? 1 : 2;
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $log_level, key => $message_key, variables => $variables});
|
|
$anvil->Alert->register({alert_level => $level, message => $message_key, variables => $variables, set_by => $THIS_FILE, sort_position => $sort_position});
|
|
}
|
|
}
|
|
|
|
# If I am scanning myself and if I see problems, I will set the health accordingly.
|
|
if (($host_name eq $anvil->Get->host_name) or ($host_name eq $anvil->Get->short_host_name))
|
|
{
|
|
$anvil->data->{sys}{scanning_myself} = 1;
|
|
}
|
|
else
|
|
{
|
|
$anvil->data->{sys}{scanning_myself} = 0;
|
|
}
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::scanning_myself" => $anvil->data->{sys}{scanning_myself} }});
|
|
|
|
# Now commit the changes.
|
|
$anvil->Database->write({debug => 2, query => $anvil->data->{'scan-ipmitool'}{queries}, source => $THIS_FILE, line => __LINE__});
|
|
$anvil->data->{'scan-ipmitool'}{queries} = [];
|
|
|
|
### Now add, update and delete 'temperature' entries.
|
|
process_temperature($anvil, $host_name);
|
|
|
|
# Now look for any sensors that are in a bad state and set the health accordingly.
|
|
check_sensor_health($anvil, $host_name);
|
|
}
|
|
|
|
return(0);
|
|
}
|
|
|
|
# This looks at various sensors, except temperature (handled in process_temperature() below) and sets the
|
|
# health table as needed for sensors out of scope.
|
|
sub check_sensor_health
|
|
{
|
|
my ($anvil, $host_name) = @_;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }});
|
|
|
|
foreach my $scan_ipmitool_sensor_name (sort {$a cmp $b} keys %{$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}})
|
|
{
|
|
# Put the new values into variables
|
|
my $scan_ipmitool_value_sensor_value = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_value_sensor_value};
|
|
my $scan_ipmitool_sensor_units = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_units};
|
|
my $scan_ipmitool_sensor_status = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_status};
|
|
my $scan_ipmitool_sensor_high_critical = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_critical};
|
|
my $scan_ipmitool_sensor_high_warning = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_warning};
|
|
my $scan_ipmitool_sensor_low_critical = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_critical};
|
|
my $scan_ipmitool_sensor_low_warning = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_warning};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
scan_ipmitool_sensor_name => $scan_ipmitool_sensor_name,
|
|
scan_ipmitool_value_sensor_value => $scan_ipmitool_value_sensor_value,
|
|
scan_ipmitool_sensor_units => $scan_ipmitool_sensor_units,
|
|
scan_ipmitool_sensor_status => $scan_ipmitool_sensor_status,
|
|
scan_ipmitool_sensor_high_critical => $scan_ipmitool_sensor_high_critical,
|
|
scan_ipmitool_sensor_high_warning => $scan_ipmitool_sensor_high_warning,
|
|
scan_ipmitool_sensor_low_critical => $scan_ipmitool_sensor_low_critical,
|
|
scan_ipmitool_sensor_low_warning => $scan_ipmitool_sensor_low_warning,
|
|
}});
|
|
|
|
if ((lc($scan_ipmitool_sensor_units) eq "c") or (lc($scan_ipmitool_sensor_units) eq "f"))
|
|
{
|
|
# Temperatures will be handled in process_temperature().
|
|
next;
|
|
}
|
|
if ($scan_ipmitool_sensor_status ne "ok")
|
|
{
|
|
my $health_source_name = $scan_ipmitool_sensor_units.":".$scan_ipmitool_sensor_name;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { health_source_name => $health_source_name }});
|
|
if (lc($scan_ipmitool_sensor_units) eq "v")
|
|
{
|
|
$health_source_name = "voltage:".$scan_ipmitool_sensor_name;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { health_source_name => $health_source_name }});
|
|
}
|
|
elsif (lc($scan_ipmitool_sensor_units) eq "w")
|
|
{
|
|
$health_source_name = "wattage:".$scan_ipmitool_sensor_name;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { health_source_name => $health_source_name }});
|
|
}
|
|
elsif (lc($scan_ipmitool_sensor_units) eq "rpm")
|
|
{
|
|
$health_source_name = "fan:".$scan_ipmitool_sensor_name;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { health_source_name => $health_source_name }});
|
|
}
|
|
elsif ($scan_ipmitool_sensor_units eq "%")
|
|
{
|
|
$health_source_name = "percentage:".$scan_ipmitool_sensor_name;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { health_source_name => $health_source_name }});
|
|
}
|
|
|
|
# Record it, if we're scanning ourselves.
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::scanning_myself" => $anvil->data->{sys}{scanning_myself} }});
|
|
if ($anvil->data->{sys}{scanning_myself})
|
|
{
|
|
# Set this to the user's requested weight, if set. Otherwise, use the default.
|
|
my $weight = $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{weight};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { weight => $weight }});
|
|
if (exists $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{weight})
|
|
{
|
|
# If it a number?
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::thresholds::${scan_ipmitool_sensor_name}::weight" => $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{weight},
|
|
}});
|
|
if (($anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{weight} =~ /^\d+$/) or ($anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{weight} =~ /^\d+\.\d+$/))
|
|
{
|
|
$weight = $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{weight};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { weight => $weight }});
|
|
}
|
|
}
|
|
|
|
# Default to '1' if not set.
|
|
$weight = 1 if not $weight;
|
|
|
|
$anvil->data->{'scan-ipmitool'}{health}{new}{$health_source_name} = $weight;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"health::new::$health_source_name" => $anvil->data->{'scan-ipmitool'}{health}{new}{$health_source_name},
|
|
}});
|
|
}
|
|
}
|
|
}
|
|
|
|
return(0);
|
|
}
|
|
|
|
# This takes the temperature sensors and feeds them into the 'temperature' table, deleting stale entries as needed.
|
|
sub process_temperature
|
|
{
|
|
my ($anvil, $host_name) = @_;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }});
|
|
|
|
$anvil->data->{'scan-ipmitool'}{queries} = [];
|
|
|
|
# First, read in all existing entries. We'll compare and UPDATE or INSERT as needed and DELETE any
|
|
# stale entries.
|
|
my $query = "
|
|
SELECT
|
|
temperature_uuid,
|
|
temperature_sensor_name,
|
|
temperature_value_c,
|
|
temperature_weight,
|
|
temperature_state,
|
|
temperature_is
|
|
FROM
|
|
temperature
|
|
WHERE
|
|
temperature_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
|
|
AND
|
|
temperature_agent_name = ".$anvil->Database->quote($THIS_FILE)."
|
|
AND
|
|
temperature_sensor_host = ".$anvil->Database->quote($host_name)."
|
|
;";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
|
|
|
|
# One or more records were found.
|
|
foreach my $row (@{$results})
|
|
{
|
|
my $scan_ipmitool_sensor_name = $row->[1];
|
|
$anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_uuid} = $row->[0];
|
|
$anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_value_c} = $row->[2];
|
|
$anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_weight} = $row->[3];
|
|
$anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_state} = $row->[4];
|
|
$anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_is} = $row->[5];
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"old::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_uuid" => $anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_uuid},
|
|
"old::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_value_c" => $anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_value_c},
|
|
"old::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_weight" => $anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_weight},
|
|
"old::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_state" => $anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_state},
|
|
"old::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_is" => $anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_is},
|
|
}});
|
|
}
|
|
|
|
# Look at the new values.
|
|
foreach my $scan_ipmitool_sensor_name (sort {$a cmp $b} keys %{$anvil->data->{new}{$host_name}{temperature}})
|
|
{
|
|
my $new_temperature_uuid = $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_uuid};
|
|
my $new_temperature_value_c = $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_value_c};
|
|
my $new_temperature_state = $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_state};
|
|
my $new_temperature_is = $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_is};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
scan_ipmitool_sensor_name => $scan_ipmitool_sensor_name,
|
|
new_temperature_uuid => $new_temperature_uuid,
|
|
new_temperature_value_c => $new_temperature_value_c,
|
|
new_temperature_state => $new_temperature_state,
|
|
new_temperature_is => $new_temperature_is,
|
|
}});
|
|
|
|
if ($new_temperature_value_c eq "na")
|
|
{
|
|
delete $anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name};
|
|
next;
|
|
}
|
|
|
|
# If the state is 'warning', set a health weight of 1 and set critical to 2.
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::scanning_myself" => $anvil->data->{sys}{scanning_myself} }});
|
|
my $new_temperature_weight = 1;
|
|
if ($anvil->data->{sys}{scanning_myself})
|
|
{
|
|
# What weight will we apply to this sensor?
|
|
$new_temperature_weight = $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{weight};
|
|
if (exists $anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_weight})
|
|
{
|
|
$new_temperature_weight = $anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_weight};
|
|
}
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_temperature_weight => $new_temperature_weight }});
|
|
if (exists $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{weight})
|
|
{
|
|
# If it a number?
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::thresholds::${scan_ipmitool_sensor_name}::weight" => $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{weight},
|
|
}});
|
|
if (($anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{weight} =~ /^\d+$/) or ($anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{weight} =~ /^\d+\.\d+$/))
|
|
{
|
|
$new_temperature_weight = $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{weight};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_temperature_weight => $new_temperature_weight }});
|
|
}
|
|
}
|
|
|
|
# Default to '1' if not set.
|
|
$new_temperature_weight = 1 if not $new_temperature_weight;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_temperature_weight => $new_temperature_weight }});
|
|
|
|
# If it's not OK, set the weight
|
|
if ($new_temperature_state ne "ok")
|
|
{
|
|
my $health_source_name = "temperature:".$scan_ipmitool_sensor_name;
|
|
$anvil->data->{'scan-ipmitool'}{health}{new}{$health_source_name} = $new_temperature_weight;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"health::new::$health_source_name" => $anvil->data->{'scan-ipmitool'}{health}{new}{$health_source_name},
|
|
}});
|
|
}
|
|
}
|
|
|
|
# Now see if the variable was seen before and, if so, if it changed.
|
|
my $temperature_uuid = "";
|
|
if (ref($anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}))
|
|
{
|
|
# Update the existing entry, if needed.
|
|
my $temperature_uuid = $anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_uuid};
|
|
}
|
|
|
|
# Generate and store the UUID.
|
|
my $sensor_host_uuid = $anvil->Get->host_uuid_from_name({host_name => $host_name});
|
|
if (not $sensor_host_uuid)
|
|
{
|
|
$sensor_host_uuid = $host_name;
|
|
}
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sensor_host_uuid => $sensor_host_uuid }});
|
|
$temperature_uuid = $anvil->Database->insert_or_update_temperature({
|
|
cache => $anvil->data->{'scan-ipmitool'}{queries},
|
|
debug => 2,
|
|
temperature_uuid => $temperature_uuid,
|
|
temperature_host_uuid => $anvil->Get->host_uuid,
|
|
temperature_agent_name => $THIS_FILE,
|
|
temperature_sensor_host => $sensor_host_uuid,
|
|
temperature_sensor_name => $scan_ipmitool_sensor_name,
|
|
temperature_value_c => $new_temperature_value_c,
|
|
temperature_state => $new_temperature_state,
|
|
temperature_is => $new_temperature_is,
|
|
temperature_weight => $new_temperature_weight,
|
|
});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { temperature_uuid => $temperature_uuid }});
|
|
|
|
# We still want this value, so delete it from the hash.
|
|
if (exists $anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name})
|
|
{
|
|
delete $anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name};
|
|
}
|
|
|
|
$anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_uuid} = $temperature_uuid;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_uuid" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_uuid},
|
|
}});
|
|
}
|
|
|
|
# Now, if any undeleted old entries remain, delete them from the database.
|
|
foreach my $scan_ipmitool_sensor_name (sort {$a cmp $b} keys %{$anvil->data->{old}{$host_name}{temperature}})
|
|
{
|
|
my $temperature_uuid = $anvil->data->{old}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_uuid};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { temperature_uuid => $temperature_uuid }});
|
|
|
|
$temperature_uuid = $anvil->Database->insert_or_update_temperature({
|
|
cache => $anvil->data->{'scan-ipmitool'}{queries},
|
|
debug => 2,
|
|
'delete' => 1,
|
|
temperature_uuid => $temperature_uuid,
|
|
});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { temperature_uuid => $temperature_uuid }});
|
|
}
|
|
|
|
# Now commit the changes.
|
|
$anvil->Database->write({query => $anvil->data->{'scan-ipmitool'}{queries}, source => $THIS_FILE, line => __LINE__});
|
|
|
|
return(0);
|
|
}
|
|
|
|
# This logs thermal sensor values that are outside nominal ranges
|
|
sub log_abnormal_temperatures
|
|
{
|
|
my ($anvil, $host_name, $scan_ipmitool_sensor_name) = @_;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
host_name => $host_name,
|
|
scan_ipmitool_sensor_name => $scan_ipmitool_sensor_name,
|
|
}});
|
|
|
|
my $new_scan_ipmitool_value_sensor_value = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_value_sensor_value};
|
|
my $new_scan_ipmitool_sensor_high_critical = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_critical};
|
|
my $new_scan_ipmitool_sensor_high_warning = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_warning};
|
|
my $new_scan_ipmitool_sensor_low_critical = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_critical};
|
|
my $new_scan_ipmitool_sensor_low_warning = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_warning};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
host_name => $host_name,
|
|
scan_ipmitool_sensor_name => $scan_ipmitool_sensor_name,
|
|
new_scan_ipmitool_value_sensor_value => $new_scan_ipmitool_value_sensor_value,
|
|
}});
|
|
|
|
# If the sensor value is 'na' or '', ignore it.
|
|
if (($new_scan_ipmitool_value_sensor_value eq "") or ($new_scan_ipmitool_value_sensor_value eq "na"))
|
|
{
|
|
return(0);
|
|
}
|
|
|
|
### High Warning
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::thresholds::defaults::high_warning" => $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{high_warning},
|
|
new_scan_ipmitool_sensor_high_warning => $new_scan_ipmitool_sensor_high_critical,
|
|
"scan-ipmitool::thresholds::${scan_ipmitool_sensor_name}::high_warning" => $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_warning},
|
|
}});
|
|
|
|
my $high_warning = $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{high_warning};
|
|
if ((defined $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_warning}) &&
|
|
( $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_warning} =~ /^\d/))
|
|
{
|
|
$high_warning = $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_warning};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { high_warning => $high_warning }});
|
|
}
|
|
elsif ($new_scan_ipmitool_sensor_high_critical =~ /^\d/)
|
|
{
|
|
$high_warning = $new_scan_ipmitool_sensor_high_critical;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { high_warning => $high_warning }});
|
|
}
|
|
|
|
### High Critical
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::thresholds::defaults::high_critical" => $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{high_critical},
|
|
new_scan_ipmitool_sensor_high_critical => $new_scan_ipmitool_sensor_high_critical,
|
|
"scan-ipmitool::thresholds::${scan_ipmitool_sensor_name}::high_critical" => $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_critical},
|
|
}});
|
|
|
|
my $high_critical = $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{high_critical};
|
|
if ((defined $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_critical}) &&
|
|
( $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_critical} =~ /^\d/))
|
|
{
|
|
$high_critical = $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_critical};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { high_critical => $high_critical }});
|
|
}
|
|
elsif ($new_scan_ipmitool_sensor_high_critical =~ /^\d/)
|
|
{
|
|
$high_critical = $new_scan_ipmitool_sensor_high_critical;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { high_critical => $high_critical }});
|
|
}
|
|
|
|
### Low Warning
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::thresholds::defaults::low_warning" => $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{low_warning},
|
|
new_scan_ipmitool_sensor_low_warning => $new_scan_ipmitool_sensor_low_warning,
|
|
"scan-ipmitool::thresholds::${scan_ipmitool_sensor_name}::low_warning" => $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_warning},
|
|
}});
|
|
|
|
my $low_warning = $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{low_warning};
|
|
if ((defined $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_warning}) &&
|
|
( $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_warning} =~ /^\d/))
|
|
{
|
|
$low_warning = $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_warning};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { low_warning => $low_warning }});
|
|
}
|
|
elsif ($new_scan_ipmitool_sensor_low_critical =~ /^\d/)
|
|
{
|
|
$low_warning = $new_scan_ipmitool_sensor_low_critical;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { low_warning => $low_warning }});
|
|
}
|
|
|
|
### Low Critical
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::thresholds::defaults::low_critical" => $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{low_critical},
|
|
new_scan_ipmitool_sensor_low_critical => $new_scan_ipmitool_sensor_low_critical,
|
|
"scan-ipmitool::thresholds::${scan_ipmitool_sensor_name}::low_critical" => $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_critical},
|
|
}});
|
|
my $low_critical = $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{low_critical};
|
|
if ((defined $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_critical}) &&
|
|
( $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_critical} =~ /^\d/))
|
|
{
|
|
$low_critical = $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_critical};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { low_critical => $low_critical }});
|
|
}
|
|
elsif ($new_scan_ipmitool_sensor_low_critical =~ /^\d/)
|
|
{
|
|
$low_critical = $new_scan_ipmitool_sensor_low_critical;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { low_critical => $low_critical }});
|
|
}
|
|
|
|
# Record the levels
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
host_name => $host_name,
|
|
scan_ipmitool_sensor_name => $scan_ipmitool_sensor_name,
|
|
high_warning => $high_warning,
|
|
high_critical => $high_critical,
|
|
low_warning => $low_warning,
|
|
low_critical => $low_critical,
|
|
new_scan_ipmitool_value_sensor_value => $new_scan_ipmitool_value_sensor_value,
|
|
}});
|
|
|
|
# Record the temperatures.
|
|
if ($new_scan_ipmitool_value_sensor_value < $low_critical)
|
|
{
|
|
# Setup the 'temperature' entry.
|
|
$anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name} = {
|
|
temperature_value_c => $new_scan_ipmitool_value_sensor_value,
|
|
temperature_state => 'critical',
|
|
temperature_is => 'low',
|
|
};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_value_c" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_value_c},
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_state" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_state},
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_is" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_is},
|
|
}});
|
|
}
|
|
elsif ($new_scan_ipmitool_value_sensor_value < $low_warning)
|
|
{
|
|
# Setup the 'temperature' entry.
|
|
$anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name} = {
|
|
temperature_value_c => $new_scan_ipmitool_value_sensor_value,
|
|
temperature_state => 'warning',
|
|
temperature_is => 'low',
|
|
};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_value_c" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_value_c},
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_state" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_state},
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_is" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_is},
|
|
}});
|
|
}
|
|
elsif ($new_scan_ipmitool_value_sensor_value > $high_critical)
|
|
{
|
|
# Setup the 'temperature' entry.
|
|
$anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name} = {
|
|
temperature_value_c => $new_scan_ipmitool_value_sensor_value,
|
|
temperature_state => 'critical',
|
|
temperature_is => 'high',
|
|
};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_value_c" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_value_c},
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_state" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_state},
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_is" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_is},
|
|
}});
|
|
}
|
|
elsif ($new_scan_ipmitool_value_sensor_value > $high_warning)
|
|
{
|
|
# Setup the 'temperature' entry.
|
|
$anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name} = {
|
|
temperature_value_c => $new_scan_ipmitool_value_sensor_value,
|
|
temperature_state => 'warning',
|
|
temperature_is => 'high',
|
|
};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_value_c" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_value_c},
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_state" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_state},
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_is" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_is},
|
|
}});
|
|
}
|
|
else
|
|
{
|
|
$anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name} = {
|
|
temperature_value_c => $new_scan_ipmitool_value_sensor_value,
|
|
temperature_state => 'ok',
|
|
temperature_is => 'nominal',
|
|
};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_value_c" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_value_c},
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_state" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_state},
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_is" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_is},
|
|
}});
|
|
}
|
|
|
|
### NOTE: Recording all temps now, so this might not be needed
|
|
# When the dashboard scans a node, it needs to know about the Ambient and Systemboard temperatures in
|
|
# order to decide whether the node is safe too boot back up or not. So if this host_name is a
|
|
# dashboard, log the 'Ambient' and 'Systemboard' temperatures (or whatever the user defined) as
|
|
# 'good', if they're not already in the 'new::temperature::x' hash.
|
|
my $host_type = $anvil->Get->host_type();
|
|
foreach my $sensor (split/,/, $anvil->data->{'scan-ipmitool'}{offline_sensor_list})
|
|
{
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
sensor => $sensor,
|
|
scan_ipmitool_sensor_name => $scan_ipmitool_sensor_name,
|
|
host_type => $host_type,
|
|
}});
|
|
if (($scan_ipmitool_sensor_name eq $sensor) &&
|
|
($host_type eq "striker") &&
|
|
(not exists $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}))
|
|
{
|
|
$anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name} = {
|
|
temperature_value_c => $new_scan_ipmitool_value_sensor_value,
|
|
temperature_state => 'ok',
|
|
temperature_is => 'nominal',
|
|
};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_value_c" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_value_c},
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_state" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_state},
|
|
"new::${host_name}::temperature::${scan_ipmitool_sensor_name}::temperature_is" => $anvil->data->{new}{$host_name}{temperature}{$scan_ipmitool_sensor_name}{temperature_is},
|
|
}});
|
|
}
|
|
}
|
|
|
|
return(0);
|
|
}
|
|
|
|
# This processes a temperature sensor, handling changes and recording values that are in a warning or
|
|
# critical state.
|
|
sub process_temperature_change
|
|
{
|
|
my ($anvil, $host_name, $scan_ipmitool_sensor_name) = @_;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
host_name => $host_name,
|
|
scan_ipmitool_sensor_name => $scan_ipmitool_sensor_name,
|
|
}});
|
|
|
|
# This is a repeat of some variables set before this function was called, but setting them here again
|
|
# saves passing in a pile of variables.
|
|
my $level = "info";
|
|
my $message_key = "scan_ipmitool_message_0005";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
level => $level,
|
|
message_key => $message_key,
|
|
}});
|
|
|
|
# New values
|
|
my $new_scan_ipmitool_value_sensor_value = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_value_sensor_value};
|
|
my $new_scan_ipmitool_sensor_high_critical = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_critical};
|
|
my $new_scan_ipmitool_sensor_high_warning = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_warning};
|
|
my $new_scan_ipmitool_sensor_low_critical = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_critical};
|
|
my $new_scan_ipmitool_sensor_low_warning = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_warning};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
new_scan_ipmitool_value_sensor_value => $new_scan_ipmitool_value_sensor_value,
|
|
new_scan_ipmitool_sensor_high_critical => $new_scan_ipmitool_sensor_high_critical,
|
|
new_scan_ipmitool_sensor_high_warning => $new_scan_ipmitool_sensor_high_warning,
|
|
new_scan_ipmitool_sensor_low_critical => $new_scan_ipmitool_sensor_low_critical,
|
|
new_scan_ipmitool_sensor_low_warning => $new_scan_ipmitool_sensor_low_warning,
|
|
}});
|
|
|
|
# If the value is "" and it's a digit-based value, switch it to '0'
|
|
if ($new_scan_ipmitool_value_sensor_value eq "")
|
|
{
|
|
$new_scan_ipmitool_value_sensor_value = 0;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_scan_ipmitool_value_sensor_value => $new_scan_ipmitool_value_sensor_value }});
|
|
}
|
|
|
|
# Old value, if it exists.
|
|
my $old_scan_ipmitool_value_sensor_value = $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_value_sensor_value} ? $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_value_sensor_value} : 0;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"sql::${host_name}::scan_ipmitool_sensor_name::${scan_ipmitool_sensor_name}::scan_ipmitool_value_sensor_value" => $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_value_sensor_value},
|
|
old_scan_ipmitool_value_sensor_value => $old_scan_ipmitool_value_sensor_value,
|
|
}});
|
|
|
|
### Buffer, used for clearing all alerts.
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::thresholds::defaults::buffer" => $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{buffer},
|
|
"scan-ipmitool::thresholds::${scan_ipmitool_sensor_name}::buffer" => $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{buffer},
|
|
}});
|
|
my $buffer = $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{buffer} ? $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{buffer} : $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{buffer};
|
|
|
|
### High Warning
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::thresholds::defaults::high_warning" => $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{high_warning},
|
|
new_scan_ipmitool_sensor_high_warning => $new_scan_ipmitool_sensor_high_critical,
|
|
"scan-ipmitool::thresholds::${scan_ipmitool_sensor_name}::high_warning" => $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_warning},
|
|
}});
|
|
my $high_warning = $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{high_warning};
|
|
|
|
if ((defined $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_warning}) &&
|
|
($anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_warning} =~ /^\d/))
|
|
{
|
|
$high_warning = $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_warning};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { high_warning => $high_warning }});
|
|
}
|
|
elsif ($new_scan_ipmitool_sensor_high_critical =~ /^\d/)
|
|
{
|
|
$high_warning = $new_scan_ipmitool_sensor_high_critical;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { high_warning => $high_warning }});
|
|
}
|
|
|
|
### High Critical
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::thresholds::defaults::high_critical" => $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{high_critical},
|
|
new_scan_ipmitool_sensor_high_critical => $new_scan_ipmitool_sensor_high_critical,
|
|
"scan-ipmitool::thresholds::${scan_ipmitool_sensor_name}::high_critical" => $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_critical},
|
|
}});
|
|
my $high_critical = $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{high_critical};
|
|
if ((defined $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_critical}) &&
|
|
($anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_critical} =~ /^\d/))
|
|
{
|
|
$high_critical = $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{high_critical};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { high_critical => $high_critical }});
|
|
}
|
|
elsif ($new_scan_ipmitool_sensor_high_critical =~ /^\d/)
|
|
{
|
|
$high_critical = $new_scan_ipmitool_sensor_high_critical;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { high_critical => $high_critical }});
|
|
}
|
|
|
|
### Low Warning
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::thresholds::defaults::low_warning" => $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{low_warning},
|
|
new_scan_ipmitool_sensor_low_warning => $new_scan_ipmitool_sensor_low_warning,
|
|
"scan-ipmitool::thresholds::${scan_ipmitool_sensor_name}::low_warning" => $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_warning},
|
|
}});
|
|
my $low_warning = $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{low_warning};
|
|
if ((defined $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_warning}) &&
|
|
($anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_warning} =~ /^\d/))
|
|
{
|
|
$low_warning = $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_warning};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { low_warning => $low_warning }});
|
|
}
|
|
elsif ($new_scan_ipmitool_sensor_low_critical =~ /^\d/)
|
|
{
|
|
$low_warning = $new_scan_ipmitool_sensor_low_critical;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { low_warning => $low_warning }});
|
|
}
|
|
|
|
### Low Critical
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::thresholds::defaults::low_critical" => $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{low_critical},
|
|
new_scan_ipmitool_sensor_low_critical => $new_scan_ipmitool_sensor_low_critical,
|
|
"scan-ipmitool::thresholds::${scan_ipmitool_sensor_name}::low_critical" => $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_critical},
|
|
}});
|
|
my $low_critical = $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{low_critical};
|
|
if ((defined $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_critical}) &&
|
|
($anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_critical} =~ /^\d/))
|
|
{
|
|
$low_critical = $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{low_critical};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { low_critical => $low_critical }});
|
|
}
|
|
elsif ($new_scan_ipmitool_sensor_low_critical =~ /^\d/)
|
|
{
|
|
$low_critical = $new_scan_ipmitool_sensor_low_critical;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { low_critical => $low_critical }});
|
|
}
|
|
|
|
### Jump delta
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::thresholds::defaults::jump" => $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{jump},
|
|
"scan-ipmitool::thresholds::${scan_ipmitool_sensor_name}::jump" => $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{jump},
|
|
}});
|
|
my $jump = $anvil->data->{'scan-ipmitool'}{thresholds}{'default'}{jump};
|
|
if ($anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{jump})
|
|
{
|
|
$jump = $anvil->data->{'scan-ipmitool'}{thresholds}{$scan_ipmitool_sensor_name}{jump};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { jump => $jump }});
|
|
}
|
|
|
|
# We append the hex address some sensor names when there are duplicates. This deals with those.
|
|
if ($scan_ipmitool_sensor_name =~ /^(.*?) \(.*?h\)$/)
|
|
{
|
|
my $jump_name = $1;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { jump_name => $jump_name }});
|
|
if ((exists $anvil->data->{'scan-ipmitool'}{thresholds}{$jump_name}) && ($anvil->data->{'scan-ipmitool'}{thresholds}{$jump_name}{jump}))
|
|
{
|
|
$jump = $anvil->data->{'scan-ipmitool'}{thresholds}{$jump_name}{jump};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { jump => $jump }});
|
|
}
|
|
}
|
|
|
|
# Final levels
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
host_name => $host_name,
|
|
scan_ipmitool_sensor_name => $scan_ipmitool_sensor_name,
|
|
high_warning => $high_warning,
|
|
high_critical => $high_critical,
|
|
low_warning => $low_warning,
|
|
low_critical => $low_critical,
|
|
jump => $jump,
|
|
buffer => $buffer,
|
|
}});
|
|
|
|
# Now, has the value climbed, fallen or stayed the same? Pretend it is rising if it is a new sensor.
|
|
if (($new_scan_ipmitool_value_sensor_value > $old_scan_ipmitool_value_sensor_value) || (not $old_scan_ipmitool_value_sensor_value))
|
|
{
|
|
### Rising.
|
|
#
|
|
# If it is over the high critical level, clear the 'warning', set the 'critical' and, if
|
|
# needed, add the entry to 'temperature'.
|
|
#
|
|
# If it is over the high warning, check and set the warning.
|
|
#
|
|
# If it is over the low warning, check and clear both critical and warning, and clear the
|
|
# 'temperature' entry, if needed.
|
|
#
|
|
# If it is over the low critical, but not low warning, clear the critical and set the warning.
|
|
# Clear the 'temperature' entry, if it exists.
|
|
#
|
|
# If the temperature jumped above the set per-cycle change limit, trigger an alarm no matter
|
|
# what.
|
|
if ($new_scan_ipmitool_value_sensor_value > $high_critical)
|
|
{
|
|
# We've gone critical. If it was previously 'warning', clear them.
|
|
foreach my $type ("temperature_high_warning", "temperature_low_warning", "temperature_low_critical")
|
|
{
|
|
my $record_locator = $host_name.":".$scan_ipmitool_sensor_name.":".$type;
|
|
my $changed = $anvil->Alert->check_alert_sent({clear => 1, record_locator => $record_locator, set_by => $THIS_FILE});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
record_locator => $record_locator,
|
|
changed => $changed,
|
|
}});
|
|
}
|
|
|
|
# Set the critical warning.
|
|
my $changed = $anvil->Alert->check_alert_sent({record_locator => $host_name.":".$scan_ipmitool_sensor_name.":temperature_high_critical", set_by => $THIS_FILE});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { changed => $changed }});
|
|
|
|
# If set, alert the user and register with 'temperature'.
|
|
if ($changed)
|
|
{
|
|
# This is the first time we climbed. Send an alert and then register the
|
|
# entry in the database's 'temperature' table.
|
|
$level = "critical";
|
|
$message_key = "scan_ipmitool_message_0006";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
level => $level,
|
|
message_key => $message_key,
|
|
}});
|
|
}
|
|
}
|
|
elsif ($new_scan_ipmitool_value_sensor_value > $high_warning)
|
|
{
|
|
# The temp is rising, so the 'high_critical' should not be set, but check/clear it
|
|
# anyway to be safe.
|
|
$anvil->Alert->check_alert_sent({clear => 1, record_locator => $host_name.":".$scan_ipmitool_sensor_name.":temperature_high_critical", set_by => $THIS_FILE});
|
|
my $changed = $anvil->Alert->check_alert_sent({clear => 0, record_locator => $host_name.":".$scan_ipmitool_sensor_name.":temperature_high_warning", set_by => $THIS_FILE});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { changed => $changed }});
|
|
if ($changed)
|
|
{
|
|
# Tell the user that the temperature is rising.
|
|
$level = "warning";
|
|
$message_key = "scan_ipmitool_message_0007";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
level => $level,
|
|
message_key => $message_key,
|
|
}});
|
|
}
|
|
}
|
|
elsif ($new_scan_ipmitool_value_sensor_value > ($low_warning + $buffer))
|
|
{
|
|
# If there was a 'low_warning' or 'low_critical', clear it and tell the user that
|
|
# we're OK now.
|
|
my $clear_critical = $anvil->Alert->check_alert_sent({clear => 1, record_locator => $host_name.":".$scan_ipmitool_sensor_name.":temperature_low_critical", set_by => $THIS_FILE});
|
|
my $clear_warning = $anvil->Alert->check_alert_sent({clear => 1, record_locator => $host_name.":".$scan_ipmitool_sensor_name.":temperature_low_warning", set_by => $THIS_FILE});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
clear_critical => $clear_critical,
|
|
clear_warning => $clear_warning,
|
|
}});
|
|
if (($clear_critical) or ($clear_warning))
|
|
{
|
|
# Tell the user we're OK again. Whether this is a 'critical' or 'warning'
|
|
# level alert depends on what it was before. We want people who only get
|
|
# critical alerts to know we're OK if they got the critical alarm.
|
|
$level = $clear_critical ? "critical" : "warning";
|
|
$message_key = "scan_ipmitool_message_0008";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
level => $level,
|
|
message_key => $message_key,
|
|
}});
|
|
}
|
|
}
|
|
elsif ($new_scan_ipmitool_value_sensor_value > ($low_critical + $buffer))
|
|
{
|
|
# It has risen above critically low levels.
|
|
my $clear_critical = $anvil->Alert->check_alert_sent({clear => 1, record_locator => $host_name.":".$scan_ipmitool_sensor_name.":temperature_low_critical", set_by => $THIS_FILE});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { clear_critical => $clear_critical }});
|
|
if ($clear_critical)
|
|
{
|
|
# Tell the user we're getting better.
|
|
$level = "critical";
|
|
$message_key = "scan_ipmitool_message_0009";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
level => $level,
|
|
message_key => $message_key,
|
|
}});
|
|
}
|
|
}
|
|
else
|
|
{
|
|
# it is floating in the safe area.
|
|
}
|
|
|
|
# Check for a jump now, if we have an old value.
|
|
if ($old_scan_ipmitool_value_sensor_value)
|
|
{
|
|
my $delta = ($new_scan_ipmitool_value_sensor_value - $old_scan_ipmitool_value_sensor_value);
|
|
if ($delta > $jump)
|
|
{
|
|
# Temperature jumped. This is a stand-alone 'warning' level alarm.
|
|
my $sensor_name = "name=$scan_ipmitool_sensor_name:units=C";
|
|
my $new_sensor_value = "value=$new_scan_ipmitool_value_sensor_value:units=C";
|
|
my $old_sensor_value = "value=$old_scan_ipmitool_value_sensor_value:units=C";
|
|
my $variables = {
|
|
sensor_name => $sensor_name,
|
|
new_sensor_value => $new_sensor_value,
|
|
old_sensor_value => $old_sensor_value
|
|
};
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "scan_ipmitool_message_0010", variables => $variables});
|
|
$anvil->Alert->register({alert_level => "warning", message => "scan_ipmitool_message_0010", variables => $variables, sort_position => 1, set_by => $THIS_FILE});
|
|
}
|
|
}
|
|
}
|
|
elsif ($new_scan_ipmitool_value_sensor_value < $old_scan_ipmitool_value_sensor_value)
|
|
{
|
|
### Falling.
|
|
#
|
|
# If it is below the high critical, but not high warning, clear the critical and set the
|
|
# warning. Clear the 'temperature' entry, if it exists.
|
|
#
|
|
# If it is below the high warning, check and clear both critical and warning, and clear the
|
|
# 'temperature' entry, if needed.
|
|
#
|
|
# If it is below the low critical level, clear the 'warning', set the 'critical' and, if
|
|
# needed, add the entry to 'temperature'.
|
|
#
|
|
# If it is below the low warning, check and set the warning.
|
|
if ($new_scan_ipmitool_value_sensor_value < $low_critical)
|
|
{
|
|
# We've gone critical. Clear previous alerts...
|
|
foreach my $type ("temperature_high_critical", "temperature_high_warning", "temperature_low_warning")
|
|
{
|
|
my $record_locator = $host_name.":".$scan_ipmitool_sensor_name.":".$type;
|
|
my $changed = $anvil->Alert->check_alert_sent({clear => 1, record_locator => $record_locator, set_by => $THIS_FILE});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
record_locator => $record_locator,
|
|
changed => $changed,
|
|
}});
|
|
}
|
|
|
|
# Now set the critical warning.
|
|
my $changed = $anvil->Alert->check_alert_sent({record_locator => $host_name.":".$scan_ipmitool_sensor_name.":temperature_low_critical", set_by => $THIS_FILE});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { changed => $changed }});
|
|
if ($changed)
|
|
{
|
|
# This is the first time we climbed. Send an alert and then register the
|
|
# entry in the database's 'temperature' table.
|
|
$level = "critical";
|
|
$message_key = "scan_ipmitool_message_0011";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
level => $level,
|
|
message_key => $message_key,
|
|
}});
|
|
}
|
|
}
|
|
elsif ($new_scan_ipmitool_value_sensor_value < $low_warning)
|
|
{
|
|
# The temp is dropping, so the 'low_critical' should not be set, but check/clear it
|
|
# anyway to be safe.
|
|
$anvil->Alert->check_alert_sent({clear => 1, record_locator => $host_name.":".$scan_ipmitool_sensor_name.":temperature_low_critical", set_by => $THIS_FILE});
|
|
my $changed = $anvil->Alert->check_alert_sent({clear => 0, record_locator => $host_name.":".$scan_ipmitool_sensor_name.":temperature_low_warning", set_by => $THIS_FILE});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { changed => $changed }});
|
|
if ($changed)
|
|
{
|
|
# Tell the user that the temperature is rising.
|
|
$level = "warning";
|
|
$message_key = "scan_ipmitool_message_0012";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
level => $level,
|
|
message_key => $message_key,
|
|
}});
|
|
}
|
|
}
|
|
elsif ($new_scan_ipmitool_value_sensor_value < ($high_warning - $buffer))
|
|
{
|
|
# If there was a 'high_warning' or 'high_critical', clear it and tell the user that
|
|
# we're OK now.
|
|
my $clear_critical = $anvil->Alert->check_alert_sent({clear => 1, record_locator => $host_name.":".$scan_ipmitool_sensor_name.":temperature_high_critical", set_by => $THIS_FILE});
|
|
my $clear_warning = $anvil->Alert->check_alert_sent({clear => 1, record_locator => $host_name.":".$scan_ipmitool_sensor_name.":temperature_high_warning", set_by => $THIS_FILE});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
clear_critical => $clear_critical,
|
|
clear_warning => $clear_warning,
|
|
}});
|
|
if (($clear_critical) or ($clear_warning))
|
|
{
|
|
# Tell the user we're OK again. Whether this is a 'critical' or 'warning'
|
|
# level alert depends on what it was before. We want people who only get
|
|
# critical alerts to know we're OK if they got the critical alarm.
|
|
$level = $clear_critical ? "critical" : "warning";
|
|
$message_key = "scan_ipmitool_message_0013";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
level => $level,
|
|
message_key => $message_key,
|
|
}});
|
|
}
|
|
}
|
|
elsif ($new_scan_ipmitool_value_sensor_value < ($high_critical - $buffer))
|
|
{
|
|
# It is below critically high levels.
|
|
my $clear_critical = $anvil->Alert->check_alert_sent({clear => 1, record_locator => $host_name.":".$scan_ipmitool_sensor_name.":temperature_low_critical", set_by => $THIS_FILE});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { clear_critical => $clear_critical }});
|
|
if ($clear_critical)
|
|
{
|
|
# Tell the user we're OK again.
|
|
$level = "critical";
|
|
$message_key = "scan_ipmitool_message_0014";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
level => $level,
|
|
message_key => $message_key,
|
|
}});
|
|
}
|
|
}
|
|
else
|
|
{
|
|
# it is floating in the safe area.
|
|
}
|
|
}
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
level => $level,
|
|
message_key => $message_key,
|
|
}});
|
|
return($level, $message_key);
|
|
}
|
|
|
|
# This reads in the last scan's data.
|
|
sub read_last_scan
|
|
{
|
|
my ($anvil, $host_name) = @_;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }});
|
|
|
|
# Make sure I don't have any stray data for this host_name.
|
|
if (exists $anvil->data->{sql}{$host_name})
|
|
{
|
|
delete $anvil->data->{sql}{$host_name};
|
|
}
|
|
|
|
# Read in existing data, if any.
|
|
my $query = "
|
|
SELECT
|
|
a.scan_ipmitool_uuid,
|
|
a.scan_ipmitool_sensor_name,
|
|
a.scan_ipmitool_sensor_units,
|
|
a.scan_ipmitool_sensor_status,
|
|
a.scan_ipmitool_sensor_high_critical,
|
|
a.scan_ipmitool_sensor_high_warning,
|
|
a.scan_ipmitool_sensor_low_critical,
|
|
a.scan_ipmitool_sensor_low_warning,
|
|
b.scan_ipmitool_value_sensor_value
|
|
FROM
|
|
scan_ipmitool a,
|
|
scan_ipmitool_values b
|
|
WHERE
|
|
a.scan_ipmitool_uuid = b.scan_ipmitool_value_scan_ipmitool_uuid
|
|
AND
|
|
a.scan_ipmitool_sensor_host = ".$anvil->Database->quote($host_name)."
|
|
AND
|
|
a.scan_ipmitool_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
|
|
;";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
|
|
my $count = @{$results};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
results => $results,
|
|
count => $count,
|
|
}});
|
|
# One or more records were found.
|
|
foreach my $row (@{$results})
|
|
{
|
|
my $scan_ipmitool_uuid = $row->[0];
|
|
my $scan_ipmitool_sensor_name = $row->[1];
|
|
my $scan_ipmitool_sensor_units = $row->[2];
|
|
my $scan_ipmitool_sensor_status = $row->[3];
|
|
my $scan_ipmitool_sensor_high_critical = $row->[4];
|
|
my $scan_ipmitool_sensor_high_warning = $row->[5];
|
|
my $scan_ipmitool_sensor_low_critical = $row->[6];
|
|
my $scan_ipmitool_sensor_low_warning = $row->[7];
|
|
my $scan_ipmitool_value_sensor_value = $row->[8];
|
|
|
|
$anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_uuid} = $scan_ipmitool_uuid;
|
|
$anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_units} = $scan_ipmitool_sensor_units;
|
|
$anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_status} = $scan_ipmitool_sensor_status;
|
|
$anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_critical} = $scan_ipmitool_sensor_high_critical ? $scan_ipmitool_sensor_high_critical : "";
|
|
$anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_warning} = $scan_ipmitool_sensor_high_warning ? $scan_ipmitool_sensor_high_warning : "";
|
|
$anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_critical} = $scan_ipmitool_sensor_low_critical ? $scan_ipmitool_sensor_low_critical : "";
|
|
$anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_warning} = $scan_ipmitool_sensor_low_warning ? $scan_ipmitool_sensor_low_warning : "";
|
|
$anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_value_sensor_value} = $scan_ipmitool_value_sensor_value;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"sql::${host_name}::scan_ipmitool_sensor_name::${scan_ipmitool_sensor_name}::scan_ipmitool_uuid" => $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_uuid},
|
|
"sql::${host_name}::scan_ipmitool_sensor_name::${scan_ipmitool_sensor_name}::scan_ipmitool_sensor_units" => $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_units},
|
|
"sql::${host_name}::scan_ipmitool_sensor_name::${scan_ipmitool_sensor_name}::scan_ipmitool_sensor_status" => $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_status},
|
|
"sql::${host_name}::scan_ipmitool_sensor_name::${scan_ipmitool_sensor_name}::scan_ipmitool_sensor_high_critical" => $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_critical},
|
|
"sql::${host_name}::scan_ipmitool_sensor_name::${scan_ipmitool_sensor_name}::scan_ipmitool_sensor_high_warning" => $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_high_warning},
|
|
"sql::${host_name}::scan_ipmitool_sensor_name::${scan_ipmitool_sensor_name}::scan_ipmitool_sensor_low_critical" => $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_critical},
|
|
"sql::${host_name}::scan_ipmitool_sensor_name::${scan_ipmitool_sensor_name}::scan_ipmitool_sensor_low_warning" => $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_sensor_low_warning},
|
|
"sql::${host_name}::scan_ipmitool_sensor_name::${scan_ipmitool_sensor_name}::scan_ipmitool_value_sensor_value" => $anvil->data->{sql}{$host_name}{scan_ipmitool_sensor_name}{$scan_ipmitool_sensor_name}{scan_ipmitool_value_sensor_value},
|
|
}});
|
|
}
|
|
|
|
return($count);
|
|
}
|
|
|
|
# Query IPMI targets. Unreachable targets will simply be ignored.
|
|
sub query_ipmi_targets
|
|
{
|
|
my ($anvil) = @_;
|
|
|
|
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{'scan-ipmitool'}{host_name}})
|
|
{
|
|
my $ipmitool_command = $anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmitool_command};
|
|
my $ipmi_password = $anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmi_password};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
host_name => $host_name,
|
|
ipmitool_command => $ipmitool_command,
|
|
ipmi_password => $anvil->Log->is_secure($ipmi_password),
|
|
}});
|
|
|
|
# Time the call.
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_ipmitool_log_0001", variables => { host_name => $host_name }});
|
|
|
|
# This will call, parse and store the information in (host_name == full host name;
|
|
# ipmi::<host_name>::scan_ipmitool_sensor_name::$sensor_name::scan_ipmitool_value_sensor_value
|
|
# ipmi::<host_name>::scan_ipmitool_sensor_name::$sensor_name::scan_ipmitool_sensor_units
|
|
# ipmi::<host_name>::scan_ipmitool_sensor_name::$sensor_name::scan_ipmitool_sensor_status
|
|
# ipmi::<host_name>::scan_ipmitool_sensor_name::$sensor_name::scan_ipmitool_sensor_high_critical
|
|
# ipmi::<host_name>::scan_ipmitool_sensor_name::$sensor_name::scan_ipmitool_sensor_high_warning
|
|
# ipmi::<host_name>::scan_ipmitool_sensor_name::$sensor_name::scan_ipmitool_sensor_low_critical
|
|
# ipmi::<host_name>::scan_ipmitool_sensor_name::$sensor_name::scan_ipmitool_sensor_low_warning
|
|
$anvil->System->collect_ipmi_data({
|
|
debug => 3,
|
|
host_name => $host_name,
|
|
ipmitool_command => $ipmitool_command,
|
|
ipmi_password => $ipmi_password,
|
|
});
|
|
|
|
# Analyze temperature sensors if this is our own data.
|
|
if (($host_name eq $anvil->Get->host_name) or ($host_name eq $anvil->Get->short_host_name))
|
|
{
|
|
foreach my $sensor_name (sort {$a cmp $b} keys %{$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}})
|
|
{
|
|
my $units = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$sensor_name}{scan_ipmitool_sensor_units};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
sensor_name => $sensor_name,
|
|
units => $units,
|
|
}});
|
|
|
|
# If this is a temperature, check to see if it is outside its nominal range and, if
|
|
# so, record it into a hash for loading into ScanCore's 'temperature' table.
|
|
if ($units eq "C")
|
|
{
|
|
log_abnormal_temperatures($anvil, $host_name, $sensor_name);
|
|
}
|
|
}
|
|
}
|
|
|
|
my $psu_count = 0;
|
|
foreach my $sensor_name (sort {$a cmp $b} keys %{$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}})
|
|
{
|
|
if ($sensor_name =~ /^PSU(\d+)/)
|
|
{
|
|
my $this_psu = $1;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { this_psu => $this_psu }});
|
|
if ($this_psu > $psu_count)
|
|
{
|
|
$psu_count = $this_psu;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { psu_count => $psu_count }});
|
|
}
|
|
}
|
|
}
|
|
|
|
# If a PSU is OK, but its wattage is 0, input power was lost. We'll switch the PSU state to
|
|
# ensure this sets a health value. We'll check for five PSUs, though very very few should
|
|
# have more than 2.
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { psu_count => $psu_count }});
|
|
foreach my $i (1..$psu_count)
|
|
{
|
|
my $psu_key = "PSU".$i;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
i => $i,
|
|
psu_key => $psu_key,
|
|
}});
|
|
|
|
if (exists $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$psu_key})
|
|
{
|
|
# TODO: Is this the key for HP machines, too?
|
|
my $wattage_key = $psu_key." Power";
|
|
my $psu_value = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$psu_key}{scan_ipmitool_value_sensor_value};
|
|
my $psu_status = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$psu_key}{scan_ipmitool_sensor_status};
|
|
my $psu_wattage = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$wattage_key}{scan_ipmitool_value_sensor_value};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
wattage_key => $wattage_key,
|
|
psu_value => $psu_value,
|
|
psu_status => $psu_status,
|
|
psu_wattage => $psu_wattage,
|
|
}});
|
|
|
|
if ((lc($psu_status) eq "ok") && (lc($psu_wattage) eq "no reading"))
|
|
{
|
|
# Change the status to 'no signal'
|
|
$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$psu_key}{scan_ipmitool_sensor_status} = "ns";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"ipmi::${host_name}::scan_ipmitool_sensor_name::${psu_key}::scan_ipmitool_sensor_status" => $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$psu_key}{scan_ipmitool_sensor_status},
|
|
}});
|
|
}
|
|
}
|
|
else
|
|
{
|
|
# This PSU doesn't exist, we're done.
|
|
last;
|
|
}
|
|
}
|
|
}
|
|
|
|
return(0);
|
|
}
|
|
|
|
# This looks for IPMI targets to scan.
|
|
sub find_ipmi_targets
|
|
{
|
|
my ($anvil) = @_;
|
|
|
|
# This will keep track of how many IPMI targets we find.
|
|
my $ipmi_targets = 0;
|
|
|
|
# If I am a node, I will only scan myself.
|
|
my $host_type = $anvil->Get->host_type();
|
|
my $host_name = $anvil->Get->host_name();
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
host_type => $host_type,
|
|
host_name => $host_name,
|
|
}});
|
|
|
|
# Do I have local IPMI access?
|
|
if ((-e '/dev/ipmi0') or (-e '/dev/ipmi/0') or (-e '/dev/ipmidev/0'))
|
|
{
|
|
my ($output, $return_code) = $anvil->System->call({timeout => 30, shell_call => $anvil->data->{path}{exe}{ipmitool}." chassis power status"});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
output => $output,
|
|
return_code => $return_code,
|
|
}});
|
|
if (not $return_code)
|
|
{
|
|
# We're good.
|
|
$ipmi_targets++;
|
|
my $host_name = $anvil->Get->host_name();
|
|
$anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{host_ipmi} = "";
|
|
$anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmitool_command} = $anvil->data->{path}{exe}{ipmitool};
|
|
$anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmi_password} = "";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::host_name::${host_name}::host_ipmi" => $anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{host_ipmi},
|
|
"scan-ipmitool::host_name::${host_name}::ipmitool_command" => $anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmitool_command},
|
|
"scan-ipmitool::host_name::${host_name}::ipmi_password" => $anvil->Log->is_secure($anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmi_password}),
|
|
}});
|
|
}
|
|
}
|
|
|
|
### NOTE: This is changed. Now that striker's directly call the targets to check temperature when
|
|
### evaluating thermal boot conditions, we don't need to store this data.
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ipmi_targets => $ipmi_targets }});
|
|
return($ipmi_targets);
|
|
|
|
# Which hosts we scan depends on if we're a Striker dashboard or not. If we are, we'll try to scan
|
|
# all machines in all Anvil! systems. Otherwise, we only scan ourselves.
|
|
=cut
|
|
if ($host_type ne "striker")
|
|
{
|
|
# We're not a dashboard, so we don't scan others.
|
|
return($ipmi_targets);
|
|
}
|
|
|
|
# Loop through Anvil! systems.
|
|
my $query = "
|
|
SELECT
|
|
anvil_name,
|
|
anvil_node1_host_uuid,
|
|
anvil_node2_host_uuid
|
|
FROM
|
|
anvils
|
|
WHERE
|
|
anvil_description != 'DELETED'
|
|
;";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
|
|
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
|
|
my $count = @{$results};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
results => $results,
|
|
count => $count,
|
|
}});
|
|
foreach my $row (@{$results})
|
|
{
|
|
# For each host_uuid, get the IPMI info.
|
|
my $anvil_name = $row->[0];
|
|
my $anvil_node1_host_uuid = $row->[1];
|
|
my $anvil_node2_host_uuid = $row->[2];
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
's1:anvil_name' => $anvil_name,
|
|
's2:anvil_node1_host_uuid' => $anvil_node1_host_uuid,
|
|
's3:anvil_node2_host_uuid' => $anvil_node2_host_uuid,
|
|
}});
|
|
|
|
my $query = "
|
|
SELECT
|
|
host_name,
|
|
host_uuid,
|
|
host_ipmi
|
|
FROM
|
|
hosts
|
|
WHERE
|
|
host_ipmi != ''
|
|
AND
|
|
(
|
|
host_uuid = ".$anvil->Database->quote($anvil_node1_host_uuid)."
|
|
OR
|
|
host_uuid = ".$anvil->Database->quote($anvil_node2_host_uuid)."
|
|
)
|
|
;";
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
|
|
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
|
|
my $count = @{$results};
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
results => $results,
|
|
count => $count,
|
|
}});
|
|
foreach my $row (@{$results})
|
|
{
|
|
# We've got an entry in the 'scan_hardware' table, so now we'll look for data in the node and
|
|
# services tables.
|
|
my $host_name = $row->[0];
|
|
my $host_uuid = $row->[1];
|
|
my $host_ipmi = $row->[2];
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
host_name => $host_name,
|
|
host_uuid => $host_uuid,
|
|
host_ipmi => $anvil->Log->is_secure($host_ipmi),
|
|
}});
|
|
|
|
# Get the ipaddress and see if I can ping the target. If I can't, there is no sense in
|
|
# recording this entry.
|
|
my $access = 0;
|
|
my $target = "";
|
|
if (($host_ipmi =~ /-a (.*?) /) or ($host_ipmi =~ /-ip (.*?) /))
|
|
{
|
|
$target = $1;
|
|
}
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }});
|
|
if ($target)
|
|
{
|
|
($access, my $average_time) = $anvil->Network->ping({ping => $target});
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
access => $access,
|
|
average_time => $average_time,
|
|
}});
|
|
}
|
|
next if not $access;
|
|
$ipmi_targets++;
|
|
|
|
# Convert to an 'ipmitool' call.
|
|
my ($ipmitool_command, $ipmi_password) = $anvil->Convert->fence_ipmilan_to_ipmitool({fence_ipmilan_command => $host_ipmi});
|
|
|
|
$anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{host_ipmi} = $host_ipmi;
|
|
$anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmitool_command} = $ipmitool_command;
|
|
$anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmi_password} = $ipmi_password;
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
"scan-ipmitool::host_name::${host_name}::host_ipmi" => $anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{host_ipmi},
|
|
"scan-ipmitool::host_name::${host_name}::ipmitool_command" => $anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmitool_command},
|
|
"scan-ipmitool::host_name::${host_name}::ipmi_password" => $anvil->Log->is_secure($anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmi_password}),
|
|
}});
|
|
}
|
|
}
|
|
=cut
|
|
return($ipmi_targets);
|
|
}
|