Merge pull request #31 from ClusterLabs/scancore-debugging

Scancore debugging

Anvil/Tools.pm

@@ -1145,6 +1145,7 @@ sub _set_paths
 				journalctl			=>	"/usr/bin/journalctl",
 				logger				=>	"/usr/bin/logger",
 				ls				=>	"/usr/bin/ls",
+				lspci				=>	"/usr/sbin/lspci",
 				lsblk				=>	"/usr/bin/lsblk",
 				lvchange			=>	"/usr/sbin/lvchange",
 				lvcreate			=>	"/usr/sbin/lvcreate",

Anvil/Tools/Database.pm

@@ -1252,7 +1252,7 @@ sub connect
 				average_time => $average_time, 
 			}});
 			
-			my $ping_time = tv_interval ($start_time, [gettimeofday]);
+			#my $ping_time = tv_interval ($start_time, [gettimeofday]);
 			#print "[".$ping_time."] - Pinged: [$host:$port:$name:$user]\n";
 			
 			if (not $pinged)

Anvil/Tools/Email.pm

@@ -623,10 +623,9 @@ Reply-To: ".$reply_to."
 		my $file_time  = $anvil->Get->date_and_time({file_name => 1});
 		my $short_uuid = $anvil->Get->uuid({short => 1});
 		my $file_name  = $anvil->data->{path}{directories}{alert_emails}."/alert_email.".$file_time.".".$short_uuid;
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0541", variables => { file => $file_name }});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0541", variables => { file => $file_name }});
 		
 		my $problem = $anvil->Storage->write_file({
-			debug => 3, 
 			file  => $file_name, 
 			body  => $email_body, 
 		});
@@ -640,7 +639,7 @@ Reply-To: ".$reply_to."
 		}
 		else
 		{
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0542", variables => { to => $to }});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0542", variables => { to => $to }});
 			my $shell_call = $anvil->data->{path}{exe}{mailx}." -t < ".$file_name;
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
 			

Anvil/Tools/Get.pm

@@ -1479,7 +1479,11 @@ sub free_memory
 
 =head2 host_type
 
-This method tries to determine the host type and returns a value suitable for use is the C<< hosts >> table.
+This method tries to determine the host type and returns a value suitable for use is the C<< hosts >> table. Returned values are;
+
+ striker - Striker dashboards
+ node    - Anvil! nodes (active protection of VMs)
+ dr      - DR Hosts (passive DR host targets)
 
  my $type = $anvil->Get->host_type();
 

Anvil/Tools/ScanCore.pm

@@ -259,7 +259,7 @@ sub call_scan_agents
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
 		
 		# Tell the user this agent is about to run...
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0252", variables => {
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0252", variables => {
 			agent_name => $agent_name,
 			timeout    => $timeout,
 		}});
@@ -269,9 +269,14 @@ sub call_scan_agents
 		{
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
 		}
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0557", variables => {
+		
+		# If an agent takes a while to run, log it with higher verbosity
+		my $runtime   = (time - $start_time);
+		my $log_level = $runtime > 10 ? 1 : $debug;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output, runtime => $runtime }});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => $log_level, key => "log_0557", variables => {
 			agent_name  => $agent_name,
-			runtime     => (time - $start_time),
+			runtime     => $runtime,
 			return_code => $return_code,
 		}});
 		

scancore-agents/scan-apc-pdu/scan-apc-pdu

@@ -75,9 +75,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 	$running_directory =~ s/^\./$ENV{PWD}/;
 }
 
-my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
+my $anvil = Anvil::Tools->new();
 
 # Make sure we're running as 'root'
 # $< == real UID, $> == effective UID
@@ -159,7 +157,7 @@ $anvil->data->{snmp} = {
 };
 
 $anvil->Storage->read_config();
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0115", variables => { program => $THIS_FILE }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 
 # Read switches
 $anvil->Get->switches;
@@ -883,7 +881,7 @@ WHERE
 					if ($new_uptime > $old_uptime)
 					{
 						# Normal, info-level
-						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0018", variables => $variables});
+						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_apc_pdu_message_0018", variables => $variables});
 						$anvil->Alert->register({
 							alert_level   => "info", 
 							message       => "scan_apc_pdu_message_0018", 
@@ -933,9 +931,9 @@ WHERE
 						old_total_wattage_draw => $old_total_wattage_draw,
 						new_total_wattage_draw => $new_total_wattage_draw, 
 					};
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0020", variables => $variables});
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_apc_pdu_message_0020", variables => $variables});
 					$anvil->Alert->register({
-						alert_level   => "notice", 
+						alert_level   => "info", 
 						message       => "scan_apc_pdu_message_0020", 
 						variables     => $variables, 
 						set_by        => $THIS_FILE, 
@@ -1035,7 +1033,7 @@ WHERE
 							old_phase_current_amperage => $old_scan_apc_pdu_phase_current_amperage,
 							new_phase_current_amperage => $new_scan_apc_pdu_phase_current_amperage, 
 						};
-						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0023", variables => $variables});
+						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_apc_pdu_message_0023", variables => $variables});
 						$anvil->Alert->register({
 							alert_level   => "info", 
 							message       => "scan_apc_pdu_message_0023", 

scancore-agents/scan-apc-ups/scan-apc-ups

@@ -40,9 +40,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 	$running_directory =~ s/^\./$ENV{PWD}/;
 }
 
-my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
+my $anvil = Anvil::Tools->new();
 
 # Make sure we're running as 'root'
 # $< == real UID, $> == effective UID
@@ -185,7 +183,7 @@ $anvil->data->{snmp} = {
 };
 
 $anvil->Storage->read_config();
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0115", variables => { program => $THIS_FILE }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 
 # Read switches
 $anvil->Get->switches;
@@ -700,6 +698,8 @@ INSERT INTO
 			# some being more critical than others.
 			if ($scan_apc_ups_last_transfer_reason ne $old_scan_apc_ups_last_transfer_reason)
 			{
+				$ups_changed = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ups_changed => $ups_changed }});
 				
 				### NOTE: This used to be 'warning' level, but it caused way too many
 				###       false alarms. If a serious issue arises, the input voltage
@@ -731,13 +731,27 @@ INSERT INTO
 					$say_old_scan_apc_ups_last_transfer_reason = 99;
 				}
 				
+				# The level of the alert will depends on the now/old state
+				my $log_level   = 3;
+				my $alert_level = "info";
+				if (($scan_apc_ups_last_transfer_reason eq "2")  or ($old_scan_apc_ups_last_transfer_reason eq "2") or 
+				    ($scan_apc_ups_last_transfer_reason eq "3")  or ($old_scan_apc_ups_last_transfer_reason eq "3") or 
+				    ($scan_apc_ups_last_transfer_reason eq "4")  or ($old_scan_apc_ups_last_transfer_reason eq "3") or 
+				    ($scan_apc_ups_last_transfer_reason eq "6")  or ($old_scan_apc_ups_last_transfer_reason eq "6") or 
+				    ($scan_apc_ups_last_transfer_reason eq "8")  or ($old_scan_apc_ups_last_transfer_reason eq "8") or 
+				    ($scan_apc_ups_last_transfer_reason eq "10") or ($old_scan_apc_ups_last_transfer_reason eq "10"))
+				{
+					$log_level   = 2;
+					$alert_level = "notice";
+				}
+				
 				my $variables = {
 					ups_name  => $scan_apc_ups_name,
 					new_value => "#!string!scan_apc_ups_last_transfer_".sprintf("%04d", $say_scan_apc_ups_last_transfer_reason)."!#",
 					old_value => "#!string!scan_apc_ups_last_transfer_".sprintf("%04d", $say_old_scan_apc_ups_last_transfer_reason)."!#",
 				};
-				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_ups_warning_0015", variables => $variables});
-				$anvil->Alert->register({alert_level => "notice", message => "scan_apc_ups_warning_0015", variables => $variables, set_by => $THIS_FILE, sort_position => $anvil->data->{'scan-apc-pdu'}{alert_sort}++});
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $log_level, key => "scan_apc_ups_warning_0015", variables => $variables});
+				$anvil->Alert->register({alert_level => $alert_level, message => "scan_apc_ups_warning_0015", variables => $variables, set_by => $THIS_FILE, sort_position => $anvil->data->{'scan-apc-pdu'}{alert_sort}++});
 			}
 			if ($scan_apc_ups_manufactured_date ne $old_scan_apc_ups_manufactured_date)
 			{

scancore-agents/scan-cluster/scan-cluster

@@ -34,9 +34,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 	$running_directory =~ s/^\./$ENV{PWD}/;
 }
 
-my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
+my $anvil = Anvil::Tools->new();
 
 # Make sure we're running as 'root'
 # $< == real UID, $> == effective UID
@@ -69,7 +67,7 @@ if ($problem)
 	$anvil->nice_exit({exit_code => 1});
 }
 
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_cluster_log_0001", variables => { program => $THIS_FILE }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 if ($anvil->data->{switches}{purge})
 {
 	# This can be called when doing bulk-database purges.
@@ -85,7 +83,7 @@ my $host_type = $anvil->Get->host_type;
 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }});
 if ($host_type ne "node")
 {
-	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_cluster_log_0002", variables => { host_type => $host_type }});
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_cluster_log_0002", variables => { host_type => $host_type }});
 	$anvil->nice_exit({exit_code => 0});
 }
 

scancore-agents/scan-drbd/scan-drbd

@@ -36,9 +36,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 	$running_directory =~ s/^\./$ENV{PWD}/;
 }
 
-my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
+my $anvil = Anvil::Tools->new();
 
 $anvil->data->{'scan-drbd'}{alert_sort} = 2;
 $anvil->data->{'scan-drbd'}{queries}    = [];
@@ -75,7 +73,7 @@ if ($problem)
 {
 	$anvil->nice_exit({exit_code => 1});
 }
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_drbd_log_0001", variables => { program => $THIS_FILE }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 
 if ($anvil->data->{switches}{purge})
 {

scancore-agents/scan-hardware/scan-hardware

@@ -31,9 +31,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 	$running_directory =~ s/^\./$ENV{PWD}/;
 }
 
-my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
+my $anvil = Anvil::Tools->new();
 
 # Make sure we're running as 'root'
 # $< == real UID, $> == effective UID
@@ -54,7 +52,7 @@ $anvil->data->{scancore}{'scan-hardware'}{swap}{high_threshold}  = 75;
 $anvil->data->{switches}{force}                                  = 0;
 
 $anvil->Storage->read_config();
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0115", variables => { program => $THIS_FILE }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "log_0115", variables => { program => $THIS_FILE }});
 
 # Read switches
 $anvil->Get->switches;
@@ -913,15 +911,15 @@ sub find_changes
 			my $say_new_scan_hardware_memory_free = $anvil->Convert->bytes_to_human_readable({'bytes' => $new_scan_hardware_memory_free})." (".$anvil->Convert->add_commas({number => $new_scan_hardware_memory_free})." #!string!scan_hardware_unit_0001!#)";
 			my $say_old_scan_hardware_memory_free = $anvil->Convert->bytes_to_human_readable({'bytes' => $old_scan_hardware_memory_free})." (".$anvil->Convert->add_commas({number => $old_scan_hardware_memory_free})." #!string!scan_hardware_unit_0001!#)";
 			$anvil->Alert->register({set_by => $THIS_FILE, alert_level => "info", message => "scan_hardware_alert_0018,!!new!".$say_new_scan_hardware_memory_free."!!,!!old!".$say_old_scan_hardware_memory_free."!!"});
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_hardware_alert_0018", variables => { new => $say_new_scan_hardware_memory_free, old => $say_old_scan_hardware_memory_free}});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_hardware_alert_0018", variables => { new => $say_new_scan_hardware_memory_free, old => $say_old_scan_hardware_memory_free}});
 		}
 		if ($new_scan_hardware_swap_free ne $old_scan_hardware_swap_free)
 		{
-			   $update                       = 1;
+			   $update                          = 1;
 			my $say_new_scan_hardware_swap_free = $anvil->Convert->bytes_to_human_readable({'bytes' => $new_scan_hardware_swap_free})." (".$anvil->Convert->add_commas({number => $new_scan_hardware_swap_free})." #!string!scan_hardware_unit_0001!#)";
 			my $say_old_scan_hardware_swap_free = $anvil->Convert->bytes_to_human_readable({'bytes' => $old_scan_hardware_swap_free})." (".$anvil->Convert->add_commas({number => $old_scan_hardware_swap_free})." #!string!scan_hardware_unit_0001!#)";
 			$anvil->Alert->register({set_by => $THIS_FILE, alert_level => "info", message => "scan_hardware_alert_0019,!!new!".$say_new_scan_hardware_swap_free."!!,!!old!".$say_old_scan_hardware_swap_free."!!"});
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_hardware_alert_0019", variables => { new => $say_new_scan_hardware_swap_free, old => $say_old_scan_hardware_swap_free}});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_hardware_alert_0019", variables => { new => $say_new_scan_hardware_swap_free, old => $say_old_scan_hardware_swap_free}});
 
 			my $new_swap_bytes_used   = $new_scan_hardware_swap_total - $new_scan_hardware_swap_free;
 			my $old_swap_bytes_used   = $old_scan_hardware_swap_total - $old_scan_hardware_swap_free;

scancore-agents/scan-ipmitool/scan-ipmitool

@@ -124,7 +124,7 @@ $anvil->data->{'scan-ipmitool'} = {
 			high_critical			=>	55,
 			low_warning			=>	5,
 			low_critical			=>	0,
-			jump				=>	5,
+			jump				=>	10,
 			### TODO: Some sensors define their hysteresis which we can read using: 
 			###       ipmitool ... sensor get "Ambient"
 			buffer				=>	2,
@@ -192,7 +192,7 @@ $anvil->data->{'scan-ipmitool'} = {
 };
 
 $anvil->Storage->read_config();
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1 , key => "log_0115", variables => { program => $THIS_FILE }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1 , key => "log_0115", variables => { program => $THIS_FILE }});
 
 # Read switches
 $anvil->data->{switches}{force} = 0;
@@ -2014,10 +2014,10 @@ sub find_ipmi_targets
 
 	# If I am a node, I will only scan myself.
 	my $host_type = $anvil->Get->host_type();
-	my $hostname  = $anvil->Get->host_name();
+	my $host_name = $anvil->Get->host_name();
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 		host_type => $host_type, 
-		hostname  => $hostname, 
+		host_name => $host_name, 
 	}});
 	
 	# Do I have local IPMI access?
@@ -2044,71 +2044,124 @@ sub find_ipmi_targets
 		}
 	}
 	
-	# Find all known hosts (except ourself) with a host_ipmi value set.
+	# Which hosts we scan depends on if we're a Striker dashboard or not. If we are, we'll try to scan 
+	# all machines in all Anvil! systems. Otherwise, we only scan ourselves.
+	if ($host_type ne "striker")
+	{
+		# We're not a dashboard, so we don't scan others.
+		return($ipmi_targets);
+	}
+	
+	# Loop through Anvil! systems.
 	my $query = "
 SELECT 
-    host_name, 
-    host_uuid, 
-    host_ipmi 
+    anvil_name, 
+    anvil_node1_host_uuid, 
+    anvil_node2_host_uuid, 
+    anvil_dr1_host_uuid 
 FROM 
-    hosts 
+    anvils 
 WHERE 
-    host_ipmi != ''
-AND 
-    host_uuid != ".$anvil->Database->quote($anvil->Get->host_uuid)."
+    anvil_description != 'DELETED'
 ;";
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
 	
 	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
 	my $count   = @{$results};
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 		results => $results, 
 		count   => $count, 
 	}});
 	foreach my $row (@{$results})
 	{
-		# We've got an entry in the 'scan_hardware' table, so now we'll look for data in the node and 
-		# services tables.
-		my $host_name = $row->[0]; 
-		my $host_uuid = $row->[1]; 
-		my $host_ipmi = $row->[2];
+		# For each host_uuid, get the IPMI info.
+		my $anvil_name            =         $row->[0]; 
+		my $anvil_node1_host_uuid =         $row->[1];
+		my $anvil_node2_host_uuid =         $row->[2];
+		my $anvil_dr1_host_uuid   = defined $row->[3] ? $row->[3] : "";
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-			host_name => $host_name, 
-			host_uuid => $host_uuid, 
-			host_ipmi => $anvil->Log->is_secure($host_ipmi),
+			's1:anvil_name'            => $anvil_name, 
+			's2:anvil_node1_host_uuid' => $anvil_node1_host_uuid, 
+			's3:anvil_node2_host_uuid' => $anvil_node2_host_uuid,
+			's4:anvil_dr1_host_uuid'   => $anvil_dr1_host_uuid, 
 		}});
 		
-		# Get the ipaddress and see if I can ping the target. If I can't, there is no sense in 
-		# recording this entry.
-		my $access = 0;
-		my $target = "";
-		if (($host_ipmi =~ /-a (.*?) /) or ($host_ipmi =~ /-ip (.*?) /))
+		my $query = "
+SELECT 
+    host_name, 
+    host_uuid, 
+    host_ipmi 
+FROM 
+    hosts 
+WHERE 
+    host_ipmi != ''
+AND 
+    (
+        host_uuid = ".$anvil->Database->quote($anvil_node1_host_uuid)."
+    OR 
+        host_uuid = ".$anvil->Database->quote($anvil_node2_host_uuid);
+		if ($anvil_dr1_host_uuid)
 		{
-			$target = $1;
+			$query .= "
+    OR 
+        host_uuid = ".$anvil->Database->quote($anvil_dr1_host_uuid);
 		}
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }});
-		if ($target)
+		$query .= "
+    )
+;";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+	
+		my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
+		my $count   = @{$results};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			results => $results, 
+			count   => $count, 
+		}});
+		foreach my $row (@{$results})
 		{
-			($access, my $average_time) = $anvil->Network->ping({ping => $target});
+			# We've got an entry in the 'scan_hardware' table, so now we'll look for data in the node and 
+			# services tables.
+			my $host_name = $row->[0]; 
+			my $host_uuid = $row->[1]; 
+			my $host_ipmi = $row->[2];
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-				access       => $access,
-				average_time => $average_time, 
+				host_name => $host_name, 
+				host_uuid => $host_uuid, 
+				host_ipmi => $anvil->Log->is_secure($host_ipmi),
+			}});
+			
+			# Get the ipaddress and see if I can ping the target. If I can't, there is no sense in 
+			# recording this entry.
+			my $access = 0;
+			my $target = "";
+			if (($host_ipmi =~ /-a (.*?) /) or ($host_ipmi =~ /-ip (.*?) /))
+			{
+				$target = $1;
+			}
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }});
+			if ($target)
+			{
+				($access, my $average_time) = $anvil->Network->ping({ping => $target});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					access       => $access,
+					average_time => $average_time, 
+				}});
+			}
+			next if not $access;
+			$ipmi_targets++;
+			
+			# Convert to an 'ipmitool' call.
+			my ($ipmitool_command, $ipmi_password) = $anvil->Convert->fence_ipmilan_to_ipmitool({fence_ipmilan_command => $host_ipmi});
+			
+			$anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{host_ipmi}        = $host_ipmi;
+			$anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmitool_command} = $ipmitool_command;
+			$anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmi_password}    = $ipmi_password;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				"scan-ipmitool::host_name::${host_name}::host_ipmi"        => $anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{host_ipmi}, 
+				"scan-ipmitool::host_name::${host_name}::ipmitool_command" => $anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmitool_command}, 
+				"scan-ipmitool::host_name::${host_name}::ipmi_password"    => $anvil->Log->is_secure($anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmi_password}), 
 			}});
 		}
-		next if not $access;
-		$ipmi_targets++;
-		
-		# Convert to an 'ipmitool' call.
-		my ($ipmitool_command, $ipmi_password) = $anvil->Convert->fence_ipmilan_to_ipmitool({fence_ipmilan_command => $host_ipmi});
-		
-		$anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{host_ipmi}        = $host_ipmi;
-		$anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmitool_command} = $ipmitool_command;
-		$anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmi_password}    = $ipmi_password;
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-			"scan-ipmitool::host_name::${host_name}::host_ipmi"        => $anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{host_ipmi}, 
-			"scan-ipmitool::host_name::${host_name}::ipmitool_command" => $anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmitool_command}, 
-			"scan-ipmitool::host_name::${host_name}::ipmi_password"    => $anvil->Log->is_secure($anvil->data->{'scan-ipmitool'}{host_name}{$host_name}{ipmi_password}), 
-		}});
 	}
 	
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ipmi_targets => $ipmi_targets }});

scancore-agents/scan-lvm/scan-lvm

@@ -36,9 +36,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 	$running_directory =~ s/^\./$ENV{PWD}/;
 }
 
-my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
+my $anvil = Anvil::Tools->new();
 
 # Make sure we're running as 'root'
 # $< == real UID, $> == effective UID
@@ -54,7 +52,7 @@ $anvil->data->{scancore}{'scan-lvm'}{disable} = 0;
 $anvil->data->{switches}{force}               = 0;
 
 $anvil->Storage->read_config();
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0115", variables => { program => $THIS_FILE }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 
 # Read switches
 $anvil->Get->switches;

scancore-agents/scan-server/scan-server

@@ -35,9 +35,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 	$running_directory =~ s/^\./$ENV{PWD}/;
 }
 
-my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
+my $anvil = Anvil::Tools->new();
 
 # Make sure we're running as 'root'
 # $< == real UID, $> == effective UID
@@ -71,7 +69,7 @@ if ($problem)
 	$anvil->nice_exit({exit_code => 1});
 }
 
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_log_0001", variables => { program => $THIS_FILE }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_server_log_0001", variables => { program => $THIS_FILE }});
 
 # There are no tables for this agent, so '--purge' is useless here.
 
@@ -80,7 +78,7 @@ my $host_type = $anvil->Get->host_type;
 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }});
 if ($host_type eq "striker")
 {
-	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_log_0002", variables => { host_type => $host_type }});
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_server_log_0002", variables => { host_type => $host_type }});
 	$anvil->nice_exit({exit_code => 0});
 }
 

scancore-agents/scan-storcli/scan-storcli

@@ -75,9 +75,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 	$running_directory =~ s/^\./$ENV{PWD}/;
 }
 
-my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
+my $anvil = Anvil::Tools->new();
 
 # Make sure we're running as 'root'
 # $< == real UID, $> == effective UID
@@ -207,7 +205,7 @@ $anvil->data->{'scan-storcli'} = {
 };
 
 $anvil->Storage->read_config();
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1 , key => "log_0115", variables => { program => $THIS_FILE }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "log_0115", variables => { program => $THIS_FILE }});
 
 # Read switches
 $anvil->data->{switches}{force} = 0;
@@ -240,7 +238,7 @@ if ($anvil->data->{switches}{purge})
 }
 
 
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_storcli_message_0001"});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "scan_storcli_message_0001"});
 
 # This does two things; It checks to see if storcli64 is installed (exits '1' if not, exits '2' if not 
 # executable) and then checks to see if any controllers are found in the system (exits '3' if not). 
@@ -2724,7 +2722,7 @@ INSERT INTO
 					low_critical_temperature  => $low_critical, 
 					low_warning_temperature	  => $low_warning, 
 				};
-				my $log_level = $alert_level eq "notice" ? 2 : 1;
+				my $log_level = $alert_level eq "notice" ? 3 : 2;
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $log_level, key => $message_key, variables => $variables});
 				$anvil->Alert->register({
 					alert_level   => $alert_level, 
@@ -3284,7 +3282,7 @@ WHERE
 							low_warning_temperature	  => $low_warning, 
 							jump                      => $jump,
 						};
-						my $log_level = $alert_level eq "notice" ? 2 : 1;
+						my $log_level = $alert_level eq "notice" ? 3 : 2;
 						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $log_level, key => $message_key, variables => $variables});
 						$anvil->Alert->register({
 							alert_level   => $alert_level, 
@@ -10040,12 +10038,37 @@ sub find_lsi_controllers
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "path::exe::storcli64" => $anvil->data->{path}{exe}{storcli64} }});
 	}
 	
-	# First, do we have storcli64 installed?
+	# Do we have storcli64 installed?
 	if (not -e $anvil->data->{path}{exe}{storcli64})
 	{
-		# Nope, exit.
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "scan_storcli_error_0001", variables => { path => $anvil->data->{path}{exe}{storcli64} }});
-		$anvil->nice_exit({exit_code => 1});
+		# Nope, Call lspci to see if there's a MegaRAID controller. If there is, the user may need to
+		# install the RPM.
+		my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{lspci}});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+			output      => $output, 
+			return_code => $return_code,
+		}});
+		
+		my $megaraid_installed = 0;
+		foreach my $line (split/\n/, $output)
+		{
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
+			
+			if ($line =~ /MegaRAID/i)
+			{
+				# This host appears to have a RAID card, but it's not installed. Lets try to 
+				# install it for them.
+				$megaraid_installed = install_storcli($anvil);
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { megaraid_installed => $megaraid_installed }});
+			}
+		}
+		
+		# exit.
+		if (not $megaraid_installed)
+		{
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "scan_storcli_error_0001", variables => { path => $anvil->data->{path}{exe}{storcli64} }});
+			$anvil->nice_exit({exit_code => 1});
+		}
 	}
 	
 	# Make sure it is executable
@@ -10083,7 +10106,7 @@ sub find_lsi_controllers
 	}
 	else
 	{
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "scan_storcli_error_0003", variables => { 
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 2, level => 0, key => "scan_storcli_error_0003", variables => { 
 			path => $anvil->data->{path}{exe}{storcli64}, 
 		}});
 		$anvil->nice_exit({exit_code => 3});
@@ -10091,3 +10114,73 @@ sub find_lsi_controllers
 	
 	return(0);
 }
+
+sub install_storcli
+{
+	my ($anvil) = @_;
+	
+	# Tell the user what we're doing. 
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "scan_storcli_note_0071"});
+	
+	# Is this a Dell?
+	my $is_dell                = 0;
+	my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{dmidecode}." --string system-manufacturer"});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+		output      => $output, 
+		return_code => $return_code,
+	}});
+	
+	foreach my $line (split/\n/, $output)
+	{
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
+		if ($line =~ /Dell/i)
+		{
+			$is_dell = 1;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { is_dell => $is_dell }});
+		}
+	}
+	
+	my $rpm_name   = $is_dell ? "perccli" : "storcli";
+	my $shell_call = $anvil->data->{path}{exe}{dnf}." -y install ".$rpm_name;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+	($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+		output      => $output, 
+		return_code => $return_code,
+	}});
+	
+	# Check now to see if the program is installed. If it is, register an alert announcing we installed it.
+	my $program = $is_dell ? $anvil->data->{path}{exe}{perccli64} : $anvil->data->{path}{exe}{storcli64};
+	if (-e $program)
+	{
+		# Installed successfully! 
+		my $variables = {
+			rpm => $rpm_name,
+		};
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_storcli_note_0072", variables => $variables});
+		$anvil->Alert->register({
+			alert_level   => "notice", 
+			message       => "scan_storcli_note_0072", 
+			show_header   => 1, 
+			variables     => $variables, 
+			sort_position => 0,
+			set_by        => $THIS_FILE, 
+		});
+		
+		# Before we return, if we installed for Dell, switch out the 'storcli' program path.
+		if ($is_dell)
+		{
+			$anvil->data->{path}{exe}{storcli64} = $anvil->data->{path}{exe}{perccli64};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "path::exe::storcli64" => $anvil->data->{path}{exe}{storcli64} }});
+		}
+		
+		return(1);
+	}
+	else
+	{
+		# Didn't work.
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "scan_storcli_note_0073"});
+	}
+	
+	return(0);
+}

scancore-agents/scan-storcli/scan-storcli.xml

@@ -334,6 +334,9 @@ The temperature of the Battery Backup Unit (BBU): [#!variable!serial_number!#] h
 - Controller: [#!variable!serial_number!#]: '#!variable!name!#' has changed: [#!variable!old_value!#] -> [#!variable!new_value!#]
   NOTE: This is expected and is no reason for concern.
 		</key>
+		<key name="scan_storcli_note_0071">The host appears to have a MegaRAID controller, but the tool to communicate with it is not installed. We'll try to install it now.</key>
+		<key name="scan_storcli_note_0072">An LSI-based (MegaRAID) controller was found, but the management tool was not installed. The RPM: [#!variable!rpm!#] has now been installed to allow monitoring storage.</key>
+		<key name="scan_storcli_note_0073">The install didn't work, we'll try again in the next scan.</key>
 
 		
 		<!-- Warnings -->

share/anvil.sql

@@ -337,7 +337,7 @@ CREATE TRIGGER trigger_sessions
 CREATE TABLE anvils (
     anvil_uuid               uuid                        not null    primary key, 
     anvil_name               text                        not null,
-    anvil_description        text                        not null,                -- This is a short, one-line (usually) description of this particular Anvil!. It is displayed in the Anvil! selection list.
+    anvil_description        text                        not null,                -- This is a short, one-line (usually) description of this particular Anvil!. It is displayed in the Anvil! selection list. This is set to 'DELETED' when an Anvil! is removed.
     anvil_password           text                        not null,                -- This is the 'hacluster' user password. It is also used to access nodes that don't have a specific password set.
     anvil_node1_host_uuid    uuid,                                                -- This is the host_uuid of the machine that is used as node 1. 
     anvil_node2_host_uuid    uuid,                                                -- This is the host_uuid of the machine that is used as node 2. 

share/words.xml

@@ -949,7 +949,7 @@ The body of the file: [#!variable!file!#] does not match the new body. The file
 		<key name="log_0246">[ Warning ] - The local system is not yet configured. Scancore will check once a minute and start running once configured.</key>
 		<key name="log_0247">[ Cleared ] - The local system is now configured, proceeding.</key>
 		<key name="log_0248">ScanCore is entering the main loop now.</key>
-		<key name="log_0249">----=] ScanCore loop finished. Sleeping for: [#!variable!run_interval!#] seconds. ]=--------------------------------------</key>	<!-- This is meant to be easily seen in the logs, hence the dashes. -->
+		<key name="log_0249">----=] ScanCore loop finished after: [#!variable!runtime!#]. Sleeping for: [#!variable!run_interval!#] seconds. ]=--------------------------------------</key>	<!-- This is meant to be easily seen in the logs, hence the dashes. -->
 		<key name="log_0250">
 The md5sum of: [#!variable!file!#] has changed since the daemon started.
 * [#!variable!old_sum!#] -> [#!variable!new_sum!#]

tools/scancore

@@ -16,7 +16,9 @@
 # - Record how long a server's migration took in the past, and use that to determine which node to evacuate 
 #   during load shed. Also, track how long it takes for servers to stop to determine when to initiate a total
 #   shutdown.
-# - 
+# - Add a '--silence-alerts --anvil <name>' and '--restore-alerts --anvil <name>' to temporarily 
+#   disable/re-enable alerts. This is to allow for quiet maintenance without stopping scancore itself.
+# 
 
 use strict;
 use warnings;
@@ -95,10 +97,11 @@ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level
 while(1)
 {
 	# Do the various pre-run tasks.
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'sys::log::level' => $anvil->data->{sys}{'log'}{level} }});
+	my $start_time = time;
 	prepare_for_run($anvil);
 	
 	# Do we have at least one database?
+	my $agent_runtime = 0;
 	if ($anvil->data->{sys}{database}{connections})
 	{
 		# Run the normal tasks
@@ -145,7 +148,10 @@ while(1)
 	{
 		$run_interval = $anvil->data->{scancore}{timing}{run_interval};
 	}
-	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0249", variables => { run_interval => $run_interval }});
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0249", variables => { 
+		run_interval => $run_interval,
+		runtime      => (time - $start_time),
+	}});
 	sleep($run_interval);
 	
 	# In case something has changed, exit.
@@ -209,7 +215,7 @@ sub prepare_for_run
 	$anvil->Get->switches();
 	$anvil->Words->read();
 	$anvil->Database->connect();
-	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0132"});
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0132"});
 	
 	# See if the mail server needs to be updated.
 	$anvil->Email->check_config;
@@ -301,9 +307,8 @@ sub startup_tasks
 	my ($anvil) = @_;
 	
 	# Make sure our stop reason is cleared.
-	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0572"});
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0572"});
 	my $variable_uuid = $anvil->Database->insert_or_update_variables({
-		debug                 => 2,
 		variable_name         => 'system::stop_reason', 
 		variable_value        => '', 
 		variable_default      => '',