Merge pull request #87 from ClusterLabs/anvil-tools-dev

Anvil tools dev

Anvil/Tools/Database.pm

@@ -14716,7 +14716,7 @@ sub resync_databases
 		
 		# If the 'schema' is 'public', there is no table in the history schema. If there is a host 
 		# column, the resync will be restricted to entries from this host uuid.
-		my $schema      = $anvil->data->{sys}{database}{table}{$table}{schema};
+		my $schema      = $anvil->data->{sys}{database}{table}{$table}{schema} ? $anvil->data->{sys}{database}{table}{$table}{schema} : "public";
 		my $host_column = $anvil->data->{sys}{database}{table}{$table}{host_column};
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			table       => $table, 
@@ -14841,6 +14841,12 @@ sub resync_databases
 				results => $results, 
 				count   => $count,
 			}});
+			
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				's1:database'     => $anvil->Get->host_name_from_uuid({host_uuid => $uuid}),
+				's2:schema.table' => $schema.".".$table,
+				's3:count'        => $count,
+			}});
 			next if not $count;
 			
 			my $row_number = 0;
@@ -15024,15 +15030,14 @@ sub resync_databases
 								# Add the host column.
 								$query = "INSERT INTO public.$table ($host_column, $uuid_column, ".$columns."modified_date) VALUES (".$anvil->Database->quote($anvil->data->{sys}{host_uuid}).", ".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');";
 							}
-							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0460", variables => { uuid => $anvil->data->{database}{$uuid}{host}, query => $query }});
+							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0460", variables => { uuid => $anvil->data->{database}{$uuid}{host}, query => $query }});
-							
+							
-							### NOTE: On some occasions, for as-yet unknown 
+							### NOTE: After an archive operationg, a record can 
-							###       reasons, a record can end up in the public 
+							###       end up in the public schema while nothing 
-							###       schema while nothing exists in the history 
+							###       exists in the history schema (which is what
-							###       schema (which is what we read during a 
+							###       we read during a resync). To deal with 
-							###       resync). To deal with this, we'll do an
+							###       this, we'll do an explicit check before 
-							###       explicit check before confirming the 
+							###       confirming the INSERT)
-							###       INSERT)
 							my $count_query = "SELECT COUNT(*) FROM public.".$table." WHERE ".$uuid_column." = ".$anvil->Database->quote($row_uuid).";";
 							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count_query => $count_query }});
 							my $count = $anvil->Database->query({uuid => $uuid, query => $count_query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
@@ -15049,12 +15054,14 @@ sub resync_databases
 									query     => $query, 
 								}});
 								$query =~ s/INSERT INTO public./INSERT INTO history./;
+								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
 								
 								push @{$anvil->data->{db_resync}{$uuid}{history}{sql}}, $query;
 							}
 							else
 							{
 								# No problem, record the query in the array
+								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
 								push @{$anvil->data->{db_resync}{$uuid}{public}{sql}}, $query;
 							}
 						} # if not exists
@@ -15102,7 +15109,7 @@ sub resync_databases
 								# Add the host column.
 								$query = "INSERT INTO history.$table ($host_column, $uuid_column, ".$columns."modified_date) VALUES (".$anvil->Database->quote($anvil->data->{sys}{host_uuid}).", ".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');";
 							}
-							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0460", variables => { uuid => $anvil->data->{database}{$uuid}{host}, query => $query }});
+							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0460", variables => { uuid => $anvil->data->{database}{$uuid}{host}, query => $query }});
 							
 							# Now record the query in the array
 							push @{$anvil->data->{db_resync}{$uuid}{history}{sql}}, $query;
@@ -15128,6 +15135,7 @@ sub resync_databases
 			
 			# If the merged array has any entries, push them in.
 			my $to_write_count = @{$merged};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { to_write_count => $to_write_count }});
 			if ($to_write_count > 0)
 			{
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0221", variables => { 
@@ -16073,9 +16081,9 @@ ORDER BY
 			if ($anvil->data->{sys}{database}{table}{$table}{row_count} > $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count})
 			{
 				# Resync needed.
-				my $difference = $anvil->Convert->add_commas({number => ($anvil->data->{sys}{database}{table}{$table}{row_count} - $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count}) });
+				my $difference = ($anvil->data->{sys}{database}{table}{$table}{row_count} - $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count});
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-					"s1:difference"                                               => $difference, 
+					"s1:difference"                                               => $anvil->Convert->add_commas({number => $difference }), 
 					"s2:sys::database::table::${table}::row_count"                => $anvil->data->{sys}{database}{table}{$table}{row_count}, 
 					"s3:sys::database::table::${table}::uuid::${uuid}::row_count" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count}, 
 				}});

Anvil/Tools/Network.pm

@@ -185,6 +185,7 @@ sub bridge_info
 	my $bridge_data = $json->decode($output);
 	foreach my $hash_ref (@{$bridge_data})
 	{
+		next if not defined $hash_ref->{master};
 		my $bridge    = $hash_ref->{master};
 		my $interface = $hash_ref->{ifname};
 		my $host      = $target ? $target : $anvil->Get->short_host_name();

Anvil/Tools/ScanCore.pm

@@ -223,7 +223,7 @@ sub call_scan_agents
 	$anvil->ScanCore->_scan_directory({directory => $anvil->data->{path}{directories}{scan_agents}});
 	
 	# Now loop through the agents I found and call them.
-	my $timeout = 30;
+	my $timeout = 60;
 	if ((exists $anvil->data->{scancore}{timing}{agent_runtime}) && ($anvil->data->{scancore}{timing}{agent_runtime} =~ /^\d+$/))
 	{
 		$timeout = $anvil->data->{scancore}{timing}{agent_runtime};
@@ -298,11 +298,18 @@ sub call_scan_agents
 			$log_level  = 1;
 			$string_key = "log_0621";
 		}
+		if ($return_code eq "124")
+		{
+			# Timed out
+			$log_level  = 1;
+			$string_key = "message_0180";
+		}
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output, runtime => $runtime }});
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => $log_level, key => $string_key, variables => {
 			agent_name  => $agent_name,
 			runtime     => $runtime,
 			return_code => $return_code,
+			timeout     => $timeout,
 		}});
 		
 		# If the return code is '124', timeout popped.
@@ -2121,7 +2128,7 @@ LIMIT 1;";
 			if ($last_update_age < 120)
 			{
 				# It was alive less than two minutes ago, we don't need to check anything.
-				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0596", variables => { 
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0596", variables => { 
 					host_name  => $host_name,
 					difference => $last_update_age, 
 				}});

Anvil/Tools/System.pm

@@ -1294,13 +1294,11 @@ sub collect_ipmi_data
 		return_code => $return_code,
 	}});
 	
-	# Delete the temp file.
+	my $delete_entries   = [];
-	unlink $temp_file;
+	my $duplicate_exists = 0;
-	
-	my $temp_count = 1;
 	foreach my $line (split/\n/, $output)
 	{
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ">> line" => $line }});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { ">> line" => $line }});
 		
 		# Clean up the output
 		$line =~ s/^\s+//;
@@ -1390,11 +1388,23 @@ sub collect_ipmi_data
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sensor_name => $sensor_name }});
 		}
 		
-		# Dells have two sensors called simply "Temp". 
+		if (exists $anvil->data->{seen_sensors}{$sensor_name})
-		if ($sensor_name eq "Temp")
 		{
-			$sensor_name = "Temp".$temp_count++;
+			$duplicate_exists                                     = 1;
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sensor_name => $sensor_name }});
+			$anvil->data->{seen_sensors}{$sensor_name}{duplicate} = 1;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				duplicate_exists                          => $duplicate_exists,
+				"seen_sensors::${sensor_name}::duplicate" => $anvil->data->{seen_sensors}{$sensor_name}{duplicate}, 
+			}});
+			
+			push @{$delete_entries}, $sensor_name;
+		}
+		else
+		{
+			$anvil->data->{seen_sensors}{$sensor_name}{duplicate} = 0;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				"seen_sensors::${sensor_name}::duplicate" => $anvil->data->{seen_sensors}{$sensor_name}{duplicate}, 
+			}});
 		}
 		
 		# Thresholds that are 'na' need to be converted to numeric
@@ -1494,6 +1504,104 @@ sub collect_ipmi_data
 		}});
 	}
 	
+	### NOTE: This is a dirty hack... It assumes the duplicates share the same thresholds, which works 
+	###       for the 'Temp' duplicates, but could well not apply to future duplicates. The real fix is
+	###       for hardware vendors to not duplicate sensor names.
+	# If there were two or more sensors with the same name, call 'ipmitool sdr elist' and change their 
+	# names to include the address and pull the values from here.
+	if ($duplicate_exists)
+	{
+		my $shell_call = $ipmitool_command." sdr elist full";
+		if ($ipmi_password)
+		{
+			$shell_call = $ipmitool_command." -f ".$temp_file." sdr elist full";
+		}
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+		
+		my ($output, $return_code) = $anvil->System->call({timeout => 30, shell_call => $shell_call});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+			output      => $output, 
+			return_code => $return_code,
+		}});
+		
+		my $duplicate_exists = 0;
+		foreach my $line (split/\n/, $output)
+		{
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ">> line" => $line }});
+			
+			# Clean up the output
+			$line =~ s/^\s+//;
+			$line =~ s/\s+$//;
+			$line =~ s/\s+\|/|/g;
+			$line =~ s/\|\s+/|/g;
+			
+			# current value -----------------.
+			#     entity ID -------------.   |
+			#        status ---------.   |   |
+			#   Hex address -----.   |   |   |
+			#   sensor name -.   |   |   |   |
+			# Columns:       |   |   |   |   |
+			#                x | x | x | x | x 
+			my ($sensor_name, 
+			    $hex_address,
+			    $status, 
+			    $entity_id, 
+			    $current_value) = split /\|/, $line;
+			next if not $sensor_name;
+			
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				sensor_name   => $sensor_name, 
+				hex_address   => $hex_address, 
+				status        => $status, 
+				entity_id     => $entity_id, 
+				current_value => $current_value, 
+			}});
+			
+			# This is a duplicate, over write the name 
+			if ((exists $anvil->data->{seen_sensors}{$sensor_name}) && ($anvil->data->{seen_sensors}{$sensor_name}{duplicate}))
+			{
+				my $units = "";
+				if ($current_value =~ /^(.*?)\s+degrees C/)
+				{
+					$current_value = $1;
+					$units         = "degrees C";
+				}
+				my $new_sensor_name = $sensor_name." (".$hex_address.")";
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					new_sensor_name => $new_sensor_name, 
+					current_value   => $current_value, 
+				}});
+				
+				$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_value_sensor_value}   = $current_value;
+				$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_sensor_units}         = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$sensor_name}{scan_ipmitool_sensor_units};
+				$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_sensor_status}        = $status;
+				$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_sensor_high_critical} = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$sensor_name}{scan_ipmitool_sensor_high_critical};
+				$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_sensor_high_warning}  = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$sensor_name}{scan_ipmitool_sensor_high_warning};
+				$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_sensor_low_critical}  = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$sensor_name}{scan_ipmitool_sensor_low_critical};
+				$anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_sensor_low_warning}   = $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$sensor_name}{scan_ipmitool_sensor_low_warning};
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					"ipmi::${host_name}::scan_ipmitool_sensor_name::${new_sensor_name}::scan_ipmitool_value_sensor_value"   => $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_value_sensor_value}, 
+					"ipmi::${host_name}::scan_ipmitool_sensor_name::${new_sensor_name}::scan_ipmitool_sensor_units"         => $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_sensor_units}, 
+					"ipmi::${host_name}::scan_ipmitool_sensor_name::${new_sensor_name}::scan_ipmitool_sensor_status"        => $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_sensor_status}, 
+					"ipmi::${host_name}::scan_ipmitool_sensor_name::${new_sensor_name}::scan_ipmitool_sensor_high_critical" => $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_sensor_high_critical}, 
+					"ipmi::${host_name}::scan_ipmitool_sensor_name::${new_sensor_name}::scan_ipmitool_sensor_high_warning"  => $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_sensor_high_warning}, 
+					"ipmi::${host_name}::scan_ipmitool_sensor_name::${new_sensor_name}::scan_ipmitool_sensor_low_critical"  => $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_sensor_low_critical}, 
+					"ipmi::${host_name}::scan_ipmitool_sensor_name::${new_sensor_name}::scan_ipmitool_sensor_low_warning"   => $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$new_sensor_name}{scan_ipmitool_sensor_low_warning}, 
+				}});
+			}
+		}
+		
+		# Delete duplicate sensor names 
+		foreach my $sensor_name (@{$delete_entries})
+		{
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sensor_name => $sensor_name }});
+			delete $anvil->data->{ipmi}{$host_name}{scan_ipmitool_sensor_name}{$sensor_name};
+		}
+	}
+	
+	# Delete the temp file.
+	unlink $temp_file;
+	
 	# Record how long it took.
 	my $sensor_read_time = (time - $read_start_time);
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sensor_read_time => $sensor_read_time }});
@@ -1502,7 +1610,6 @@ sub collect_ipmi_data
 		'time'    => $anvil->Convert->time({'time' => $sensor_read_time}),
 	}});
 	
-	
 	return($problem);
 }
 

scancore-agents/scan-apc-pdu/scan-apc-pdu

@@ -769,10 +769,11 @@ sub find_changes
 				}});
 				
 				my $variables = {
-					say_new_link_speed_mbps => $say_new_link_speed_mbps, 
+					name                => $pdu_host_name,
-					say_new_link_speed_hr   => $say_new_link_speed_hr, 
+					new_link_speed_mbps => $say_new_link_speed_mbps, 
-					say_old_link_speed_mbps => $say_old_link_speed_mbps, 
+					new_link_speed_hr   => $say_new_link_speed_hr, 
-					say_old_link_speed_hr   => $say_old_link_speed_hr, 
+					old_link_speed_mbps => $say_old_link_speed_mbps, 
+					old_link_speed_hr   => $say_old_link_speed_hr, 
 				};
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0015", variables => $variables});
 				$anvil->Alert->register({

scancore-agents/scan-apc-pdu/scan-apc-pdu.xml

@@ -44,7 +44,7 @@ NOTE: All string keys MUST be prefixed with the agent name! ie: 'scan_apc_pdu_lo
 		<key name="scan_apc_pdu_message_0019">The PDU: [#!variable!name!#] appears to have rebooted. The uptime changed from; [#!variable!old_uptime!#] to: [#!variable!new_uptime!#]. Was the power restored after a power outage?</key>
 		<key name="scan_apc_pdu_message_0020">The wattage draw through the PDU: [#!variable!name!#] has changed from: [#!variable!old_total_wattage_draw!#] to: [#!variable!new_total_wattage_draw!#].</key>
 		<key name="scan_apc_pdu_message_0021">The phase number: [#!variable!phase!#] on the PDU: [#!variable!name!#] has returned.</key>
-		<key name="scan_apc_pdu_message_0022">The maximum amperage threshold on the phase: [#!variable!phase!#] of the PDU: [#!variable!name!#] has changed from: [#!variable!old_outlet_count!#] to: [#!variable!new_outlet_count!#].</key>
+		<key name="scan_apc_pdu_message_0022">The maximum amperage threshold on the phase: [#!variable!phase!#] of the PDU: [#!variable!name!#] has changed from: [#!variable!old_phase_max_amperage#] to: [#!variable!new_phase_max_amperage!#].</key>
 		<key name="scan_apc_pdu_message_0023">The amperage draw on phase: [#!variable!phase!#] of the PDU: [#!variable!name!#] has changed from: [#!variable!old_phase_current_amperage!#] to: [#!variable!new_phase_current_amperage!#].</key>
 		<key name="scan_apc_pdu_message_0024">The amperage drawing through the phase: [#!variable!phase!#] of the PDU: [#!variable!name!#] is below the low-warning threshold. This is not a concern for the Anvil!, but would only happen if a user configured a low draw alert. So this could be a concern to a user of this Anvil! system.</key>
 		<key name="scan_apc_pdu_message_0025">The amperage drawing through the phase: [#!variable!phase!#] of the PDU: [#!variable!name!#] has rised over the high critical alert threshold!</key>

scancore-agents/scan-apc-ups/scan-apc-ups

@@ -1459,9 +1459,10 @@ WHERE
 						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { battery_changed => $battery_changed }});
 						
 						my $variables = {
-							ups_name  => $scan_apc_ups_name,
+							battery_number => $battery_number, 
-							new_value => $scan_apc_ups_battery_model,
+							ups_name       => $scan_apc_ups_name,
-							old_value => $old_scan_apc_ups_battery_model,
+							new_value      => $scan_apc_ups_battery_model,
+							old_value      => $old_scan_apc_ups_battery_model,
 						};
 						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_apc_ups_message_0012", variables => $variables});
 						$anvil->Alert->register({alert_level => "notice", message => "scan_apc_ups_message_0012", variables => $variables, sort_position => $anvil->data->{'scan-apc-ups'}{alert_sort}++, set_by => $THIS_FILE});

scancore-agents/scan-ipmitool/scan-ipmitool

@@ -180,17 +180,8 @@ $anvil->data->{'scan-ipmitool'} = {
 		'RAID Controller'		=>	{
 			jump				=>	15,
 		},
-		# On Dells, 'Temp{1..4}' change a lot, so we bump the jump.
+		# On Dells, 'Temp (xxh)' change a lot, so we bump the jump.
-		'Temp1'				=>	{
+		'Temp'				=>	{
-			jump				=>	20,
-		},
-		'Temp2'				=>	{
-			jump				=>	20,
-		},
-		'Temp3'				=>	{
-			jump				=>	20,
-		},
-		'Temp4'				=>	{
 			jump				=>	20,
 		},
 	},
@@ -735,11 +726,13 @@ INSERT INTO
     scan_ipmitool_values 
 (
     scan_ipmitool_value_uuid, 
+    scan_ipmitool_value_host_uuid, 
     scan_ipmitool_value_scan_ipmitool_uuid, 
     scan_ipmitool_value_sensor_value, 
     modified_date
 ) VALUES (
     ".$anvil->Database->quote($scan_ipmitool_value_uuid).", 
+    ".$anvil->Database->quote($anvil->Get->host_uuid).", 
     ".$anvil->Database->quote($scan_ipmitool_uuid).", 
     ".$anvil->Database->quote($new_scan_ipmitool_value_sensor_value).",  
     ".$anvil->Database->quote($anvil->data->{sys}{database}{timestamp})."
@@ -1577,6 +1570,18 @@ sub process_temperature_change
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { jump => $jump }});
 	}
 	
+	# We append the hex address some sensor names when there are duplicates. This deals with those.
+	if ($scan_ipmitool_sensor_name =~ /^(.*?) \(.*?h\)$/)
+	{
+		my $jump_name = $1;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { jump_name => $jump_name }});
+		if ((exists $anvil->data->{'scan-ipmitool'}{thresholds}{$jump_name}) && ($anvil->data->{'scan-ipmitool'}{thresholds}{$jump_name}{jump}))
+		{
+			$jump = $anvil->data->{'scan-ipmitool'}{thresholds}{$jump_name}{jump};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { jump => $jump }});
+		}
+	}
+	
 	# Final levels
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 		host_name                 => $host_name, 
@@ -1935,6 +1940,7 @@ sub query_ipmi_targets
 		# ipmi::<host_name>::scan_ipmitool_sensor_name::$sensor_name::scan_ipmitool_sensor_low_critical
 		# ipmi::<host_name>::scan_ipmitool_sensor_name::$sensor_name::scan_ipmitool_sensor_low_warning
 		$anvil->System->collect_ipmi_data({
+			debug            => 3,
 			host_name        => $host_name, 
 			ipmitool_command => $ipmitool_command, 
 			ipmi_password    => $ipmi_password, 

share/anvil.sql

@@ -1088,8 +1088,8 @@ BEGIN
          ip_address_subnet_mask, 
          ip_address_gateway, 
          ip_address_default_gateway, 
-         ip_address_dns, 
          ip_address_note, 
+         ip_address_dns, 
          modified_date)
     VALUES
         (history_ip_addresses.ip_address_uuid, 
@@ -1907,8 +1907,8 @@ BEGIN
          history_temperature.temperature_agent_name,
          history_temperature.temperature_sensor_host, 
          history_temperature.temperature_sensor_name,
-         history_temperature.temperature_weight, 
          history_temperature.temperature_value_c,
+         history_temperature.temperature_weight, 
          history_temperature.temperature_state,
          history_temperature.temperature_is,
          history_temperature.modified_date);

tools/Makefile.am

@@ -20,6 +20,7 @@ dist_sbin_SCRIPTS	= \
 			  anvil-manage-firewall \
 			  anvil-manage-keys \
 			  anvil-manage-power \
+			  anvil-manage-storage \
 			  anvil-migrate-server \
 			  anvil-parse-fence-agents \
 			  anvil-provision-server \

tools/anvil-manage-storage

@@ -0,0 +1,42 @@
+#!/usr/bin/perl
+# 
+# This program will manage storage; Growing virtual disks, adding virtual disks, inserting and ejecting ISO 
+# images into virtual optical media.
+# 
+# Exit codes;
+# 0 = Normal exit.
+# 1 = No database connection.
+# 
+# TODO: 
+# 
+
+use strict;
+use warnings;
+use Anvil::Tools;
+
+my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
+my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
+if (($running_directory =~ /^\./) && ($ENV{PWD}))
+{
+	$running_directory =~ s/^\./$ENV{PWD}/;
+}
+
+# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
+$| = 1;
+
+my $anvil = Anvil::Tools->new();
+
+$anvil->data->{switches}{'server'}     = "";	# server name or uuid
+$anvil->data->{switches}{'anvil'}      = "";	# Only required for server name collisions
+$anvil->data->{switches}{'drive'}      = "";	# drive
+$anvil->data->{switches}{'expand-to'}  = "";
+$anvil->data->{switches}{'insert-iso'} = "";
+$anvil->data->{switches}{'eject-iso'}  = "";
+$anvil->data->{switches}{'show'}       = "";
+$anvil->Get->switches;
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
+$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+	'switches::job-uuid'  => $anvil->data->{switches}{'job-uuid'},
+	'switches::host-uuid' => $anvil->data->{switches}{'host-uuid'},
+	'switches::host-name' => $anvil->data->{switches}{'host-name'}, 
+}});

tools/anvil-safe-start

@@ -15,6 +15,7 @@
 # - Add job support
 # - Make this work on DR hosts.
 # - 'pcs quorum unblock' could be useful in sole-survivor cold starts.
+# - Start DRBD resources if the VMs are running already on the peer.
 # 
 
 use strict;
@@ -110,6 +111,9 @@ start_pacemaker($anvil);
 # Boot servers.
 boot_servers($anvil);
 
+# Start DRBD resources locally for VMs running on the peer already
+check_drbd($anvil);
+
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0281"});
 
 $anvil->nice_exit({exit_code => 0});
@@ -118,6 +122,15 @@ $anvil->nice_exit({exit_code => 0});
 # Functions                                                                                                 #
 #############################################################################################################
 
+sub check_drbd
+{
+	my ($anvil) = @_;
+	
+	
+	
+	return(0);
+}
+
 # This boots the servers.
 sub boot_servers
 {

tools/anvil-update-states

@@ -437,7 +437,7 @@ sub update_network
 	closedir(DIRECTORY);
 	
 	# Find what interfaces are connected to which bridges
-	$anvil->Network->bridge_info({debug => 3});
+	$anvil->Network->bridge_info({debug => 2});
 	delete $anvil->data->{interface_to_bridge} if exists $anvil->data->{interface_to_bridge};
 	foreach my $bridge_name (sort {$a cmp $b} keys %{$anvil->data->{bridge}{$local_host}})
 	{
@@ -538,7 +538,7 @@ sub update_network
 					if (($bridge_uuid) && ($ip_address))
 					{
 						my $ip_address_uuid = $anvil->Database->insert_or_update_ip_addresses({
-							debug                      => 3,
+							debug                      => 2,
 							file                       => $THIS_FILE,
 							line                       => __LINE__,
 							ip_address_on_type         => $type, 
@@ -595,7 +595,7 @@ sub update_network
 					if (($bond_uuid) && ($ip_address))
 					{
 						my $ip_address_uuid = $anvil->Database->insert_or_update_ip_addresses({
-							debug                      => 3,
+							debug                      => 2,
 							file                       => $THIS_FILE,
 							line                       => __LINE__,
 							ip_address_on_type         => $type, 
@@ -658,7 +658,7 @@ sub update_network
 					if (($network_interface_uuid) && ($ip_address))
 					{
 						my $ip_address_uuid = $anvil->Database->insert_or_update_ip_addresses({
-							debug                      => 3,
+							debug                      => 2,
 							file                       => $THIS_FILE,
 							line                       => __LINE__,
 							ip_address_on_type         => $type, 
@@ -951,7 +951,7 @@ WHERE
 		my $ip_address_default_gateway = $row->[6];
 		my $ip_address_dns             = $row->[7];
 		my $ip_address_note            = $row->[8];
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			ip_address_on_type         => $ip_address_on_type,
 			ip_address_on_uuid         => $ip_address_on_uuid,
 			ip_address_address         => $ip_address_address,