anvil/scancore-agents/scan-server/scan-server

#!/usr/bin/perl
# 
# This scans the nodes and DR host for VMs 
# 
# NOTE: The data stored here is not bound to a given host. As such, only hosted VMs are processed.
# 
# Examples;
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Startup failure (not running as root, no DB, bad file read, etc)
# 2 = libvirtd is not running.
# 
# BUG:
# - Check that an update on disk is not overwritten by the old config still being in memory for a still-
#   running VM (specifically, RAM updates)
# 
# TODO: 
# - Move location constraints to the host node if the server is not on the preferred host (this happens after
#   recovering from a node loss).
# - Update the fence delay to favour the active host
# - If a server isn't running, and the database definition has been updated, it isn't updated on disk.
# 

use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
use Text::Diff;

# Disable buffering
$| = 1;

# Prevent a discrepency between UID/GID and EUID/EGID from throwing an error.
$< = $>;
$( = $);

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

my $anvil = Anvil::Tools->new();

# Make sure we're running as 'root'
# $< == real UID, $> == effective UID
if (($< != 0) && ($> != 0))
{
	# Not root
	print $anvil->Words->string({key => "error_0005"})."\n";
	$anvil->nice_exit({exit_code => 1});
}

$anvil->data->{scancore}{'scan-server'}{disable}         = 0;
$anvil->data->{scancore}{'scan-server'}{'auto-undefine'} = 1;
$anvil->data->{switches}{force}                          = 0;

$anvil->Storage->read_config();

# Read switches
$anvil->Get->switches;

# Handle start-up tasks
my $problem = $anvil->ScanCore->agent_startup({agent => $THIS_FILE});
if ($problem)
{
	$anvil->nice_exit({exit_code => 1});
}

$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_server_log_0001", variables => { program => $THIS_FILE }});

# There are no tables for this agent, so '--purge' is useless here.

# Before we do anything, are we a node or a DR host?
my $host_type = $anvil->Get->host_type;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }});
if ($host_type eq "striker")
{
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_server_log_0002", variables => { host_type => $host_type }});
	$anvil->nice_exit({exit_code => 0});
}

# This is more than data collection in most agents, as it actually handles the changes on the fly
collect_data($anvil);

# Look for migration times written out by ocf:alteeve:server.
record_migration_times($anvil);

# Check that there's a DRBD fence rule for each server.
check_drbd_fence_rules($anvil);

# Get screenshot from every server that is alive.
get_screenshots($anvil);

# Shut down.
$anvil->ScanCore->agent_shutdown({agent => $THIS_FILE});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# Check that there's a DRBD fence rule for each server.
sub check_drbd_fence_rules
{
	my ($anvil) = @_;
	
	my $host_type = $anvil->Get->host_type();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
	if ($host_type ne "node")
	{
		return(0);
	}
	
	my $problem = $anvil->Cluster->parse_cib();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
	if ($problem)
	{
		return(0);
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"cib::parsed::local::ready" => $anvil->data->{cib}{parsed}{'local'}{ready},
	}});
	if (not $anvil->data->{cib}{parsed}{'local'}{ready})
	{
		return(0);
	}
	
	foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{'scan-server'}{server_name}})
	{
		my $drbd_fence_rule_exists    = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_rule}{'exists'};
		my $drbd_fence_rule_attribute = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_rule}{attribute};
		my $drbd_fence_rule_operation = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_rule}{operation};
		my $drbd_fence_rule_value     = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_rule}{value};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:server_name'               => $server_name,
			's2:drbd_fence_rule_exists'    => $drbd_fence_rule_exists, 
			's3:drbd_fence_rule_attribute' => $drbd_fence_rule_attribute, 
			's4:drbd_fence_rule_operation' => $drbd_fence_rule_operation, 
			's5:drbd_fence_rule_value'     => $drbd_fence_rule_value, 
		}});
		
		### TODO: Verify that the other values are correct.
		# If it's missing, add it 
		if (not $drbd_fence_rule_exists)
		{
			# Create it.
			my $variables = {
				server => $server_name,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0019", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0019", variables => $variables, set_by => $THIS_FILE});
			
			# 
			my $shell_call = $anvil->data->{path}{exe}{pcs}." constraint location ".$server_name." rule score=-INFINITY drbd-fenced_".$server_name." eq 1";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output, 
				return_code => $return_code,
			}});
		}
	}
	
	return(0);
}

# Gets a screenshot for each server that is alive.
sub get_screenshots
{
	my $anvil      = shift;
	my $parameters = shift;
	my $debug      = $parameters->{debug} // 3;

	my $anvil_uuid      = $anvil->Cluster->get_anvil_uuid();
	my $local_host_uuid = $anvil->Get->host_uuid();

	my $query   = "SELECT host_name FROM hosts WHERE host_type = 'striker';";
	my $results = $anvil->Database->query({ query => $query, source => $THIS_FILE, line => __LINE__ });
	my $count   = @{$results};

	$anvil->Log->variables({ source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query, count => $count } });

	return (1) if ($count == 0);

	# Start the CSV with the first element, then append.
	my $striker_name_csv = $results->[0]->[0];

	foreach my $row ( @{$results}[1 .. $#{$results}] )
	{
		my $host_name = $row->[0];

		$striker_name_csv = "$striker_name_csv,$host_name";
	}

	$anvil->Log->variables({ source => $THIS_FILE, line => __LINE__, level => $debug, list => { striker_name_csv => $striker_name_csv } });

	foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
	{
		my $server_uuid  = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};

		my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
		my $server_state     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};

		$anvil->Log->variables({ source => $THIS_FILE, line => __LINE__, level => $debug, list => {
			server_host_uuid => $server_host_uuid,
			server_name      => $server_name,
			server_state     => $server_state,
			server_uuid      => $server_uuid,
		} });

		next if ( ($server_host_uuid ne $local_host_uuid) || (not $server_state eq "running") );

		my ($syscall_output, $syscall_rcode) = $anvil->System->call({
			debug      => $debug,
			line       => __LINE__,
			shell_call => $anvil->data->{path}{exe}{'anvil-get-server-screenshot'}." --server-uuid '$server_uuid' --request-host-name '$striker_name_csv' &",
			source     => $THIS_FILE,
		});

		$anvil->Log->variables({ source => $THIS_FILE, line => __LINE__, level => $debug, list => {
			syscall_output => $syscall_output,
			syscall_rcode  => $syscall_rcode,
		}});
	}

	return (0);
}

# Look for migration times written out by ocf:alteeve:server.
sub record_migration_times
{
	my ($anvil) = @_;
	
	my $directory = "/tmp/anvil";
	if (-d $directory)
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { directory => $directory }});
		local(*DIRECTORY);
		opendir(DIRECTORY, $directory);
		while(my $file = readdir(DIRECTORY))
		{
			next if $file eq ".";
			next if $file eq "..";
			next if $file !~ /^migration-duration\./;
			my $full_path = $directory."/".$file;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				file      => $file,
				full_path => $full_path,
			}});
			
			my $body =  $anvil->Storage->read_file({file => $full_path});
			   $body =~ s/\n//;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { body => $body }});
			if ($body =~ /server_name=(.*?),migration_took=(.*?)$/)
			{
				my $server_name    = $1;
				my $migration_took = $2;
				my $anvil_uuid     = $anvil->Cluster->get_anvil_uuid;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					server_name    => $server_name,
					migration_took => $migration_took, 
					anvil_uuid     => $anvil_uuid, 
				}});
				my $server_uuid = $anvil->Get->server_uuid_from_name({
					server_name => $server_name,
					anvil_uuid  => $anvil_uuid,
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_uuid => $server_uuid }});
				if (($server_uuid) && ($migration_took))
				{
					my ($variable_uuid) = $anvil->Database->insert_or_update_variables({
						file                  => $THIS_FILE, 
						line                  => __LINE__, 
						variable_name         => "server::migration_duration", 
						variable_value        => $migration_took, 
						variable_default      => "", 
						variable_description  => "message_0236", 
						variable_section      => "servers", 
						variable_source_uuid  => $server_uuid, 
						variable_source_table => "servers", 
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }});
				}
			}
			unlink $full_path;
		}
	}
	
	return(0);
}

# This reads in all the data we can find about servers running locally. This is more than data collection in
# most agents, as it actually handles the changes on the fly.
sub collect_data
{
	my ($anvil) = @_;
	
	# NOTE: We don't check if libvirtd is running anymore as it auto-starts / stops on EL8+.
	# Load data we know about
	$anvil->Database->get_anvils();
	$anvil->Database->get_servers();
	$anvil->Database->get_server_definitions();

	# Find our Anvil! UUID
	my $target     = $anvil->Get->short_host_name();
	my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { anvil_uuid => $anvil_uuid }});
	
	# Look for any servers in the database but not yet written to disk.
	if (exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid})
	{
		foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
		{
			my $server_uuid  = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
			my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				's1:server_name'  => $server_name,
				's2:server_state' => $server_state, 
				's3:server_uuid'  => $server_uuid, 
			}});
			
			next if $server_state eq "DELETED";
			
			# If the definition file doesn't exist at all, create the file.
			my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { xml_file => $xml_file }});
			if (not -f $xml_file)
			{
				my $server_definition = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition => $server_definition }});
				
				# Make very sure the definition XML is valid
				if (not $server_definition)
				{
					# Asked to write an empty definition file!
					my $variables = {
						server => $server_name, 
					};
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0020", variables => $variables });
					$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0020", variables => $variables, set_by => $THIS_FILE});
					$anvil->nice_exit({exit_code => 1});
				}
				
				local $@;
				my $xml        = XML::Simple->new();
				my $server_xml = "";
				my $test       = eval { $server_xml = $xml->XMLin($server_definition, KeyAttr => {}, ForceArray => 1) };
				if (not $test)
				{
					chomp $@;
					my $error =  "[ Error ] - The was a problem parsing: [".$server_definition."]. The error was:\n";
					$error .= "===========================================================\n";
					$error .= $@."\n";
					$error .= "===========================================================\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", list => { error => $error }});
					return(1);
				}
				
				if ($server_definition)
				{
					# Register an alert.
					my $variables = {
						server => $server_name,
					};
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0016", variables => $variables});
					$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0016", variables => $variables, set_by => $THIS_FILE});
					
					my $return = $anvil->Storage->write_file({
						body      => $server_definition,
						file      => $xml_file,
						overwrite => 1,
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'return' => $return }});
				}
			}
		}
	}
	
	my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list --all";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	
	my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code,
	}});
	foreach my $line (split/\n/, $output)
	{
		$line = $anvil->Words->clean_spaces({string => $line});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
	
		# This is set to 1 when we add a server, supressing detailed change checks as we'll have 
		# already registered an alert.
		my $added = 0;
		
		### TODO: We may want to handle crashed / idle servers.
=cut
* Server states;
running     - The domain is currently running on a CPU
idle        - The domain is idle, and not running or runnable.  This can be caused because the domain is waiting on IO (a traditional wait state) or has gone to sleep because there was nothing else for it to do.
paused      - The domain has been paused. This can happen when a server is migrating to this host, or through the administrator running virsh suspend.  When in a paused state the domain will still consume allocated resources like memory, but will not be eligible for scheduling by the hypervisor.
in shutdown - The domain is in the process of shutting down, i.e. the guest operating system has been notified and should be in the process of stopping its operations gracefully.
shut off    - The domain is not running.  Usually this indicates the domain has been shut down completely, or has not been started.
crashed     - The domain has crashed, which is always a violent ending.  Usually this state can only occur if the domain has been configured not to restart on crash.
pmsuspended - The domain has been suspended by guest power management, e.g. entered into s3 state.
- Ones we set:
migrating - Set and cleared by Server->migrate_virsh();
DELETED   - Marks a server as no longer existing
=cut
		
		if (($line =~ /^\d+ (.*?) (.*)$/) or ($line =~ /- (.*?) (.*)$/))
		{
			my $server_name  = $1;
			my $server_state = $2;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				server_name  => $server_name, 
				server_state => $server_state, 
			}});
			
			# Parse out the server UUID.
			my $virsh_definition = get_and_parse_virsh_definition($anvil, $server_name);
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { virsh_definition => $virsh_definition }});
			
			# Does the XML definition file exist yet?
			my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { xml_file => $xml_file }});
			if (not -e $xml_file)
			{
				# No, generate it. This will also load and parse the file after it's written.
				my $on_disk_definition = update_on_disk_definition($anvil, $server_name, $virsh_definition);
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { on_disk_definition => $on_disk_definition }});
			}
			else
			{
				# Yes, parse it.
				my $on_disk_definition = get_and_parse_disk_definition($anvil, $server_name);
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { on_disk_definition => $on_disk_definition }});
			}
			
			# We'll compare the memory allocated to the server from the on-disk definition and the memory
			# currently used as reported by the in-memory definition and use that to update the 
			# 'server_ram_in_use' and 'server_configured_ram' columns.
			my $server_uuid           = $anvil->data->{server}{$target}{$server_name}{'from_virsh'}{info}{uuid};
			my $server_ram_in_use     = $anvil->data->{server}{$target}{$server_name}{'from_virsh'}{memory};
			my $server_configured_ram = $anvil->data->{server}{$target}{$server_name}{'from_disk'}{memory};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				server_uuid           => $server_uuid,
				server_ram_in_use     => $anvil->Convert->add_commas({number => $server_ram_in_use})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_in_use}).")",
				server_configured_ram => $anvil->Convert->add_commas({number => $server_configured_ram})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_configured_ram}).")",
			}});
			
			# If we saw the server boot (or saw it for the first time at all), this will store 
			# the boot time.
			my $server_boot_time = 0;
			
			# Is this server in the database already?
			if (exists $anvil->data->{servers}{server_uuid}{$server_uuid})
			{
				# Yes, see if we're looking at a state change
				if ($anvil->data->{servers}{server_uuid}{$server_uuid}{server_state} eq "migrating")
				{
					# Ignore this server, scan_cluster will clear this when the migration is 
					# complete. We don't touch anything during a migration.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_message_0001", variables => { server => $server_name }});
					next;
				}
				elsif ($anvil->data->{servers}{server_uuid}{$server_uuid}{server_state} eq "shut off")
				{
					# It booted. Record the start time.
					my $runtime          = $anvil->Server->get_runtime({server => $server_name});
					my $server_boot_time = time - $runtime;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						runtime          => $runtime,
						server_boot_time => $server_boot_time." (".$anvil->Get->date_and_time({use_time => $server_boot_time}).")",
					}});
				}
			}
			else
			{
				# It's not in the database yet, add it. 
				   $added            = 1;
				my $runtime          = $anvil->Server->get_runtime({server => $server_name});
				my $server_boot_time = time - $runtime;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					runtime          => $runtime,
					server_boot_time => $server_boot_time." (".$anvil->Get->date_and_time({use_time => $server_boot_time}).")",
				}});

				my $got_server_uuid = $anvil->Database->insert_or_update_servers({
					debug                 => 2, 
					server_uuid           => $server_uuid, 
					server_name           => $server_name, 
					server_state          => $server_state, 
					server_anvil_uuid     => $anvil_uuid, 
					server_host_uuid      => $anvil->Get->host_uuid, 
					server_ram_in_use     => $server_ram_in_use, 
					server_configured_ram => $server_configured_ram, 
					server_boot_time      => $server_boot_time,
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { got_server_uuid => $got_server_uuid }});
				
				# Store the definition from virsh
				my $server_definition_uuid = $anvil->Database->insert_or_update_server_definitions({
					debug                         => 2,
					server_definition_xml         => $virsh_definition,  
					server_definition_server_uuid => $server_uuid, 
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { server_definition_uuid => $server_definition_uuid }});
				
				# Make sure the firewall is updated.
				$anvil->Network->manage_firewall();
				
				# Reload the servers.
				$anvil->Database->get_servers();
				$anvil->Database->get_server_definitions();
				
				# Register an alert.
				my $variables  = {
					server     => $server_name,
					definition => $virsh_definition,
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0006", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0006", variables => $variables, set_by => $THIS_FILE});
			}
			
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_uuid}           = $server_uuid;
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time}      = $server_boot_time ? $server_boot_time : $anvil->data->{servers}{server_uuid}{$server_uuid}{server_boot_time};
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_state}          = $server_state;
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_anvil_uuid}     = $anvil_uuid;
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_host_uuid}      = $anvil->Get->host_uuid;
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_ram_in_use}     = $server_ram_in_use;
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_configured_ram} = $server_configured_ram;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				"scan-server::server_name::${server_name}::server_uuid"           => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_uuid},
				"scan-server::server_name::${server_name}::server_boot_time"      => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time}." (".$anvil->Get->date_and_time({use_time => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time}}).")", 
				"scan-server::server_name::${server_name}::server_state"          => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_state}, 
				"scan-server::server_name::${server_name}::server_anvil_uuid"     => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_anvil_uuid}, 
				"scan-server::server_name::${server_name}::server_host_uuid"      => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_host_uuid}, 
				"scan-server::server_name::${server_name}::server_ram_in_use"     => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_ram_in_use}, 
				"scan-server::server_name::${server_name}::server_configured_ram" => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_configured_ram}, 
			}});
			
			if ($server_state eq "shut off")
			{
				my $host_type = $anvil->Get->host_type;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
				if ($host_type eq "node")
				{
					# The definition was likely changed by the user while it was running. We 
					# should have already picked up the changes, but just to be safe, check for
					# differences.
					my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." dumpxml --inactive ".$server_name;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($virsh_definition, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $virsh_definition, 
						return_code => $return_code,
					}});
					
					# Make sure the definition we read was valid.
					my $problem = $anvil->Server->parse_definition({
						server     => $server_name,
						source     => "from_virsh", 
						definition => $virsh_definition, 
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
					
					if (not $problem)
					{
						# The definition may have certainly changed, so update it in case 
						# needed.
						update_definitions_from_virsh($anvil, $server_name, $server_uuid, $virsh_definition);
						
						# Now undefine the server
						$shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." undefine ".$server_name;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
						
						(my $output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							output      => $output, 
							return_code => $return_code,
						}});
					}
				}
				
				# Set the boot time back to zero.
				$anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time} = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"scan-server::server_name::${server_name}::server_boot_time" => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time}, 
				}});
			}
		}
	}
	
	# Now loop through the found servers and see if any definitions have changed and need to be 
	# propogated out.
	foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{'scan-server'}{server_name}})
	{
		# Set the XML file path and read the UUID
		my $xml_file    = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
		my $server_uuid = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			server_name => $server_name,
			server_uuid => $server_uuid, 
			xml_file    => $xml_file,
		}});
		
		# Record that we've already checked this definition file so we don't waste time doing it 
		# again below.
		$anvil->data->{checked_definition_files}{$xml_file} = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"checked_definition_files::${xml_file}" => $anvil->data->{checked_definition_files}{$xml_file},
		}});
		
		# Get the definitions
		my $virsh_definition    = get_and_parse_virsh_definition($anvil, $server_name);
		my $database_definition = get_and_parse_database_definition($anvil, $server_name, $server_uuid);
		my $on_disk_definition  = get_and_parse_disk_definition($anvil, $server_name);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			virsh_definition    => $virsh_definition,
			database_definition => $database_definition, 
			on_disk_definition  => $on_disk_definition,
		}});

		# If the 'server_updated_by_user' value is newer than the file age, and there is a difference in the definition, update the file.
		my $current_time           = time;
		my $user_update_time       = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_updated_by_user};
		my $user_update_age        = $current_time - $user_update_time;
		my $database_modified_time = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{unix_modified_time};
		my $database_age           = $current_time - $database_modified_time;
		my $file_modified_time     = $anvil->data->{file_stat}{$xml_file}{modified_time};
		my $file_age               = $current_time - $file_modified_time;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:current_time'           => $current_time,
			's2:user_update_time'       => $user_update_time, 
			's3:user_update_age'        => $user_update_age, 
			's4:database_modified_time' => $database_modified_time, 
			's5:database_age'           => $database_age, 
			's6:file_modified_time'     => $file_modified_time, 
			's7:file_age'               => $file_age, 
		}});
		
		### NOTE: A simplified version of this is below when checking files on disk for servers that 
		###       aren't running. If working on this section, check below as well.
		# If the user edited the server via Striker or directly via the on-disk definition, update 
		# the other and then load the changes into virsh.
		if ($database_definition ne $on_disk_definition)
		{
			# Either the change came from Striker (check $user_update_age) or the change happened
			# on disk (check $file_modified_time). Which ever is more recent wins.
			if ($user_update_age < $file_modified_time)
			{
				# Get the diffs before we update
				my $disk_difference  = diff \$database_definition, \$on_disk_definition, { STYLE => 'Unified' };
				my $virsh_difference = diff \$database_definition, \$virsh_definition, { STYLE => 'Unified' };
				
				# The user updated the definition from Striker, so update the file and then push the new file into virsh.
				   $on_disk_definition = update_on_disk_definition($anvil, $server_name, $database_definition);
				   $file_modified_time = $anvil->data->{file_stat}{$xml_file}{modified_time};
				   $file_age           = $current_time - $file_modified_time;
				my $variables          = {
					server           => $server_name,
					definition_file  => $xml_file,
					disk_difference  => $disk_difference,
					virsh_difference => $virsh_difference,
					new_definition   => $database_definition,
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0001", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0001", variables => $variables, set_by => $THIS_FILE});
			}
			else
			{
				# Get the diffs before we update
				my $db_difference    = diff \$on_disk_definition, \$database_definition, { STYLE => 'Unified' };
				my $virsh_difference = diff \$database_definition, \$virsh_definition, { STYLE => 'Unified' };
				
				# The disk was updated, so push the change into the database and into virsh.
				   $database_definition = update_database_definition($anvil, $server_name, $server_uuid, $on_disk_definition);
				my $variables           = {
					server           => $server_name,
					db_difference    => $db_difference,
					virsh_difference => $virsh_difference,
					new_definition   => $on_disk_definition,
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0002", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0002", variables => $variables, set_by => $THIS_FILE});
			}
			
			# In either case, update the virsh definition from the disk now (either to load the 
			# changes that originated from the database or from a direct file edit).
			$virsh_definition = redefine_server_from_disk($anvil, $server_name);
		}
		elsif (($virsh_definition ne $database_definition) && ($virsh_definition ne $on_disk_definition))
		{
			# The last possible source of change is a direct edit of the definition via a virsh 
			# tool, like virt-manager or a command line virsh call. Either way, the virsh 
			# definition won't match either the database or the on-disk. So in this case, we 
			# use the virsh definition to update both.
			my $disk_difference = diff \$virsh_definition, \$on_disk_definition, { STYLE => 'Unified' };
			my $db_difference   = diff \$virsh_definition, \$database_definition, { STYLE => 'Unified' };
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				disk_difference => $disk_difference,
				db_difference   => $db_difference, 
			}});
			
			   $on_disk_definition  = update_on_disk_definition($anvil, $server_name, $virsh_definition);
			   $database_definition = update_database_definition($anvil, $server_name, $server_uuid, $virsh_definition);
			my $variables           = {
				server          => $server_name,
				definition_file => $xml_file,
				disk_difference => $disk_difference,
				db_difference   => $db_difference,
				new_definition  => $virsh_definition,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0003", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0003", variables => $variables, set_by => $THIS_FILE});
		}
		
		my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." dumpxml ".$server_name;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($virsh_live_definition, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $virsh_definition, return_code => $return_code }});
		my $problem = $anvil->Server->parse_definition({
			server     => $server_name,
			source     => "from_live_virsh", 
			definition => $virsh_live_definition, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		if ($problem)
		{
			$virsh_live_definition = "";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { virsh_live_definition => $virsh_live_definition }});
		}
		
		# Now that definition updates are dealth with, has anything else changed?
		my $server_state              = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_state};
		my $server_boot_time          = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time};
		my $server_anvil_uuid         = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_anvil_uuid};
		my $server_user_stop          = $server_state eq "shut off" ? $anvil->data->{servers}{server_uuid}{$server_uuid}{server_user_stop} : 0;
		my $server_host_uuid          = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_host_uuid};
		my $server_ram_in_use         = $anvil->data->{server}{$target}{$server_name}{from_live_virsh}{memory};
		my $server_configured_ram     = $anvil->data->{server}{$target}{$server_name}{from_disk}{memory};
		# Old values
		my $old_server_name           = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
		my $old_server_state          = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
		my $old_server_boot_time      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_boot_time};
		my $old_server_anvil_uuid     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid};
		my $old_server_user_stop      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_user_stop};
		my $old_server_host_uuid      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
		my $old_server_ram_in_use     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_ram_in_use};
		my $old_server_configured_ram = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_configured_ram};
		
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:server_uuid'                => $server_uuid,
			's2:server_name'                => $server_name,
			's3:old_server_name'            => $old_server_name,
			's4:server_state'               => $server_state, 
			's5:old_server_state'           => $old_server_state, 
			's6:server_boot_time'           => $server_boot_time." (".$anvil->Get->date_and_time({use_time => $server_boot_time}).")",
			's7:old_server_boot_time'       => $old_server_boot_time." (".$anvil->Get->date_and_time({use_time => $old_server_boot_time}).")", 
			's8:server_anvil_uuid'          => $server_anvil_uuid, 
			's9:old_server_anvil_uuid'      => $old_server_anvil_uuid, 
			's10:server_user_stop'          => $server_user_stop,
			's11:$old_server_user_stop'     => $old_server_user_stop, 
			's12:server_host_uuid'          => $server_host_uuid, 
			's13:old_server_host_uuid'      => $old_server_host_uuid, 
			's14:server_ram_in_use'         => $anvil->Convert->add_commas({number => $server_ram_in_use})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_in_use}).")", 
			's15:old_server_ram_in_use'     => $anvil->Convert->add_commas({number => $old_server_ram_in_use})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $old_server_ram_in_use}).")", 
			's16:server_configured_ram'     => $anvil->Convert->add_commas({number => $server_configured_ram})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_configured_ram}).")",
			's17:old_server_configured_ram' => $anvil->Convert->add_commas({number => $old_server_configured_ram})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $old_server_configured_ram}).")",
		}});
		
		# Find changes.
		my $update = 0;
		if ($server_name ne $old_server_name)
		{
			# Server name has changed.
			my $variables  = { 
				old_name => $old_server_name,
				new_name => $server_name,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0004", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0004", variables => $variables, set_by => $THIS_FILE });
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		if ($server_state ne $old_server_state)
		{
			# The state has changed.
			my $variables  = { 
				server    => $server_name,
				old_state => $old_server_state,
				new_state => $server_state,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0005", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0005", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
			
			# Make sure the boot time has been updated if the state has transitioned from "shut 
			# off".
			if ($old_server_state eq "shut off")
			{
				my $runtime          = $anvil->Server->get_runtime({server => $server_name});
				my $server_boot_time = time - $runtime;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					server_boot_time => $server_boot_time." (".$anvil->Get->date_and_time({use_time => $server_boot_time}).")",
				}});
			}
			
			# If the server was migrated or booted, update the firewall. With luck, this happened
			# already, so this is a back-stop mainly.
			if (($old_server_state eq "migrating") or ($old_server_state eq "shut off"))
			{
				# Make sure the firewall is updated.
				$anvil->Network->manage_firewall();
			}
		}
		if ($server_boot_time ne $old_server_boot_time)
		{
			# Boot time changed (expected on shut down or boot up)
			my $variables  = { 
				server              => $server_name,
				old_boot_time_date  => $anvil->Get->date_and_time({use_time => $old_server_boot_time}),
				old_boot_time_epoch => $anvil->Convert->add_commas({number => $old_server_boot_time}),
				new_boot_time_date  => $anvil->Get->date_and_time({use_time => $server_boot_time}),
				new_boot_time_epoch => $anvil->Convert->add_commas({number => $server_boot_time}),
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0008", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0008", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		if ($server_anvil_uuid ne $old_server_anvil_uuid)
		{
			# Moved between Anvil! systems
			my $variables  = { 
				server         => $server_name,
				old_anvil_name => $anvil->Get->anvil_name_from_uuid({anvil_uuid => $old_server_anvil_uuid}),
				old_anvil_uuid => $old_server_anvil_uuid,
				new_anvil_name => $anvil->Get->anvil_name_from_uuid({anvil_uuid => $server_anvil_uuid}),
				new_anvil_uuid => $server_anvil_uuid,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0009", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0009", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		if ($server_user_stop ne $old_server_user_stop)
		{
			# Server booted and cleared this or the user shut down the server and this got set)
			my $say_user_stop = $server_user_stop eq "TRUE" ? "scan_server_alert_0010" : "scan_server_alert_0011";
			my $variables     = { server => $server_name };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => $say_user_stop, variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => $say_user_stop, variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		if ($server_host_uuid ne $old_server_host_uuid)
		{
			# Server migrated (to the peer or to a new Anvil!)
			my $variables  = { 
				server        => $server_name,
				old_host_name => $old_server_host_uuid eq "NULL" ? "NULL" : $anvil->Get->host_name_from_uuid({host_uuid => $old_server_host_uuid}),
				old_host_uuid => $old_server_host_uuid,
				new_host_name => $server_host_uuid     eq "NULL" ? "NULL" : $anvil->Get->host_name_from_uuid({host_uuid => $server_host_uuid}),
				new_host_uuid => $server_host_uuid,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0012", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0012", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		if ($server_ram_in_use ne $old_server_ram_in_use)
		{
			# The amount of RAM the server uses has changed, likely after rebooting after a 
			# config change.
			my $variables  = { 
				server               => $server_name,
				old_ram_in_use_short => $anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_in_use}),
				old_ram_in_use_bytes => $anvil->Convert->add_commas({number => $server_ram_in_use}),
				new_ram_in_use_short => $anvil->Convert->bytes_to_human_readable({'bytes' => $old_server_ram_in_use}),
				new_ram_in_use_bytes => $anvil->Convert->add_commas({number => $old_server_ram_in_use}),
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0013", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0013", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		if ($server_configured_ram ne $old_server_configured_ram)
		{
			# The amount of allocated RAM has changed, but if this value doesn't match
			# 'server_ram_in_use', the user needs to reboot. 
			my $say_ram   = $server_configured_ram eq $server_ram_in_use ? "scan_server_alert_0014" : "scan_server_alert_0015";
			my $variables = { 
				server                   => $server_name,
				old_configured_ram_short => $anvil->Convert->bytes_to_human_readable({'bytes' => $server_configured_ram}),
				old_configured_ram_bytes => $anvil->Convert->add_commas({number => $server_configured_ram}),
				new_configured_ram_short => $anvil->Convert->bytes_to_human_readable({'bytes' => $old_server_configured_ram}),
				new_configured_ram_bytes => $anvil->Convert->add_commas({number => $old_server_configured_ram}),
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => $say_ram, variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => $say_ram, variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }});
		if ($update)
		{
			# Update.
			$anvil->Database->insert_or_update_servers({
				debug                           => 2,
				server_uuid                     => $server_uuid,
				server_name                     => $server_name,
				server_state                    => $server_state, 
				server_boot_time                => $server_boot_time,
				server_anvil_uuid               => $server_anvil_uuid,
				server_user_stop                => $server_user_stop,
				server_host_uuid                => $server_host_uuid,
				server_ram_in_use               => $server_ram_in_use,
				server_configured_ram           => $server_configured_ram,
				server_start_after_server_uuid  => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_after_server_uuid},
				server_start_delay              => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_delay},
				server_live_migration           => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_live_migration},
				server_pre_migration_file_uuid  => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_pre_migration_file_uuid},
				server_pre_migration_arguments  => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_pre_migration_arguments},
				server_post_migration_file_uuid => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_post_migration_file_uuid},
				server_post_migration_arguments => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_post_migration_arguments},
			});
		}
		
		# Delete the database record so we know this one has been dealt with.
		delete $anvil->data->{servers}{server_uuid}{$server_uuid};
	}
	
	# If servers are off, and the definition file is edited, we need to be able to pick that up and 
	# propogate it out.
	my $directory = $anvil->data->{path}{directories}{shared}{definitions};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { directory => $directory }});

	local(*DIRECTORY);
	opendir(DIRECTORY, $directory);
	while(my $file = readdir(DIRECTORY))
	{
		next if $file eq ".";
		next if $file eq "..";
		my $xml_file = $directory."/".$file;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			file     => $file,
			xml_file => $xml_file,
		}});
		my $server_name = ($file =~ /^(.*)\.xml$/)[0];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_name => $server_name }});
		next if not $server_name; 
		
		my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_uuid => $server_uuid }});
		next if not $server_uuid;
		
		# If the server is running, we would have processed it above.
		if (exists $anvil->data->{checked_definition_files}{$xml_file})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"checked_definition_files::${xml_file}" => $anvil->data->{checked_definition_files}{$xml_file},
			}});
			next if $anvil->data->{checked_definition_files}{$xml_file};
		}
		
		# Still here? check the age of the file on disk and the age of the database entry.
		my $database_definition = get_and_parse_database_definition($anvil, $server_name, $server_uuid);
		my $on_disk_definition  = get_and_parse_disk_definition($anvil, $server_name);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			database_definition => $database_definition, 
			on_disk_definition  => $on_disk_definition,
		}});

		# If the 'server_updated_by_user' value is newer than the file age, and there is a difference in the definition, update the file.
		my $current_time           = time;
		my $database_modified_time = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{unix_modified_time};
		my $database_age           = $current_time - $database_modified_time;
		my $file_modified_time     = $anvil->data->{file_stat}{$xml_file}{modified_time};
		my $file_age               = $current_time - $file_modified_time;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:current_time'           => $current_time,
			's2:database_modified_time' => $database_modified_time, 
			's3:database_age'           => $database_age, 
			's4:file_modified_time'     => $file_modified_time, 
			's5:file_age'               => $file_age, 
		}});
		if ($database_definition ne $on_disk_definition)
		{
			# Is the database version or on-file newer?
			if ($file_modified_time > $database_modified_time)
			{
				# File is newer, Get the diffs before we update
				my $db_difference    = diff \$on_disk_definition, \$database_definition, { STYLE => 'Unified' };
				
				# The disk was updated, so push the change into the database and into virsh.
				   $database_definition = update_database_definition($anvil, $server_name, $server_uuid, $on_disk_definition);
				my $variables           = {
					server         => $server_name,
					db_difference  => $db_difference,
					new_definition => $on_disk_definition,
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0017", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0017", variables => $variables, set_by => $THIS_FILE});
			}
			else
			{
				# DB is newer
				my $disk_difference  = diff \$database_definition, \$on_disk_definition, { STYLE => 'Unified' };
				
				# The user updated the definition from Striker, so update the file and then push the new file into virsh.
				   $on_disk_definition = update_on_disk_definition($anvil, $server_name, $database_definition);
				   $file_modified_time = $anvil->data->{file_stat}{$xml_file}{modified_time};
				   $file_age           = $current_time - $file_modified_time;
				my $variables          = {
					server           => $server_name,
					definition_file  => $xml_file,
					disk_difference  => $disk_difference,
					new_definition   => $database_definition,
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0018", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0018", variables => $variables, set_by => $THIS_FILE});
			}
		}
	}
	
	# Lastly, if we can access our peer, see what servers are running there. This will let us know if a 
	# server that had been running on us before is not off (versus migrated).
	$anvil->Cluster->get_peers();
	my $peer_is   = $anvil->data->{sys}{anvil}{peer_is};
	my $peer_name = $anvil->data->{sys}{anvil}{$peer_is}{host_name};
	my $peer_uuid = $anvil->data->{sys}{anvil}{$peer_is}{host_uuid};
	my $password  = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:peer_is'   => $peer_is,
		's2:peer_name' => $peer_name, 
		's3:peer_uuid' => $peer_uuid, 
		's4:password'  => $anvil->Log->is_secure($password), 
	}});
	
	# If we can access our peer or DR host, see what servers are running on them. We do no parsing or 
	# processing of servers on other machines, we only care what servers are running.
	my $peer_access = $anvil->Remote->test_access({
		target   => $peer_name,
		password => $password,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { peer_access => $peer_access }});
	if ($peer_access)
	{
		# Get a list of servers running on our peer. 
		my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list --all";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target      => $peer_name,
			password    => $password,
			shell_call  => $shell_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		foreach my $line (split/\n/, $output)
		{
			$line = $anvil->Words->clean_spaces({string => $line});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
			
			if ($line =~ /^\d+ (.*?) (.*)$/)
			{
				my $server_name  = $1;
				my $server_state = $2;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					server_name  => $server_name, 
					server_state => $server_state, 
				}});
				
				$anvil->data->{'scan-server'}{server_name}{$server_name}{server_state} = $server_state;
				$anvil->data->{'scan-server'}{server_name}{$server_name}{host}         = $peer_name;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"scan-server::server_name::${server_name}::server_state" => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_state}, 
					"scan-server::server_name::${server_name}::host"         => $anvil->data->{'scan-server'}{server_name}{$server_name}{host}, 
				}});
			}
		}
	}
	
	# Now, loop through all remaining servers from when we loaded them out of the database. Already 
	# processed servers will be removed, so what's left are either on the peer, or shut off now.
	foreach my $server_uuid (sort {$a cmp $b} keys %{$anvil->data->{servers}{server_uuid}})
	{
		my $server_name      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
		my $host_anvil_uuid  = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid};
		my $host_anvil_name  = $anvil->Get->anvil_name_from_uuid({anvil_uuid => $host_anvil_uuid});
		my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
		my $server_host_name = $server_host_uuid eq "NULL" ? "NULL" : $anvil->Get->host_name_from_uuid({host_uuid => $server_host_uuid});
		my $server_state     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:server_name'      => $server_name,
			's2:host_anvil_uuid'  => $host_anvil_uuid, 
			's3:host_anvil_name'  => $host_anvil_name, 
			's4:server_host_uuid' => $server_host_uuid, 
			's5:server_host_name' => $server_host_name, 
			's6:server_state'     => $server_state, 
		}});
		
		# Skip if this isn't our Anvil!
		next if $host_anvil_uuid ne $anvil_uuid;
		
		# If the server is on the peer, skip it.
		next if $server_host_uuid ne $anvil->Get->host_uuid;
		
		# If we're here, the server used to be on us, and isn't on the peer, so mark it as off.
		if (($server_state ne "shut off") && ($server_state ne "DELETED"))
		{
			# Mark it as being off now.
			my $query = "
UPDATE 
    servers 
SET 
    server_state     = 'shut off', 
    server_boot_time = '0' 
WHERE 
    server_uuid = ".$anvil->Database->quote($server_uuid)."
;";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
			$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
			
			# Now register an alert.
			my $variables  = { server => $server_name };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0007", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0007", variables => $variables, set_by => $THIS_FILE});
		}
	}
	
	return(0);
}

# This reads the definition file from the database and parses it.
sub get_and_parse_database_definition
{
	my ($anvil, $server_name, $server_uuid) = @_;
	
	my $database_definition = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_definition => $database_definition }});
	
	my $problem = $anvil->Server->parse_definition({
		server     => $server_name,
		source     => "from_db", 
		definition => $database_definition, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
	if ($problem)
	{
		# The definition is not valid.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_server_log_0003", variables => { definition => $database_definition }});
		$database_definition = "";
	}
	
	return($database_definition);
}

# This reads the definition file for a given server and parses it, returning the definition XML.
sub get_and_parse_disk_definition
{
	my ($anvil, $server_name) = @_;
	
	my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { xml_file => $xml_file }});

	my $on_disk_definition = $anvil->Storage->read_file({file => $xml_file});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { on_disk_definition => $on_disk_definition }});
	
	$anvil->Storage->get_file_stats({file_path => $xml_file});
	
	my $problem = $anvil->Server->parse_definition({
		debug      => 2,
		server     => $server_name,
		source     => "from_disk", 
		definition => $on_disk_definition, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
	if ($problem)
	{
		# The definition is not valid.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_server_log_0003", variables => { definition => $on_disk_definition }});
		$on_disk_definition = "";
	}
	
	return($on_disk_definition);
}

# This dumps the definition for a given server and parses it, returning the definition XML.
sub get_and_parse_virsh_definition
{
	my ($anvil, $server_name) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { server_name => $server_name }});

	my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." dumpxml --inactive ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	
	my ($virsh_definition, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $virsh_definition, 
		return_code => $return_code,
	}});
	
	my $problem = $anvil->Server->parse_definition({
		debug      => 2,
		server     => $server_name,
		source     => "from_virsh", 
		definition => $virsh_definition, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
	if ($problem)
	{
		# The definition is not valid.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_server_log_0003", variables => { definition => $virsh_definition }});
		$virsh_definition = "";
	}
	
	return($virsh_definition);
}

# This defines the server using the on-disk XML file. Effectively, this updates the 'inactive' XML definition
# in virsh.
sub redefine_server_from_disk
{
	my ($anvil, $server_name) = @_;
	
	my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { xml_file => $xml_file }});
	
	# Push the new definition into virsh (it won't take effect until a reboot likely, but it will update
	# the 'inactive' definition immediately.
	my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." define ".$xml_file;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	
	my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code,
	}});
	
	# Now undefine the server again so it disappears when stopped. 
	$shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." undefine ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	
	($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		output      => $output, 
		return_code => $return_code,
	}});
	
	# Re-read and parse the new (inactive) definition
	my $virsh_definition = get_and_parse_virsh_definition($anvil, $server_name);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { virsh_definition => $virsh_definition }});
	
	return($virsh_definition);
}

# This takes the XML definition from virsh and sees if it's different from the version in the database or on
# disk, updating them if so.
sub update_definitions_from_virsh
{
	my ($anvil, $server_name, $server_uuid, $virsh_definition) = @_;
	
	my $problem = $anvil->Server->parse_definition({
		debug      => 2,
		server     => $server_name,
		source     => "from_virsh", 
		definition => $virsh_definition, 
	});
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
	if (not $problem)
	{
		# Parsed successfully. Update the database (it'll check if something actually changed).
		my $server_definition_uuid = $anvil->Database->insert_or_update_server_definitions({
			debug                         => 2,
			server_definition_xml         => $virsh_definition,  
			server_definition_server_uuid => $server_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition_uuid => $server_definition_uuid }});
		
		# Reload the database definitions
		$anvil->Database->get_server_definitions();
		
		my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { xml_file => $xml_file }});
		
		# Just write out the file. If nothing else, it'll update the mtime.
		my $return = $anvil->Storage->write_file({
			debug     => 2,
			body      => $virsh_definition,
			file      => $xml_file,
			overwrite => 1,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'return' => $return }});
	}
	
	return(0);
}

# This updates the database definition and parses the new definition.
sub update_database_definition
{
	my ($anvil, $server_name, $server_uuid, $new_definition) = @_;
	
	my $server_definition_uuid = $anvil->Database->insert_or_update_server_definitions({
		debug                         => 2,
		server_definition_xml         => $new_definition,  
		server_definition_server_uuid => $server_uuid, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { server_definition_uuid => $server_definition_uuid }});

	my $problem = $anvil->Server->parse_definition({
		server     => $server_name,
		source     => "new_definition", 
		definition => $new_definition, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
	
	return($new_definition);
}

# This writes an XML definition to the on-disk definition file, then loads the file stats and parses the 
# definition.
sub update_on_disk_definition
{
	my ($anvil, $server_name, $new_definition) = @_;
	
	# Make very sure the definition XML is valid
	if (not $new_definition)
	{
		# Asked to write an empty definition file!
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0020", variables => $variables });
		$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0020", variables => $variables, set_by => $THIS_FILE});
		$anvil->nice_exit({exit_code => 1});
	}
	
	local $@;
	my $xml        = XML::Simple->new();
	my $server_xml = "";
	my $test       = eval { $server_xml = $xml->XMLin($new_definition, KeyAttr => {}, ForceArray => 1) };
	if (not $test)
	{
		chomp $@;
		my $error =  "[ Error ] - The was a problem parsing: [".$new_definition."]. The error was:\n";
		   $error .= "===========================================================\n";
		   $error .= $@."\n";
		   $error .= "===========================================================\n";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", list => { error => $error }});
		return(1);
	}
	
	my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { xml_file => $xml_file }});
	
	# Write the file
	my $return = $anvil->Storage->write_file({
		body        => $new_definition,
		file        => $xml_file,
		overwrite   => 1,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'return' => $return }});
	$anvil->Storage->get_file_stats({
		file_path => $xml_file,
	});
	
	# Read the file back in and parse it.
	my $on_disk_definition = get_and_parse_disk_definition($anvil, $server_name);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { on_disk_definition => $on_disk_definition }});
	
	return($on_disk_definition);
}