#!/usr/bin/perl
# 
# This scans the nodes and DR host for VMs 
# 
# NOTE: The data stored here is not bound to a given host. As such, only hosted VMs are processed.
# 
# Examples;
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Startup failure (not running as root, no DB, bad file read, etc)
# 2 = libvirtd is not running.
# 
# BUG:
# - Servers that are off, but not marked as such in the DB, needs to be updated. (See: line ~1105)
# 
# TODO: 
# - Move location constraints to the host node if the server is not on the preferred host (this happens after
#   recovering from a node loss).
# - Update the fence delay to favour the active host
# - If a server isn't running, and the database definition has been updated, it isn't updated on disk.
# 

use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
use Text::Diff;
use Sys::Virt;

# Disable buffering
$| = 1;

# Prevent a discrepency between UID/GID and EUID/EGID from throwing an error.
$< = $>;
$( = $);

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

my $anvil = Anvil::Tools->new();

# Make sure we're running as 'root'
# $< == real UID, $> == effective UID
if (($< != 0) && ($> != 0))
{
	# Not root
	print $anvil->Words->string({key => "error_0005"})."\n";
	$anvil->nice_exit({exit_code => 1});
}

$anvil->data->{scancore}{'scan-server'}{disable}         = 0;
$anvil->data->{scancore}{'scan-server'}{'auto-undefine'} = 1;
$anvil->data->{switches}{force}                          = 0;

$anvil->Storage->read_config();

# Read switches
$anvil->Get->switches;

# Handle start-up tasks
my $problem = $anvil->ScanCore->agent_startup({agent => $THIS_FILE});
if ($problem)
{
	$anvil->nice_exit({exit_code => 1});
}

$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_server_log_0001", variables => { program => $THIS_FILE }});

# There are no tables for this agent, so '--purge' is useless here.

# Before we do anything, are we a node or a DR host?
my $host_type = $anvil->Get->host_type;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
if ($host_type eq "striker")
{
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_server_log_0002", variables => { host_type => $host_type }});
	$anvil->nice_exit({exit_code => 0});
}

### NOTE: Don't use 'localhost', Sys::Virt translates it to '::1' which Net::OpenSSH breaks on.
# Connect to the local libvirtd API, and handle problems if not.
connect_to_virsh($anvil, $anvil->Get->short_host_name);

# This is more than data collection in most agents, as it actually handles the changes on the fly
collect_data($anvil);

# Look for migration times written out by ocf:alteeve:server.
record_migration_times($anvil);

# Check that there's a DRBD fence rule for each server.
check_drbd_fence_rules($anvil);

# Shut down.
$anvil->ScanCore->agent_shutdown({agent => $THIS_FILE});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# Connect to the local libvirtd API, and handle problems if not.
sub connect_to_virsh
{
	my ($anvil, $target) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }});
	
	# Always do a test access to make sure we've got fingerprints recorded.
	$anvil->Remote->test_access({
		debug  => 2,
		target => $target,
	});
	
	my $record_locator                           = "scan_server::qemu::".$target."::no_access";
	my $is_local                                 = $anvil->Network->is_local({host => $target });
	   $anvil->data->{qemu}{$target}{connection} = "";
	   $anvil->data->{qemu}{$target}{uri}        = "qemu+ssh://".$target."/system";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		record_locator         => $record_locator,
		is_local               => $is_local, 
		"qemu::${target}::uri" => $anvil->data->{qemu}{$target}{uri}, 
	}});
	eval { $anvil->data->{qemu}{$target}{connection} = Sys::Virt->new(uri => $anvil->data->{qemu}{$target}{uri}); };
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"qemu::${target}::connection" => $anvil->data->{qemu}{$target}{connection}, 
		'$@'                          => $@,
	}});
	my $variables = { 
		target => $target, 
		error  => $@,
	};
	if ($@)
	{
		my $key     = $is_local ? "scan_server_alert_0021" : "scan_server_alert_0023";
		my $changed = $anvil->Alert->check_alert_sent({
			record_locator => $record_locator,
			set_by         => $THIS_FILE,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			key     => $key,
			changed => $changed,
		}});
		if ($changed)
		{
			# Register an alert.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => $key, variables => $variables});
			$anvil->Alert->register({
				alert_level => "notice", 
				message     => $key, 
				variables   => $variables, 
				set_by      => $THIS_FILE, 
			});
		}
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => $key, variables => { $variables }});
		
		if ($is_local)
		{
			$anvil->nice_exit({exit_code => 1});
		}
		else
		{
			return(1);
		}
	}
	else
	{
		# Clear the error, if we previously failed to connect.
		my $changed = $anvil->Alert->check_alert_sent({
			record_locator => $record_locator,
			set_by         => $THIS_FILE,
			clear          => 1,
		});
		if ($changed)
		{
			# Register an alert-cleared event.
			my $key = $is_local ? "scan_server_alert_0022" : "scan_server_alert_0024";
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => $key});
			$anvil->Alert->register({
				alert_level => "notice", 
				clear       => 1,
				message     => $key, 
				set_by      => $THIS_FILE,
				variables   => $variables, 
			});
		}
	}
	
	return(0);
}

# Check that there's a DRBD fence rule for each server.
sub check_drbd_fence_rules
{
	my ($anvil) = @_;
	
	my $host_type = $anvil->Get->host_type();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
	if ($host_type ne "node")
	{
		return(0);
	}
	
	my $problem = $anvil->Cluster->parse_cib();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
	if ($problem)
	{
		return(0);
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"cib::parsed::local::ready" => $anvil->data->{cib}{parsed}{'local'}{ready},
	}});
	if (not $anvil->data->{cib}{parsed}{'local'}{ready})
	{
		return(0);
	}
	
	foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{'scan-server'}{server_name}})
	{
		my $drbd_fence_rule_exists    = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_rule}{'exists'};
		my $drbd_fence_rule_attribute = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_rule}{attribute};
		my $drbd_fence_rule_operation = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_rule}{operation};
		my $drbd_fence_rule_value     = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_rule}{value};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:server_name'               => $server_name,
			's2:drbd_fence_rule_exists'    => $drbd_fence_rule_exists, 
			's3:drbd_fence_rule_attribute' => $drbd_fence_rule_attribute, 
			's4:drbd_fence_rule_operation' => $drbd_fence_rule_operation, 
			's5:drbd_fence_rule_value'     => $drbd_fence_rule_value, 
		}});
		
		### TODO: Verify that the other values are correct.
		# If it's missing, add it 
		if (not $drbd_fence_rule_exists)
		{
			# Create it.
			my $variables = {
				server => $server_name,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0019", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0019", variables => $variables, set_by => $THIS_FILE});
			
			# 
			my $shell_call = $anvil->data->{path}{exe}{pcs}." constraint location ".$server_name." rule score=-INFINITY drbd-fenced_".$server_name." eq 1";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output, 
				return_code => $return_code,
			}});
		}
	}
	
	return(0);
}

# Look for migration times written out by ocf:alteeve:server.
sub record_migration_times
{
	my ($anvil) = @_;
	
	my $directory = "/tmp/anvil";
	if (-d $directory)
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { directory => $directory }});
		local(*DIRECTORY);
		opendir(DIRECTORY, $directory);
		while(my $file = readdir(DIRECTORY))
		{
			next if $file eq ".";
			next if $file eq "..";
			next if $file !~ /^migration-duration\./;
			my $full_path = $directory."/".$file;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				file      => $file,
				full_path => $full_path,
			}});
			
			my $body =  $anvil->Storage->read_file({file => $full_path});
			   $body =~ s/\n//;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { body => $body }});
			if ($body =~ /server_name=(.*?),migration_took=(.*?)$/)
			{
				my $server_name    = $1;
				my $migration_took = $2;
				my $anvil_uuid     = $anvil->Cluster->get_anvil_uuid;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					server_name    => $server_name,
					migration_took => $migration_took, 
					anvil_uuid     => $anvil_uuid, 
				}});
				my $server_uuid = $anvil->Get->server_uuid_from_name({
					server_name => $server_name,
					anvil_uuid  => $anvil_uuid,
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_uuid => $server_uuid }});
				if (($server_uuid) && ($migration_took))
				{
					my ($variable_uuid) = $anvil->Database->insert_or_update_variables({
						file                  => $THIS_FILE, 
						line                  => __LINE__, 
						variable_name         => "server::migration_duration", 
						variable_value        => $migration_took, 
						variable_default      => "", 
						variable_description  => "message_0236", 
						variable_section      => "servers", 
						variable_source_uuid  => $server_uuid, 
						variable_source_table => "servers", 
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }});
				}
			}
			unlink $full_path;
		}
	}
	
	return(0);
}

# This reads in all the data we can find about servers running locally. This is more than data collection in
# most agents, as it actually handles the changes on the fly.
sub collect_data
{
	my ($anvil) = @_;
	
	# NOTE: We don't check if libvirtd is running anymore as it auto-starts / stops on EL8+.
	# Load data we know about
	$anvil->Database->get_anvils();
	$anvil->Database->get_servers();
	$anvil->Database->get_server_definitions();

	# Find our Anvil! UUID
	my $target     = $anvil->Get->short_host_name();
	my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
	
	# Look for any servers in the database but not yet written to disk.
	if (exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid})
	{
		foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
		{
			my $server_uuid  = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
			my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:server_name'  => $server_name,
				's2:server_state' => $server_state, 
				's3:server_uuid'  => $server_uuid, 
			}});
			
			next if $server_state eq "DELETED";
			
			# If the definition file doesn't exist at all, create the file.
			my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { xml_file => $xml_file }});
			if (not -f $xml_file)
			{
				my $server_definition = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition => $server_definition }});
				
				# Make very sure the definition XML is valid
				if (not $server_definition)
				{
					# Asked to write an empty definition file!
					my $variables = {
						server => $server_name, 
					};
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0020", variables => $variables });
					$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0020", variables => $variables, set_by => $THIS_FILE});
					$anvil->nice_exit({exit_code => 1});
				}
				
				local $@;
				my $xml        = XML::Simple->new();
				my $server_xml = "";
				my $test       = eval { $server_xml = $xml->XMLin($server_definition, KeyAttr => {}, ForceArray => 1) };
				if (not $test)
				{
					chomp $@;
					my $error =  "[ Error ] - The was a problem parsing: [".$server_definition."]. The error was:\n";
					   $error .= "===========================================================\n";
					   $error .= $@."\n";
					   $error .= "===========================================================\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", list => { error => $error }});
					return(1);
				}
				
				if ($server_definition)
				{
					# Register an alert.
					my $variables = {
						server => $server_name,
					};
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0016", variables => $variables});
					$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0016", variables => $variables, set_by => $THIS_FILE});
					
					my $return = $anvil->Storage->write_file({
						body      => $server_definition,
						file      => $xml_file,
						overwrite => 1,
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'return' => $return }});
				}
			}
		}
	}
	
	# Get the list of servers
	my $stream  = $anvil->data->{qemu}{$target}{connection}->new_stream();
	my @domains = $anvil->data->{qemu}{$target}{connection}->list_all_domains();
	foreach my $domain (@domains)
	{
		my $server_name = $domain->get_name;
		my $server_id   = $domain->get_id == -1 ? "" : $domain->get_id; 
		my $server_uuid = $domain->get_uuid_string;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			server_name   => $server_name,
			server_id     => $server_id, 
			server_uuid   => $server_uuid, 
		}});
		
		# Record the handle to the domain.
		$anvil->data->{domain}{$server_name}{handle}     = $domain;
		$anvil->data->{domain}{$server_name}{persistent} = $anvil->data->{domain}{$server_name}{handle}->is_persistent();
		$anvil->data->{domain}{$server_name}{updated}    = $anvil->data->{domain}{$server_name}{handle}->is_updated();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"domain::${server_name}::handle"     => $anvil->data->{domain}{$server_name}{handle}, 
			"domain::${server_name}::persistent" => $anvil->data->{domain}{$server_name}{persistent}, 
			"domain::${server_name}::updated"    => $anvil->data->{domain}{$server_name}{updated}, 
		}});
		
		# Get the server state.
		my $server_state = get_server_state($anvil, $server_name);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_state => $server_state }});
		
		$anvil->data->{'scan-server'}{server_name}{$server_name}{server_state} = $server_state;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"scan-server::server_name::${server_name}::server_state" => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_state}, 
		}});
		
		# This is set to 1 when we add a server, supressing detailed change checks as we'll have 
		# already registered an alert.
		my $added = 0;
		
		# Parse out the server UUID.
		my $virsh_definition = get_and_parse_virsh_definition($anvil, $server_name);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { virsh_definition => $virsh_definition }});
		
		# Does the XML definition file exist yet?
		my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { xml_file => $xml_file }});
		if (not -e $xml_file)
		{
			# No, generate it. This will also load and parse the file after it's written.
			my $on_disk_definition = update_on_disk_definition($anvil, $server_name, $virsh_definition);
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { on_disk_definition => $on_disk_definition }});
		}
		else
		{
			# Yes, parse it.
			my $on_disk_definition = get_and_parse_disk_definition($anvil, $server_name);
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { on_disk_definition => $on_disk_definition }});
		}
		
		# We'll compare the memory allocated to the server from the on-disk definition and the memory
		# currently used as reported by the in-memory definition and use that to update the 
		# 'server_ram_in_use' and 'server_configured_ram' columns.
		my $server_ram_in_use     = $anvil->data->{server}{$target}{$server_name}{'from_virsh'}{memory};
		my $server_configured_ram = $anvil->data->{server}{$target}{$server_name}{'from_disk'}{memory};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			server_ram_in_use     => $anvil->Convert->add_commas({number => $server_ram_in_use})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_in_use}).")",
			server_configured_ram => $anvil->Convert->add_commas({number => $server_configured_ram})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_configured_ram}).")",
		}});
		
		# If we saw the server boot (or saw it for the first time at all), this will store 
		# the boot time.
		my $server_boot_time = 0;
		
		# Is this server in the database already?
		if (exists $anvil->data->{servers}{server_uuid}{$server_uuid})
		{
			# Yes, see if we're looking at a state change
			if ($anvil->data->{servers}{server_uuid}{$server_uuid}{server_state} eq "migrating")
			{
				# Ignore this server, scan_cluster will clear this when the migration is 
				# complete. We don't touch anything during a migration.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_message_0001", variables => { server => $server_name }});
				next;
			}
			elsif ((not $anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time}) or ($anvil->data->{servers}{server_uuid}{$server_uuid}{server_state} eq "shut off"))
			{
				# It booted. Record the start time.
				my $runtime          = $anvil->Server->get_runtime({server => $server_name});
				   $server_boot_time = $runtime ? time - $runtime : 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					runtime          => $runtime,
					server_boot_time => $server_boot_time." (".$anvil->Get->date_and_time({use_time => $server_boot_time}).")",
				}});
			}
		}
		else
		{
			# It's not in the database yet, add it. 
			   $added            = 1;
			my $runtime          = $anvil->Server->get_runtime({server => $server_name});
			   $server_boot_time = $runtime ? time - $runtime : 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				runtime          => $runtime,
				server_boot_time => $server_boot_time." (".$anvil->Get->date_and_time({use_time => $server_boot_time}).")",
			}});

			my $got_server_uuid = $anvil->Database->insert_or_update_servers({
				debug                 => 2, 
				server_uuid           => $server_uuid, 
				server_name           => $server_name, 
				server_state          => $server_state, 
				server_anvil_uuid     => $anvil_uuid, 
				server_host_uuid      => $anvil->Get->host_uuid, 
				server_ram_in_use     => $server_ram_in_use, 
				server_configured_ram => $server_configured_ram, 
				server_boot_time      => $server_boot_time,
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { got_server_uuid => $got_server_uuid }});
			
			# Store the definition from virsh
			my $server_definition_uuid = $anvil->Database->insert_or_update_server_definitions({
				debug                         => 2,
				server_definition_xml         => $virsh_definition,  
				server_definition_server_uuid => $server_uuid, 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition_uuid => $server_definition_uuid }});
			
			# Make sure the firewall is updated.
			$anvil->Network->manage_firewall();
			
			# Reload the servers.
			$anvil->Database->get_servers();
			$anvil->Database->get_server_definitions();
			
			# Register an alert.
			my $variables  = {
				server     => $server_name,
				definition => $virsh_definition,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0006", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0006", variables => $variables, set_by => $THIS_FILE});
		}
		
		# For 'server_boot_time', we only update if we actually read the uptime.
		if (not $server_boot_time)
		{
			$server_boot_time = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_boot_time};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				server_boot_time => $server_boot_time." (".$anvil->Get->date_and_time({use_time => $server_boot_time}).")", 
			}});
		}
		$anvil->data->{'scan-server'}{server_name}{$server_name}{server_uuid}           = $server_uuid;
		$anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time}      = $server_boot_time;
		$anvil->data->{'scan-server'}{server_name}{$server_name}{server_state}          = $server_state;
		$anvil->data->{'scan-server'}{server_name}{$server_name}{server_anvil_uuid}     = $anvil_uuid;
		$anvil->data->{'scan-server'}{server_name}{$server_name}{server_host_uuid}      = $anvil->Get->host_uuid;
		$anvil->data->{'scan-server'}{server_name}{$server_name}{server_ram_in_use}     = $server_ram_in_use;
		$anvil->data->{'scan-server'}{server_name}{$server_name}{server_configured_ram} = $server_configured_ram;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"scan-server::server_name::${server_name}::server_uuid"           => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_uuid},
			"scan-server::server_name::${server_name}::server_boot_time"      => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time}." (".$anvil->Get->date_and_time({use_time => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time}}).")", 
			"scan-server::server_name::${server_name}::server_state"          => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_state}, 
			"scan-server::server_name::${server_name}::server_anvil_uuid"     => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_anvil_uuid}, 
			"scan-server::server_name::${server_name}::server_host_uuid"      => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_host_uuid}, 
			"scan-server::server_name::${server_name}::server_ram_in_use"     => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_ram_in_use}, 
			"scan-server::server_name::${server_name}::server_configured_ram" => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_configured_ram}, 
		}});
		
		if ($server_state eq "shut off")
		{
			my $host_type = $anvil->Get->host_type;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
			if ($host_type eq "node")
			{
				# The definition may have changed, so update it in case needed.
				update_definitions_from_virsh($anvil, $server_name, $server_uuid, $virsh_definition);
				
				# Now undefine the server
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_message_0002", variables => { server => $server_name }});
				$anvil->data->{domain}{$server_name}{handle}->undefine();
			}
			
			# Set the boot time back to zero.
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time} = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"scan-server::server_name::${server_name}::server_boot_time" => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time}, 
			}});
		}
	}
	
	# Now loop through the found servers and see if any definitions have changed and need to be 
	# propogated out.
	foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{'scan-server'}{server_name}})
	{
		# Set the XML file path and read the UUID
		my $server_uuid   = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_uuid};
		my $is_persistent = $anvil->data->{domain}{$server_name}{persistent} // 0;
		my $is_updated    = $anvil->data->{domain}{$server_name}{updated}    // 0;
		my $xml_file      = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			server_name   => $server_name,
			server_uuid   => $server_uuid, 
			is_persistent => $is_persistent,  
			is_updated    => $is_updated, 
			xml_file      => $xml_file,
		}});
		
		# Record that we've already checked this definition file so we don't waste time doing it 
		# again below.
		$anvil->data->{checked_definition_files}{$xml_file} = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"checked_definition_files::${xml_file}" => $anvil->data->{checked_definition_files}{$xml_file},
		}});
		
		# Get the definitions
		my $virsh_definition    = get_and_parse_virsh_definition($anvil, $server_name);
		my $database_definition = get_and_parse_database_definition($anvil, $server_name, $server_uuid);
		my $on_disk_definition  = get_and_parse_disk_definition($anvil, $server_name);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			virsh_definition    => $virsh_definition,
			database_definition => $database_definition, 
			on_disk_definition  => $on_disk_definition,
		}});

		# If the 'server_updated_by_user' value is newer than the file age, and there is a difference in the definition, update the file.
		my $current_time           = time;
		my $user_update_time       = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_updated_by_user};
		my $user_update_age        = $current_time - $user_update_time;
		my $database_modified_time = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{unix_modified_time};
		my $database_age           = $current_time - $database_modified_time;
		my $file_modified_time     = $anvil->data->{file_stat}{$xml_file}{modified_time};
		my $file_age               = $current_time - $file_modified_time;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:current_time'           => $current_time,
			's2:user_update_time'       => $user_update_time, 
			's3:user_update_age'        => $user_update_age, 
			's4:database_modified_time' => $database_modified_time, 
			's5:database_age'           => $database_age, 
			's6:file_modified_time'     => $file_modified_time, 
			's7:file_age'               => $file_age, 
		}});
		
		### NOTE: A simplified version of this is below when checking files on disk for servers that 
		###       aren't running. If working on this section, check below as well.
		# If the user edited the server via Striker or directly via the on-disk definition, update 
		# the other and then load the changes into virsh.
		if ($is_persistent)
		{
			# The definition was likely updated by the user using virt-manager or virsh.
			
		}
		
		if ($database_definition ne $on_disk_definition)
		{
			# Either the change came from Striker (check $user_update_age) or the change happened
			# on disk (check $file_modified_time). Which ever is more recent wins.
			if ($user_update_age < $file_modified_time)
			{
				# Get the diffs before we update
				my $disk_difference  = diff \$database_definition, \$on_disk_definition, { STYLE => 'Unified' };
				my $virsh_difference = diff \$database_definition, \$virsh_definition, { STYLE => 'Unified' };
				
				# The user updated the definition from Striker, so update the file and then push the new file into virsh.
				   $on_disk_definition = update_on_disk_definition($anvil, $server_name, $database_definition);
				   $file_modified_time = $anvil->data->{file_stat}{$xml_file}{modified_time};
				   $file_age           = $current_time - $file_modified_time;
				my $variables          = {
					server           => $server_name,
					definition_file  => $xml_file,
					disk_difference  => $disk_difference,
					virsh_difference => $virsh_difference,
					new_definition   => $database_definition,
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0001", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0001", variables => $variables, set_by => $THIS_FILE});
			}
			else
			{
				# Get the diffs before we update
				my $db_difference    = diff \$on_disk_definition, \$database_definition, { STYLE => 'Unified' };
				my $virsh_difference = diff \$database_definition, \$virsh_definition, { STYLE => 'Unified' };
				
				# The disk was updated, so push the change into the database and into virsh.
				   $database_definition = update_database_definition($anvil, $server_name, $server_uuid, $on_disk_definition);
				my $variables           = {
					server           => $server_name,
					db_difference    => $db_difference,
					virsh_difference => $virsh_difference,
					new_definition   => $on_disk_definition,
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0002", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0002", variables => $variables, set_by => $THIS_FILE});
			}
			
			# In either case, update the virsh definition from the disk now (either to load the 
			# changes that originated from the database or from a direct file edit).
			$virsh_definition = redefine_server_from_disk($anvil, $server_name);
		}
		elsif ($is_persistent)
		{
			# When the user modifies the definition using a virsh tool, like virt-manager, 
			# 'is_persistent' will be set. In such a case, copy the virsh definition to the DB
			# and on disk, and then undefine the server so that we know the changes have been
			# propogated.
			my $disk_difference = diff \$virsh_definition, \$on_disk_definition, { STYLE => 'Unified' };
			my $db_difference   = diff \$virsh_definition, \$database_definition, { STYLE => 'Unified' };
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				disk_difference => $disk_difference,
				db_difference   => $db_difference, 
			}});
			
			if (($disk_difference) or ($db_difference))
			{
				   $on_disk_definition  = update_on_disk_definition($anvil, $server_name, $virsh_definition);
				   $database_definition = update_database_definition($anvil, $server_name, $server_uuid, $virsh_definition);
				my $variables           = {
					server          => $server_name,
					definition_file => $xml_file,
					disk_difference => $disk_difference,
					db_difference   => $db_difference,
					new_definition  => $virsh_definition,
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0003", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0003", variables => $variables, set_by => $THIS_FILE});
			}
			
			# Undefine the server now.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_message_0002", variables => { server => $server_name }});
			$anvil->data->{domain}{$server_name}{handle}->undefine();
		}
		
		# Now that definition updates are dealth with, has anything else changed?
		my $server_state              = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_state};
		my $server_boot_time          = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time};
		my $server_anvil_uuid         = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_anvil_uuid};
		my $server_user_stop          = $server_state eq "shut off" ? $anvil->data->{servers}{server_uuid}{$server_uuid}{server_user_stop} : 0;
		my $server_host_uuid          = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_host_uuid};
		my $server_ram_in_use         = $anvil->data->{server}{$target}{$server_name}{from_virsh}{memory};
		my $server_configured_ram     = $anvil->data->{server}{$target}{$server_name}{from_disk}{memory};
		# Old values
		my $old_server_name           = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
		my $old_server_state          = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
		my $old_server_boot_time      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_boot_time};
		my $old_server_anvil_uuid     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid};
		my $old_server_user_stop      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_user_stop};
		my $old_server_host_uuid      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
		my $old_server_ram_in_use     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_ram_in_use};
		my $old_server_configured_ram = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_configured_ram};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:server_uuid'                => $server_uuid,
			's2:server_name'                => $server_name,
			's3:old_server_name'            => $old_server_name,
			's4:server_state'               => $server_state, 
			's5:old_server_state'           => $old_server_state, 
			's6:server_boot_time'           => $server_boot_time." (".$anvil->Get->date_and_time({use_time => $server_boot_time}).")",
			's7:old_server_boot_time'       => $old_server_boot_time." (".$anvil->Get->date_and_time({use_time => $old_server_boot_time}).")", 
			's8:server_anvil_uuid'          => $server_anvil_uuid, 
			's9:old_server_anvil_uuid'      => $old_server_anvil_uuid, 
			's10:server_user_stop'          => $server_user_stop,
			's11:$old_server_user_stop'     => $old_server_user_stop, 
			's12:server_host_uuid'          => $server_host_uuid, 
			's13:old_server_host_uuid'      => $old_server_host_uuid, 
			's14:server_ram_in_use'         => $anvil->Convert->add_commas({number => $server_ram_in_use})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_in_use}).")", 
			's15:old_server_ram_in_use'     => $anvil->Convert->add_commas({number => $old_server_ram_in_use})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $old_server_ram_in_use}).")", 
			's16:server_configured_ram'     => $anvil->Convert->add_commas({number => $server_configured_ram})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_configured_ram}).")",
			's17:old_server_configured_ram' => $anvil->Convert->add_commas({number => $old_server_configured_ram})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $old_server_configured_ram}).")",
		}});
		
		# Find changes.
		my $update = 0;
		if ($server_name ne $old_server_name)
		{
			# Server name has changed.
			my $variables  = { 
				old_name => $old_server_name,
				new_name => $server_name,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0004", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0004", variables => $variables, set_by => $THIS_FILE });
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }});
		}
		if ($server_state ne $old_server_state)
		{
			# The state has changed.
			my $variables  = { 
				server    => $server_name,
				old_state => $old_server_state,
				new_state => $server_state,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0005", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0005", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }});
			
			# Make sure the boot time has been updated if the state has transitioned from "shut 
			# off".
			if ($old_server_state eq "shut off")
			{
				my $runtime          = $anvil->Server->get_runtime({server => $server_name});
				my $server_boot_time = time - $runtime;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					server_boot_time => $server_boot_time." (".$anvil->Get->date_and_time({use_time => $server_boot_time}).")",
				}});
			}
			
			# If the server was migrated or booted, update the firewall. With luck, this happened
			# already, so this is a back-stop mainly.
			if (($old_server_state eq "migrating") or ($old_server_state eq "shut off"))
			{
				# Make sure the firewall is updated.
				$anvil->Network->manage_firewall();
			}
		}
		if ($server_boot_time ne $old_server_boot_time)
		{
			# What's the difference? If it's only a few seconds, don't do anything.
			$server_boot_time     = 1 if not $server_boot_time;
			$old_server_boot_time = 1 if not $old_server_boot_time;
			my $diffrerence =  $server_boot_time - $old_server_boot_time;
			   $diffrerence =~ s/^-//;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { diffrerence => $diffrerence }});
			
			if ($diffrerence > 5)
			{
				# Boot time changed (expected on shut down or boot up)
				my $variables  = { 
					server              => $server_name,
					old_boot_time_date  => $anvil->Get->date_and_time({use_time => $old_server_boot_time}),
					old_boot_time_epoch => $anvil->Convert->add_commas({number => $old_server_boot_time}),
					new_boot_time_date  => $anvil->Get->date_and_time({use_time => $server_boot_time}),
					new_boot_time_epoch => $anvil->Convert->add_commas({number => $server_boot_time}),
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0008", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0008", variables => $variables, set_by => $THIS_FILE});
				$update = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }});
			}
		}
		if ($server_anvil_uuid ne $old_server_anvil_uuid)
		{
			# Moved between Anvil! systems
			my $variables  = { 
				server         => $server_name,
				old_anvil_name => $anvil->Get->anvil_name_from_uuid({anvil_uuid => $old_server_anvil_uuid}),
				old_anvil_uuid => $old_server_anvil_uuid,
				new_anvil_name => $anvil->Get->anvil_name_from_uuid({anvil_uuid => $server_anvil_uuid}),
				new_anvil_uuid => $server_anvil_uuid,
			};
			
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0009", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0009", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }});
		}
		if ($server_user_stop ne $old_server_user_stop)
		{
			# Server booted and cleared this or the user shut down the server and this got set)
			my $say_user_stop = $server_user_stop eq "TRUE" ? "scan_server_alert_0010" : "scan_server_alert_0011";
			my $variables     = { server => $server_name };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => $say_user_stop, variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => $say_user_stop, variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }});
		}
		if ($server_host_uuid ne $old_server_host_uuid)
		{
			# Server migrated (to the peer or to a new Anvil!)
			my $variables  = { 
				server        => $server_name,
				old_host_name => $old_server_host_uuid eq "NULL" ? "NULL" : $anvil->Get->host_name_from_uuid({host_uuid => $old_server_host_uuid}),
				old_host_uuid => $old_server_host_uuid,
				new_host_name => $server_host_uuid     eq "NULL" ? "NULL" : $anvil->Get->host_name_from_uuid({host_uuid => $server_host_uuid}),
				new_host_uuid => $server_host_uuid,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0012", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0012", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }});
		}
		if ($server_ram_in_use ne $old_server_ram_in_use)
		{
			# The amount of RAM the server uses has changed, likely after rebooting after a 
			# config change.
			my $variables  = { 
				server               => $server_name,
				old_ram_in_use_short => $anvil->Convert->bytes_to_human_readable({'bytes' => $old_server_ram_in_use}),
				old_ram_in_use_bytes => $anvil->Convert->add_commas({number => $old_server_ram_in_use}),
				new_ram_in_use_short => $anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_in_use}),
				new_ram_in_use_bytes => $anvil->Convert->add_commas({number => $server_ram_in_use}),
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0013", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0013", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }});
		}
		if ($server_configured_ram ne $old_server_configured_ram)
		{
			# The amount of allocated RAM has changed, but if this value doesn't match
			# 'server_ram_in_use', the user needs to reboot. 
			my $say_ram   = $server_configured_ram eq $server_ram_in_use ? "scan_server_alert_0014" : "scan_server_alert_0015";
			my $variables = { 
				server                   => $server_name,
				old_configured_ram_short => $anvil->Convert->bytes_to_human_readable({'bytes' => $old_server_configured_ram}),
				old_configured_ram_bytes => $anvil->Convert->add_commas({number => $old_server_configured_ram}),
				new_configured_ram_short => $anvil->Convert->bytes_to_human_readable({'bytes' => $server_configured_ram}),
				new_configured_ram_bytes => $anvil->Convert->add_commas({number => $server_configured_ram}),
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => $say_ram, variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => $say_ram, variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }});
		}
		
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }});
		if ($update)
		{
			# Update.
			$anvil->Database->insert_or_update_servers({
				debug                           => 2,
				server_uuid                     => $server_uuid,
				server_name                     => $server_name,
				server_state                    => $server_state, 
				server_boot_time                => $server_boot_time,
				server_anvil_uuid               => $server_anvil_uuid,
				server_user_stop                => $server_user_stop,
				server_host_uuid                => $server_host_uuid,
				server_ram_in_use               => $server_ram_in_use,
				server_configured_ram           => $server_configured_ram,
				server_start_after_server_uuid  => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_after_server_uuid},
				server_start_delay              => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_delay},
				server_live_migration           => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_live_migration},
				server_pre_migration_file_uuid  => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_pre_migration_file_uuid},
				server_pre_migration_arguments  => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_pre_migration_arguments},
				server_post_migration_file_uuid => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_post_migration_file_uuid},
				server_post_migration_arguments => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_post_migration_arguments},
			});
		}
		
		# Delete the database record so we know this one has been dealt with.
		delete $anvil->data->{servers}{server_uuid}{$server_uuid};
	}
	
	# If servers are off, and the definition file is edited, we need to be able to pick that up and 
	# propogate it out.
	my $directory = $anvil->data->{path}{directories}{shared}{definitions};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { directory => $directory }});

	local(*DIRECTORY);
	opendir(DIRECTORY, $directory);
	while(my $file = readdir(DIRECTORY))
	{
		next if $file eq ".";
		next if $file eq "..";
		my $xml_file = $directory."/".$file;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			file     => $file,
			xml_file => $xml_file,
		}});
		my $server_name = ($file =~ /^(.*)\.xml$/)[0];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_name => $server_name }});
		next if not $server_name; 
		
		my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_uuid => $server_uuid }});
		next if not $server_uuid;
		
		# If the server is running, we would have processed it above.
		if (exists $anvil->data->{checked_definition_files}{$xml_file})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"checked_definition_files::${xml_file}" => $anvil->data->{checked_definition_files}{$xml_file},
			}});
			next if $anvil->data->{checked_definition_files}{$xml_file};
		}
		
		# Still here? check the age of the file on disk and the age of the database entry.
		my $database_definition = get_and_parse_database_definition($anvil, $server_name, $server_uuid);
		my $on_disk_definition  = get_and_parse_disk_definition($anvil, $server_name);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			database_definition => $database_definition, 
			on_disk_definition  => $on_disk_definition,
		}});

		# If the 'server_updated_by_user' value is newer than the file age, and there is a difference in the definition, update the file.
		my $current_time           = time;
		my $database_modified_time = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{unix_modified_time};
		my $database_age           = $current_time - $database_modified_time;
		my $file_modified_time     = $anvil->data->{file_stat}{$xml_file}{modified_time};
		my $file_age               = $current_time - $file_modified_time;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:current_time'           => $current_time,
			's2:database_modified_time' => $database_modified_time, 
			's3:database_age'           => $database_age, 
			's4:file_modified_time'     => $file_modified_time, 
			's5:file_age'               => $file_age, 
		}});
		if ($database_definition ne $on_disk_definition)
		{
			# Is the database version or on-file newer?
			if ($file_modified_time > $database_modified_time)
			{
				# File is newer, Get the diffs before we update
				my $db_difference    = diff \$on_disk_definition, \$database_definition, { STYLE => 'Unified' };
				
				# The disk was updated, so push the change into the database and into virsh.
				   $database_definition = update_database_definition($anvil, $server_name, $server_uuid, $on_disk_definition);
				my $variables           = {
					server         => $server_name,
					db_difference  => $db_difference,
					new_definition => $on_disk_definition,
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0017", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0017", variables => $variables, set_by => $THIS_FILE});
			}
			else
			{
				# DB is newer
				my $disk_difference  = diff \$database_definition, \$on_disk_definition, { STYLE => 'Unified' };
				
				# The user updated the definition from Striker, so update the file and then push the new file into virsh.
				   $on_disk_definition = update_on_disk_definition($anvil, $server_name, $database_definition);
				   $file_modified_time = $anvil->data->{file_stat}{$xml_file}{modified_time};
				   $file_age           = $current_time - $file_modified_time;
				my $variables          = {
					server           => $server_name,
					definition_file  => $xml_file,
					disk_difference  => $disk_difference,
					new_definition   => $database_definition,
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0018", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0018", variables => $variables, set_by => $THIS_FILE});
			}
		}
	}
	
	# Lastly, if we can access our peer, see what servers are running there. This will let us know if a 
	# server that had been running on us before is not off (versus migrated).
	$anvil->Cluster->get_peers();
	my $peer_is   = $anvil->data->{sys}{anvil}{peer_is};
	my $peer_uuid = $anvil->data->{sys}{anvil}{$peer_is}{host_uuid};
	my $peer_name = $anvil->data->{hosts}{host_uuid}{$peer_uuid}{short_host_name};
	my $password  = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:peer_is'   => $peer_is,
		's2:peer_name' => $peer_name, 
		's3:peer_uuid' => $peer_uuid, 
		's4:password'  => $anvil->Log->is_secure($password), 
	}});
	
	# If we can access our peer or DR host, see what servers are running on them. We do no parsing or 
	# processing of servers on other machines, we only care what servers are running.
	my $peer_access = $anvil->Remote->test_access({
		target   => $peer_name,
		password => $password,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { peer_access => $peer_access }});
	if ($peer_access)
	{
		# Get a list of servers running on our peer. 
		my ($problem) = connect_to_virsh($anvil, $peer_name);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		if (not $problem)
		{
			my $stream  = $anvil->data->{qemu}{$peer_name}{connection}->new_stream();
			my @domains = $anvil->data->{qemu}{$peer_name}{connection}->list_all_domains();
			foreach my $domain (@domains)
			{
				my $server_name   = $domain->get_name;
				my $server_id     = $domain->get_id == -1 ? "" : $domain->get_id; 
				my $server_uuid   = $domain->get_uuid_string;
				my $is_updated    = $domain->is_updated();
				my $is_persistent = $domain->is_persistent();
				my $os_type       = $domain->get_os_type();
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					server_name   => $server_name,
					server_id     => $server_id, 
					server_uuid   => $server_uuid, 
					is_updated    => $is_updated, 
					is_persistent => $is_persistent, 
					os_type       => $os_type, 
				}});
				
				### OK
				# Record the handle to the domain.
				$anvil->data->{domain}{$server_name}{handle} = $domain;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"domain::${server_name}::handle" => $anvil->data->{domain}{$server_name}{handle}, 
				}});
				
				# Get the server state.
				my $server_state = get_server_state($anvil, $server_name);
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_state => $server_state }});
				
				$anvil->data->{'scan-server'}{server_name}{$server_name}{server_state} = $server_state;
				$anvil->data->{'scan-server'}{server_name}{$server_name}{host}         = $peer_name;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"scan-server::server_name::${server_name}::server_state" => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_state}, 
					"scan-server::server_name::${server_name}::host"         => $anvil->data->{'scan-server'}{server_name}{$server_name}{host}, 
				}});
			}
		}
	}
	
	# Now, loop through all remaining servers from when we loaded them out of the database. Already 
	# processed servers will be removed, so what's left are either on the peer, or shut off now.
	foreach my $server_uuid (sort {$a cmp $b} keys %{$anvil->data->{servers}{server_uuid}})
	{
		my $server_name      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
		my $host_anvil_uuid  = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid};
		my $host_anvil_name  = $anvil->Get->anvil_name_from_uuid({anvil_uuid => $host_anvil_uuid});
		my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
		my $server_host_name = $server_host_uuid eq "NULL" ? "NULL" : $anvil->Get->host_name_from_uuid({host_uuid => $server_host_uuid});
		my $server_state     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:server_name'      => $server_name,
			's2:host_anvil_uuid'  => $host_anvil_uuid, 
			's3:host_anvil_name'  => $host_anvil_name, 
			's4:server_host_uuid' => $server_host_uuid, 
			's5:server_host_name' => $server_host_name, 
			's6:server_state'     => $server_state, 
		}});
		
		# Skip if this isn't our Anvil!
		next if $host_anvil_uuid ne $anvil_uuid;
		
		# If the server is on the peer, skip it.
		next if $server_host_uuid ne $anvil->Get->host_uuid;
		
		# If we're here, the server used to be on us, and isn't on the peer, so mark it as off.
		if (($server_state ne "shut off") && ($server_state ne "DELETED"))
		{
			# Mark it as being off now.
			my $query = "
UPDATE 
    servers 
SET 
    server_state     = 'shut off', 
    server_boot_time = '0' 
WHERE 
    server_uuid = ".$anvil->Database->quote($server_uuid)."
;";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
			$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
			
			# Now register an alert.
			my $variables  = { server => $server_name };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0007", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0007", variables => $variables, set_by => $THIS_FILE});
		}
	}
	
	return(0);
}

sub get_server_state
{
	my ($anvil, $server_name) = @_;
	
	### States:
	# 0 = no state
	# 1 = running        - The domain is currently running on a CPU
	# 2 = blocked (idle) - the domain is blocked on resource. This can be caused because the domain is waiting on IO (a traditional wait state) or has gone to sleep because there was nothing else for it to do.
	# 3 = paused         - The domain has been paused, usually occurring through the administrator running virsh suspend.  When in a paused state the domain will still consume allocated resources like memory, but will not be eligible for scheduling by the hypervisor.
	# 4 = in shutdown    - The domain is in the process of shutting down, i.e. the guest operating system has been notified and should be in the process of stopping its operations gracefully.
	# 5 = shut off       - The domain is not running.  Usually this indicates the domain has been shut down completely, or has not been started.
	# 6 = crashed        - The domain has crashed, which is always a violent ending.  Usually this state can only occur if the domain has been configured not to restart on crash.
	# 7 = pmsuspended    - The domain has been suspended by guest power management, e.g. entered into s3 state.
	my ($state, $reason) = $anvil->data->{domain}{$server_name}{handle}->get_state();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		'state' => $state, 
		reason  => $reason,
	}});
	
	### Reasons are dependent on the state. 
	### See: https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainShutdownReason
	my $server_state = "unknown";
	if ($state == 1)    { $server_state = "running"; }	# Server is running.
	elsif ($state == 2) { $server_state = "blocked"; }	# Server is blocked (IO contention?).
	elsif ($state == 3) { $server_state = "paused"; }	# Server is paused (migration target?).
	elsif ($state == 4) { $server_state = "in shutdown"; }	# Server is shutting down.
	elsif ($state == 5) { $server_state = "shut off"; }	# Server is shut off.
	elsif ($state == 6) { $server_state = "crashed"; }	# Server is crashed!
	elsif ($state == 7) { $server_state = "pmsuspended"; }	# Server is suspended.
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_state => $server_state }});
	
	return($server_state);
}

# This reads the definition file from the database and parses it.
sub get_and_parse_database_definition
{
	my ($anvil, $server_name, $server_uuid) = @_;
	
	my $database_definition = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { database_definition => $database_definition }});
	
	my $problem = $anvil->Server->parse_definition({
		debug      => 3,
		server     => $server_name,
		source     => "from_db", 
		definition => $database_definition, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	if ($problem)
	{
		# The definition is not valid.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_server_log_0003", variables => { definition => $database_definition }});
		$database_definition = "";
	}
	
	return($database_definition);
}

# This reads the definition file for a given server and parses it, returning the definition XML.
sub get_and_parse_disk_definition
{
	my ($anvil, $server_name) = @_;
	
	my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { xml_file => $xml_file }});

	my $on_disk_definition = $anvil->Storage->read_file({file => $xml_file});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { on_disk_definition => $on_disk_definition }});
	
	$anvil->Storage->get_file_stats({file_path => $xml_file});
	
	my $problem = $anvil->Server->parse_definition({
		debug      => 3,
		server     => $server_name,
		source     => "from_disk", 
		definition => $on_disk_definition, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	if ($problem)
	{
		# The definition is not valid.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_server_log_0003", variables => { definition => $on_disk_definition }});
		$on_disk_definition = "";
	}
	
	return($on_disk_definition);
}

# This dumps the definition for a given server and parses it, returning the definition XML.
sub get_and_parse_virsh_definition
{
	my ($anvil, $server_name) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { server_name => $server_name }});
	
	# Get the inactive XML (changes requested by the user may not match the in-memory XML)
	my $virsh_definition_active = $anvil->data->{domain}{$server_name}{handle}->get_xml_description();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { virsh_definition_active => $virsh_definition_active }});
	
	my $virsh_definition_inactive = $anvil->data->{domain}{$server_name}{handle}->get_xml_description(Sys::Virt::Domain::XML_INACTIVE);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { virsh_definition_inactive => $virsh_definition_inactive }});
	
	# There will always be a difference because things like ID, vnc port, vnet device, etc will always 
	# be different. So later, we might want to parse both and look for hardware changes. Until then, this
	# isn't useful yet.
	my $virsh_difference = diff \$virsh_definition_active, \$virsh_definition_inactive, { STYLE => 'Unified' };
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { virsh_difference => $virsh_difference }});
	
	my $problem = $anvil->Server->parse_definition({
		debug      => 3,
		server     => $server_name,
		source     => "from_virsh", 
		definition => $virsh_definition_inactive, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
	if ($problem)
	{
		# The definition is not valid.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_log_0003", variables => { definition => $virsh_definition_inactive }});
		$virsh_definition_inactive = "";
	}
	
	return($virsh_definition_inactive);
}

# This defines the server using the on-disk XML file. Effectively, this updates the 'inactive' XML definition
# in virsh.
sub redefine_server_from_disk
{
	my ($anvil, $server_name) = @_;
	
	my $xml_file =  $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
	my $body     =  $anvil->Storage->read_file({file => $xml_file});
	   $body     =~ s/\n//;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		xml_file => $xml_file,
		body     => $body,
	}});
	
	# Push the new definition into virsh (it won't take effect until a reboot likely, but it will update
	# the 'inactive' definition immediately.
	my $target = $anvil->Get->short_host_name;
	$anvil->data->{qemu}{$target}{connection}->define_domain($body);
	
	# Now undefine the server again so it disappears when stopped. 
	$anvil->data->{domain}{$server_name}{handle}->undefine;
	
	# Re-read and parse the new (inactive) definition
	my $virsh_definition = get_and_parse_virsh_definition($anvil, $server_name);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { virsh_definition => $virsh_definition }});
	
	return($virsh_definition);
}

# This takes the XML definition from virsh and sees if it's different from the version in the database or on
# disk, updating them if so.
sub update_definitions_from_virsh
{
	my ($anvil, $server_name, $server_uuid, $virsh_definition) = @_;
	
	my $problem = $anvil->Server->parse_definition({
		debug      => 3,
		server     => $server_name,
		source     => "from_virsh", 
		definition => $virsh_definition, 
	});
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	if (not $problem)
	{
		# Parsed successfully. Update the database (it'll check if something actually changed).
		my $server_definition_uuid = $anvil->Database->insert_or_update_server_definitions({
			debug                         => 2,
			server_definition_xml         => $virsh_definition,  
			server_definition_server_uuid => $server_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition_uuid => $server_definition_uuid }});
		
		# Reload the database definitions
		$anvil->Database->get_server_definitions();
		
		my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { xml_file => $xml_file }});
		
		# Just write out the file. If nothing else, it'll update the mtime.
		my $return = $anvil->Storage->write_file({
			debug     => 2,
			body      => $virsh_definition,
			file      => $xml_file,
			overwrite => 1,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'return' => $return }});
	}
	
	return(0);
}

# This updates the database definition and parses the new definition.
sub update_database_definition
{
	my ($anvil, $server_name, $server_uuid, $new_definition) = @_;
	
	my $server_definition_uuid = $anvil->Database->insert_or_update_server_definitions({
		debug                         => 2,
		server_definition_xml         => $new_definition,  
		server_definition_server_uuid => $server_uuid, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition_uuid => $server_definition_uuid }});

	my $problem = $anvil->Server->parse_definition({
		debug      => 3,
		server     => $server_name,
		source     => "new_definition", 
		definition => $new_definition, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	
	return($new_definition);
}

# This writes an XML definition to the on-disk definition file, then loads the file stats and parses the 
# definition.
sub update_on_disk_definition
{
	my ($anvil, $server_name, $new_definition) = @_;
	
	# Make very sure the definition XML is valid
	if (not $new_definition)
	{
		# Asked to write an empty definition file!
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0020", variables => $variables });
		$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0020", variables => $variables, set_by => $THIS_FILE});
		$anvil->nice_exit({exit_code => 1});
	}
	
	local $@;
	my $xml        = XML::Simple->new();
	my $server_xml = "";
	my $test       = eval { $server_xml = $xml->XMLin($new_definition, KeyAttr => {}, ForceArray => 1) };
	if (not $test)
	{
		chomp $@;
		my $error =  "[ Error ] - The was a problem parsing: [".$new_definition."]. The error was:\n";
		   $error .= "===========================================================\n";
		   $error .= $@."\n";
		   $error .= "===========================================================\n";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", list => { error => $error }});
		return(1);
	}
	
	my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { xml_file => $xml_file }});
	
	# Write the file
	my $return = $anvil->Storage->write_file({
		body        => $new_definition,
		file        => $xml_file,
		overwrite   => 1,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'return' => $return }});
	$anvil->Storage->get_file_stats({
		file_path => $xml_file,
	});
	
	# Read the file back in and parse it.
	my $on_disk_definition = get_and_parse_disk_definition($anvil, $server_name);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { on_disk_definition => $on_disk_definition }});
	
	return($on_disk_definition);
}