anvil/scancore-agents/scan-server/scan-server

#!/usr/bin/perl
# 
# This scans the nodes and DR host for VMs 
# 
# NOTE: The data stored here is not bound to a given host. As such, only hosted VMs are processed.
# 
# Examples;
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Startup failure (not running as root, no DB, bad file read, etc)
# 2 = libvirtd is not running.
# 
# TODO: 
# - 
# 

use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
use Text::Diff;

# Disable buffering
$| = 1;

# Prevent a discrepency between UID/GID and EUID/EGID from throwing an error.
$< = $>;
$( = $);

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});

# Make sure we're running as 'root'
# $< == real UID, $> == effective UID
if (($< != 0) && ($> != 0))
{
	# Not root
	print $anvil->Words->string({key => "error_0005"})."\n";
	$anvil->nice_exit({exit_code => 1});
}

$anvil->data->{scancore}{'scan-server'}{disable}         = 0;
$anvil->data->{scancore}{'scan-server'}{'auto-undefine'} = 1;
$anvil->data->{switches}{force}                          = 0;

$anvil->Storage->read_config();

# Read switches
$anvil->Get->switches;

# If we're disabled and '--force' wasn't used, exit.
if (($anvil->data->{scancore}{'scan-server'}{disable}) && (not $anvil->data->{switches}{force}))
{
	# Exit.
	$anvil->nice_exit({exit_code => 0});
}

# Handle start-up tasks
my $problem = $anvil->ScanCore->agent_startup({agent => $THIS_FILE});
if ($problem)
{
	$anvil->nice_exit({exit_code => 1});
}

$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_log_0001", variables => { program => $THIS_FILE }});

# There are no tables for this agent, so '--purge' is useless here.

# Before we do anything, are we a node or a DR host?
my $host_type = $anvil->Get->host_type;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }});
if ($host_type eq "striker")
{
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_log_0002", variables => { host_type => $host_type }});
	$anvil->nice_exit({exit_code => 0});
}

# This is more than data collection in most agents, as it actually handles the changes on the fly
collect_data($anvil);

# Mark that we ran.
$anvil->Database->insert_or_update_updated({updated_by => $THIS_FILE});

$anvil->nice_exit({exit_code => 0});

#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This reads in all the data we can find about servers running locally. This is more than data collection in
# most agents, as it actually handles the changes on the fly.
sub collect_data
{
	my ($anvil) = @_;
	
	# Is 'libvirtd' running?
	my $running = $anvil->System->check_daemon({daemon => "libvirtd"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { running => $running }});
	if (not $running)
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "scan_server_log_0003"});
		$anvil->nice_exit({exit_code => 2});
	}
	
	# Load data we know about
	$anvil->Database->get_anvils();
	$anvil->Database->get_servers();
	$anvil->Database->get_server_definitions();

	# Find our Anvil! UUID
	my $target     = $anvil->Get->short_host_name();
	my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { anvil_uuid => $anvil_uuid }});
	
	my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." list --all", source => $THIS_FILE, line => __LINE__});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output, return_code => $return_code }});
	foreach my $line (split/\n/, $output)
	{
		$line = $anvil->Words->clean_spaces({string => $line});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
	
		# This is set to 1 when we add a server, supressing detailed change checks as we'll have 
		# already registered an alert.
		my $added = 0;
		
		### TODO: We may want to handle crashed / idle servers.
=cut
* Server states;
running     - The domain is currently running on a CPU
idle        - The domain is idle, and not running or runnable.  This can be caused because the domain is waiting on IO (a traditional wait state) or has gone to sleep because there was nothing else for it to do.
paused      - The domain has been paused, usually occurring through the administrator running virsh suspend.  When in a paused state the domain will still consume allocated resources like memory, but will not be eligible for scheduling by the hypervisor.
in shutdown - The domain is in the process of shutting down, i.e. the guest operating system has been notified and should be in the process of stopping its operations gracefully.
shut off    - The domain is not running.  Usually this indicates the domain has been shut down completely, or has not been started.
crashed     - The domain has crashed, which is always a violent ending.  Usually this state can only occur if the domain has been configured not to restart on crash.
pmsuspended - The domain has been suspended by guest power management, e.g. entered into s3 state.
- Ones we set:
migrating - Set and cleared by Server->migrate_virsh();
DELETED   - Marks a server as no longer existing
=cut
		
		if (($line =~ /^\d+ (.*?) (.*)$/) or ($line =~ /- (.*?) (.*)$/))
		{
			my $server_name  = $1;
			my $server_state = $2;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				server_name  => $server_name, 
				server_state => $server_state, 
			}});
			
			# Parse out the server UUID.
			my $virsh_definition = get_and_parse_virsh_definition($anvil, $server_name);
			
			# Does the XML definition file exist yet?
			my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { xml_file => $xml_file }});
			if (not -e $xml_file)
			{
				# No, generate it. This will also load and parse the file after it's written.
				my $on_disk_definition = update_on_disk_definition($anvil, $server_name, $virsh_definition);
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { on_disk_definition => $on_disk_definition }});
			}
			else
			{
				# Yes, parse it.
				my $on_disk_definition = get_and_parse_disk_definition($anvil, $server_name);
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { on_disk_definition => $on_disk_definition }});
			}
			
			# We'll compare the memory allocated to the server from the on-disk definition and the memory
			# currently used as reported by the in-memory definition and use that to update the 
			# 'server_ram_in_use' and 'server_configured_ram' columns.
			my $server_uuid           = $anvil->data->{server}{$target}{$server_name}{'from_virsh'}{info}{uuid};
			my $server_ram_in_use     = $anvil->data->{server}{$target}{$server_name}{'from_virsh'}{memory};
			my $server_configured_ram = $anvil->data->{server}{$target}{$server_name}{'from_disk'}{memory};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				server_uuid           => $server_uuid,
				server_ram_in_use     => $anvil->Convert->add_commas({number => $server_ram_in_use})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_in_use}).")",
				server_configured_ram => $anvil->Convert->add_commas({number => $server_configured_ram})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_configured_ram}).")",
			}});
			
			# If we saw the server boot (or saw it for the first time at all), this will store 
			# the boot time.
			my $server_boot_time = 0;
			
			# Is this server in the database already?
			if (exists $anvil->data->{servers}{server_uuid}{$server_uuid})
			{
				# Yes, see if we're looking at a state change
				if ($anvil->data->{servers}{server_uuid}{$server_uuid}{server_state} eq "migrating")
				{
					# Ignore this server, scan_cluster will clear this when the migration is 
					# complete. We don't touch anything during a migration.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_message_0001", variables => { server => $server_name }});
					next;
				}
				elsif ($anvil->data->{servers}{server_uuid}{$server_uuid}{server_state} eq "shut off")
				{
					# It booted. Record the start time.
					my $runtime          = $anvil->Server->get_runtime({server => $server_name});
					my $server_boot_time = time - $runtime;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						runtime          => $runtime,
						server_boot_time => $server_boot_time." (".$anvil->Get->date_and_time({use_time => $server_boot_time}).")",
					}});
				}
			}
			else
			{
				# It's not in the database yet, add it. 
				   $added            = 1;
				my $runtime          = $anvil->Server->get_runtime({server => $server_name});
				my $server_boot_time = time - $runtime;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					runtime          => $runtime,
					server_boot_time => $server_boot_time." (".$anvil->Get->date_and_time({use_time => $server_boot_time}).")",
				}});

				my $got_server_uuid = $anvil->Database->insert_or_update_servers({
					debug                 => 2, 
					server_uuid           => $server_uuid, 
					server_name           => $server_name, 
					server_state          => $server_state, 
					server_anvil_uuid     => $anvil_uuid, 
					server_host_uuid      => $anvil->Get->host_uuid, 
					server_ram_in_use     => $server_ram_in_use, 
					server_configured_ram => $server_configured_ram, 
					server_boot_time      => $server_boot_time,
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { got_server_uuid => $got_server_uuid }});
				
				# Store the definition from virsh
				my $server_definition_uuid = $anvil->Database->insert_or_update_server_definitions({
					debug                         => 2,
					server_definition_xml         => $virsh_definition,  
					server_definition_server_uuid => $server_uuid, 
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { server_definition_uuid => $server_definition_uuid }});
				
				# Reload the servers.
				$anvil->Database->get_servers();
				$anvil->Database->get_server_definitions();
				
				# Register an alert.
				my $variables  = {
					server     => $server_name,
					definition => $virsh_definition,
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0006", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0006", variables => $variables, set_by => $THIS_FILE});
			}
			
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_uuid}           = $server_uuid;
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time}      = $server_boot_time ? $server_boot_time : $anvil->data->{servers}{server_uuid}{$server_uuid}{server_boot_time};
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_state}          = $server_state;
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_anvil_uuid}     = $anvil_uuid;
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_host_uuid}      = $anvil->Get->host_uuid;
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_ram_in_use}     = $server_ram_in_use;
			$anvil->data->{'scan-server'}{server_name}{$server_name}{server_configured_ram} = $server_configured_ram;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				"scan-server::server_name::${server_name}::server_uuid"           => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_uuid},
				"scan-server::server_name::${server_name}::server_boot_time"      => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time}." (".$anvil->Get->date_and_time({use_time => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time}}).")", 
				"scan-server::server_name::${server_name}::server_state"          => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_state}, 
				"scan-server::server_name::${server_name}::server_anvil_uuid"     => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_anvil_uuid}, 
				"scan-server::server_name::${server_name}::server_host_uuid"      => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_host_uuid}, 
				"scan-server::server_name::${server_name}::server_ram_in_use"     => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_ram_in_use}, 
				"scan-server::server_name::${server_name}::server_configured_ram" => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_configured_ram}, 
			}});
			
			if ($server_state eq "shut off")
			{
				my $host_type = $anvil->Get->host_type;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
				if ($host_type eq "node")
				{
					# The definition was likely changed by the user while it was running. We 
					# should have already picked up the changes, but just to be safe, check for
					# differences.
					my ($virsh_definition, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." dumpxml --inactive ".$server_name, source => $THIS_FILE, line => __LINE__});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $virsh_definition, return_code => $return_code }});
					
					# The definition may have certainly changed, so update it in case 
					# needed.
					update_definitions_from_virsh($anvil, $server_name, $server_uuid, $virsh_definition);
					
					# Now undefine the server
					(my $output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." undefine ".$server_name, source => $THIS_FILE, line => __LINE__});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
				}
				
				# Set the boot time back to zero.
				$anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time} = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"scan-server::server_name::${server_name}::server_boot_time" => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time}, 
				}});
			}
		}
	}
	
	# Now loop through the found servers and see if any definitions have changed and need to be 
	# propogated out.
	foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{'scan-server'}{server_name}})
	{
		# Set the XML file path and read the UUID
		my $xml_file    = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
		my $server_uuid = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			server_name => $server_name,
			server_uuid => $server_uuid, 
			xml_file    => $xml_file,
		}});
		
		# Get the definitions
		my $virsh_definition    = get_and_parse_virsh_definition($anvil, $server_name);
		my $on_disk_definition  = get_and_parse_disk_definition($anvil, $server_name);
		my $database_definition = get_and_parse_database_definition($anvil, $server_name, $server_uuid);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { on_disk_definition => $on_disk_definition }});

		# If the 'server_updated_by_user' value is newer than the file age, and there is a difference in the definition, update the file.
		my $current_time           = time;
		my $user_update_time       = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_updated_by_user};
		my $user_update_age        = $current_time - $user_update_time;
		my $database_modified_time = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{unix_modified_time};
		my $database_age           = $current_time - $database_modified_time;
		my $file_modified_time     = $anvil->data->{file_stat}{$xml_file}{modified_time};
		my $file_age               = $current_time - $file_modified_time;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:current_time'           => $current_time,
			's2:user_update_time'       => $user_update_time, 
			's3:user_update_age'        => $user_update_age, 
			's4:database_modified_time' => $database_modified_time, 
			's5:database_age'           => $database_age, 
			's6:file_modified_time'     => $file_modified_time, 
			's7:file_age'               => $file_age, 
		}});
		
		# If the user edited the server via Striker or directly via the on-disk definition, update 
		# the other and then load the changes into virsh.
		if ($database_definition ne $on_disk_definition)
		{
			# Either the change came from Striker (check $user_update_age) or the change happened
			# on disk (check $file_modified_time). Which ever is more recent wins.
			if ($user_update_age < $file_modified_time)
			{
				# Get the diffs before we update
				my $disk_difference  = diff \$database_definition, \$on_disk_definition, { STYLE => 'Unified' };
				my $virsh_difference = diff \$database_definition, \$virsh_definition, { STYLE => 'Unified' };
				
				# The user updated the definition from Striker, so update the file and then push the new file into virsh.
				   $on_disk_definition = update_on_disk_definition($anvil, $server_name, $database_definition);
				   $file_modified_time = $anvil->data->{file_stat}{$xml_file}{modified_time};
				   $file_age           = $current_time - $file_modified_time;
				my $variables          = {
					server           => $server_name,
					definition_file  => $xml_file,
					disk_difference  => $disk_difference,
					virsh_difference => $virsh_difference,
					new_definition   => $database_definition,
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0001", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0001", variables => $variables, set_by => $THIS_FILE});
			}
			else
			{
				# Get the diffs before we update
				my $db_difference    = diff \$on_disk_definition, \$database_definition, { STYLE => 'Unified' };
				my $virsh_difference = diff \$database_definition, \$virsh_definition, { STYLE => 'Unified' };
				
				# The disk was updated, so push the change into the database and into virsh.
				   $database_definition = update_database_definition($anvil, $server_name, $server_uuid, $on_disk_definition);
				my $variables           = {
					server           => $server_name,
					db_difference    => $db_difference,
					virsh_difference => $virsh_difference,
					new_definition   => $on_disk_definition,
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0002", variables => $variables});
				$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0002", variables => $variables, set_by => $THIS_FILE});
			}
			
			# In either case, update the virsh definition from the disk now (either to load the 
			# changes that originated from the database or from a direct file edit).
			$virsh_definition = redefine_server_from_disk($anvil, $server_name);
		}
		elsif (($virsh_definition ne $database_definition) && ($virsh_definition ne $on_disk_definition))
		{
			# The last possible source of change is a direct edit of the definition via a virsh 
			# tool, like virt-manager or a command line virsh call. Either way, the virsh 
			# definition won't match either the database or the on-disk. So in this case, we 
			# use the virsh definition to update both.
			my $disk_difference = diff \$virsh_definition, \$on_disk_definition, { STYLE => 'Unified' };
			my $db_difference   = diff \$virsh_definition, \$database_definition, { STYLE => 'Unified' };
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				disk_difference => $disk_difference,
				db_difference   => $db_difference, 
			}});
			
			   $on_disk_definition  = update_on_disk_definition($anvil, $server_name, $virsh_definition);
			   $database_definition = update_database_definition($anvil, $server_name, $server_uuid, $virsh_definition);
			my $variables           = {
				server          => $server_name,
				definition_file => $xml_file,
				disk_difference => $disk_difference,
				db_difference   => $db_difference,
				new_definition  => $virsh_definition,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0003", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0003", variables => $variables, set_by => $THIS_FILE});
		}
		
		my ($virsh_live_definition, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." dumpxml ".$server_name, source => $THIS_FILE, line => __LINE__});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $virsh_definition, return_code => $return_code }});
		$anvil->Server->parse_definition({
			server     => $server_name,
			source     => "from_live_virsh", 
			definition => $virsh_live_definition, 
		});
		
		# Now that definition updates are dealth with, has anything else changed?
		my $server_state              = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_state};
		my $server_boot_time          = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_boot_time};
		my $server_anvil_uuid         = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_anvil_uuid};
		my $server_user_stop          = $server_state eq "shut off" ? $anvil->data->{servers}{server_uuid}{$server_uuid}{server_user_stop} : 0;
		my $server_host_uuid          = $anvil->data->{'scan-server'}{server_name}{$server_name}{server_host_uuid};
		my $server_ram_in_use         = $anvil->data->{server}{$target}{$server_name}{from_live_virsh}{memory};
		my $server_configured_ram     = $anvil->data->{server}{$target}{$server_name}{from_disk}{memory};
		# Old values
		my $old_server_name           = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
		my $old_server_state          = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
		my $old_server_boot_time      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_boot_time};
		my $old_server_anvil_uuid     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid};
		my $old_server_user_stop      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_user_stop};
		my $old_server_host_uuid      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
		my $old_server_ram_in_use     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_ram_in_use};
		my $old_server_configured_ram = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_configured_ram};
		
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:server_uuid'                => $server_uuid,
			's2:server_name'                => $server_name,
			's3:old_server_name'            => $old_server_name,
			's4:server_state'               => $server_state, 
			's5:old_server_state'           => $old_server_state, 
			's6:server_boot_time'           => $server_boot_time." (".$anvil->Get->date_and_time({use_time => $server_boot_time}).")",
			's7:old_server_boot_time'       => $old_server_boot_time." (".$anvil->Get->date_and_time({use_time => $old_server_boot_time}).")", 
			's8:server_anvil_uuid'          => $server_anvil_uuid, 
			's9:old_server_anvil_uuid'      => $old_server_anvil_uuid, 
			's10:server_user_stop'          => $server_user_stop,
			's11:$old_server_user_stop'     => $old_server_user_stop, 
			's12:server_host_uuid'          => $server_host_uuid, 
			's13:old_server_host_uuid'      => $old_server_host_uuid, 
			's14:server_ram_in_use'         => $anvil->Convert->add_commas({number => $server_ram_in_use})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_in_use}).")", 
			's15:old_server_ram_in_use'     => $anvil->Convert->add_commas({number => $old_server_ram_in_use})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $old_server_ram_in_use}).")", 
			's16:server_configured_ram'     => $anvil->Convert->add_commas({number => $server_configured_ram})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_configured_ram}).")",
			's17:old_server_configured_ram' => $anvil->Convert->add_commas({number => $old_server_configured_ram})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $old_server_configured_ram}).")",
		}});
		
		# Find changes.
		my $update = 0;
		if ($server_name ne $old_server_name)
		{
			# Server name has changed.
			my $variables  = { 
				old_name => $old_server_name,
				new_name => $server_name,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0004", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0004", variables => $variables, set_by => $THIS_FILE });
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		if ($server_state ne $old_server_state)
		{
			# The state has changed.
			my $variables  = { 
				server    => $server_name,
				old_state => $old_server_state,
				new_state => $server_state,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0005", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0005", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
			
			# Make sure the boot time has been updated if the state has transitioned from "shut 
			# off".
			if ($old_server_state eq "shut off")
			{
				my $runtime          = $anvil->Server->get_runtime({server => $server_name});
				my $server_boot_time = time - $runtime;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					server_boot_time => $server_boot_time." (".$anvil->Get->date_and_time({use_time => $server_boot_time}).")",
				}});
			}
		}
		if ($server_boot_time ne $old_server_boot_time)
		{
			# Boot time changed (expected on shut down or boot up)
			my $variables  = { 
				server              => $server_name,
				old_boot_time_date  => $anvil->Get->date_and_time({use_time => $old_server_boot_time}),
				old_boot_time_epoch => $anvil->Convert->add_commas({number => $old_server_boot_time}),
				new_boot_time_date  => $anvil->Get->date_and_time({use_time => $server_boot_time}),
				new_boot_time_epoch => $anvil->Convert->add_commas({number => $server_boot_time}),
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0008", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0008", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		if ($server_anvil_uuid ne $old_server_anvil_uuid)
		{
			# Moved between Anvil! systems
			my $variables  = { 
				server         => $server_name,
				old_anvil_name => $anvil->Get->anvil_name_from_uuid({anvil_uuid => $old_server_anvil_uuid}),
				old_anvil_uuid => $old_server_anvil_uuid,
				new_anvil_name => $anvil->Get->anvil_name_from_uuid({anvil_uuid => $server_anvil_uuid}),
				new_anvil_uuid => $server_anvil_uuid,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0009", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0009", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		if ($server_user_stop ne $old_server_user_stop)
		{
			# Server booted and cleared this or the user shut down the server and this got set)
			my $say_user_stop = $server_user_stop eq "TRUE" ? "scan_server_alert_0010" : "scan_server_alert_0011";
			my $variables     = { server => $server_name };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => $say_user_stop, variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => $say_user_stop, variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		if ($server_host_uuid ne $old_server_host_uuid)
		{
			# Server migated (to the peer or to a new Anvil!)
			my $variables  = { 
				server        => $server_name,
				old_host_name => $old_server_host_uuid eq "NULL" ? "NULL" : $anvil->Get->host_name_from_uuid({host_uuid => $old_server_host_uuid}),
				old_host_uuid => $old_server_host_uuid,
				new_host_name => $server_host_uuid     eq "NULL" ? "NULL" : $anvil->Get->host_name_from_uuid({host_uuid => $server_host_uuid}),
				new_host_uuid => $server_host_uuid,
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0012", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0012", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		if ($server_ram_in_use ne $old_server_ram_in_use)
		{
			# The amount of RAM the server uses has changed, likely after rebooting after a 
			# config change.
			my $variables  = { 
				server               => $server_name,
				old_ram_in_use_short => $anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_in_use}),
				old_ram_in_use_bytes => $anvil->Convert->add_commas({number => $server_ram_in_use}),
				new_ram_in_use_short => $anvil->Convert->bytes_to_human_readable({'bytes' => $old_server_ram_in_use}),
				new_ram_in_use_bytes => $anvil->Convert->add_commas({number => $old_server_ram_in_use}),
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0013", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0013", variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		if ($server_configured_ram ne $old_server_configured_ram)
		{
			# The amount of allocated RAM has changed, but if this value doesn't match
			# 'server_ram_in_use', the user needs to reboot. 
			my $say_ram   = $server_configured_ram eq $server_ram_in_use ? "scan_server_alert_0014" : "scan_server_alert_0015";
			my $variables = { 
				server                   => $server_name,
				old_configured_ram_short => $anvil->Convert->bytes_to_human_readable({'bytes' => $server_configured_ram}),
				old_configured_ram_bytes => $anvil->Convert->add_commas({number => $server_configured_ram}),
				new_configured_ram_short => $anvil->Convert->bytes_to_human_readable({'bytes' => $old_server_configured_ram}),
				new_configured_ram_bytes => $anvil->Convert->add_commas({number => $old_server_configured_ram}),
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => $say_ram, variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => $say_ram, variables => $variables, set_by => $THIS_FILE});
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update => $update }});
		}
		
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }});
		if ($update)
		{
			# Update.
			$anvil->Database->insert_or_update_servers({
				debug                           => 2,
				server_uuid                     => $server_uuid,
				server_name                     => $server_name,
				server_state                    => $server_state, 
				server_boot_time                => $server_boot_time,
				server_anvil_uuid               => $server_anvil_uuid,
				server_user_stop                => $server_user_stop,
				server_host_uuid                => $server_host_uuid,
				server_ram_in_use               => $server_ram_in_use,
				server_configured_ram           => $server_configured_ram,
				server_start_after_server_uuid  => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_after_server_uuid},
				server_start_delay              => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_delay},
				server_live_migration           => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_live_migration},
				server_pre_migration_file_uuid  => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_pre_migration_file_uuid},
				server_pre_migration_arguments  => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_pre_migration_arguments},
				server_post_migration_file_uuid => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_post_migration_file_uuid},
				server_post_migration_arguments => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_post_migration_arguments},
			});
		}
		
		# Delete the database record so we know this one has been dealt with.
		delete $anvil->data->{servers}{server_uuid}{$server_uuid};
	}
	
	# Lastly, if we can access our peer, see what servers are running there. This will let us know if a 
	# server that had been running on us before is not off (versus migrated).
	$anvil->Cluster->get_peers();
	my $peer_is   = $anvil->data->{sys}{anvil}{peer_is};
	my $peer_name = $anvil->data->{sys}{anvil}{$peer_is}{host_name};
	my $peer_uuid = $anvil->data->{sys}{anvil}{$peer_is}{host_uuid};
	my $password  = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:peer_is'   => $peer_is,
		's2:peer_name' => $peer_name, 
		's3:peer_uuid' => $peer_uuid, 
		's4:password'  => $anvil->Log->is_secure($password), 
	}});
	
	# If we can access our peer or DR host, see what servers are running on them. We do no parsing or 
	# processing of servers on other machines, we only care what servers are running.
	my $peer_access = $anvil->Remote->test_access({
		target   => $peer_name,
		password => $password,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { peer_access => $peer_access }});
	if ($peer_access)
	{
		# Get a list of servers running on our peer. 
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target      => $peer_name,
			password    => $password,
			shell_call  => $anvil->data->{path}{exe}{virsh}." list --all",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		foreach my $line (split/\n/, $output)
		{
			$line = $anvil->Words->clean_spaces({string => $line});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
			
			if ($line =~ /^\d+ (.*?) (.*)$/)
			{
				my $server_name  = $1;
				my $server_state = $2;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					server_name  => $server_name, 
					server_state => $server_state, 
				}});
				
				$anvil->data->{'scan-server'}{server_name}{$server_name}{server_state} = $server_state;
				$anvil->data->{'scan-server'}{server_name}{$server_name}{host}         = $peer_name;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"scan-server::server_name::${server_name}::server_state" => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_state}, 
					"scan-server::server_name::${server_name}::host"         => $anvil->data->{'scan-server'}{server_name}{$server_name}{host}, 
				}});
			}
		}
	}
	
	# Now, loop through all remaining servers from when we loaded them out of the database. Already 
	# processed servers will be removed, so what's left are either on the peer, or shut off now.
	foreach my $server_uuid (sort {$a cmp $b} keys %{$anvil->data->{servers}{server_uuid}})
	{
		my $server_name      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
		my $host_anvil_uuid  = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid};
		my $host_anvil_name  = $anvil->Get->anvil_name_from_uuid({anvil_uuid => $host_anvil_uuid});
		my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
		my $server_host_name = $server_host_uuid eq "NULL" ? "NULL" : $anvil->Get->host_name_from_uuid({host_uuid => $server_host_uuid});
		my $server_state     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:server_name'      => $server_name,
			's2:host_anvil_uuid'  => $host_anvil_uuid, 
			's3:host_anvil_name'  => $host_anvil_name, 
			's4:server_host_uuid' => $server_host_uuid, 
			's5:server_host_name' => $server_host_name, 
			's6:server_state'     => $server_state, 
		}});
		
		# Skip if this isn't our Anvil!
		next if $host_anvil_uuid ne $anvil_uuid;
		
		# If the server is on the peer, skip it.
		next if $server_host_uuid ne $anvil->Get->host_uuid;
		
		# If we're here, the server used to be on us, and isn't on the peer, so mark it as off.
		if ($server_state ne "shut off")
		{
			# Mark it as being off now.
			my $query = "
UPDATE 
    servers 
SET 
    server_state     = 'shut off', 
    server_boot_time = '0' 
WHERE 
    server_uuid = ".$anvil->Database->quote($server_uuid)."
;";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
			$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
			
			# Now register an alert.
			my $variables  = { server => $server_name };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0007", variables => $variables});
			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0007", variables => $variables, set_by => $THIS_FILE});
		}
	}
	
	return(0);
}

# This reads the definition file from the database and parses it.
sub get_and_parse_database_definition
{
	my ($anvil, $server_name, $server_uuid) = @_;
	
	my $database_definition = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_definition => $database_definition }});
	
	$anvil->Server->parse_definition({
		server     => $server_name,
		source     => "from_db", 
		definition => $database_definition, 
	});
	
	return($database_definition);
}

# This reads the definition file for a given server and parses it, returning the definition XML.
sub get_and_parse_disk_definition
{
	my ($anvil, $server_name) = @_;
	
	my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { xml_file => $xml_file }});

	my $on_disk_definition = $anvil->Storage->read_file({file => $xml_file});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { on_disk_definition => $on_disk_definition }});
	
	$anvil->Storage->get_file_stats({file_path => $xml_file});
	
	$anvil->Server->parse_definition({
		server     => $server_name,
		source     => "from_disk", 
		definition => $on_disk_definition, 
	});
	
	return($on_disk_definition);
}

# This dumps the definition for a given server and parses it, returning the definition XML.
sub get_and_parse_virsh_definition
{
	my ($anvil, $server_name) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { server_name => $server_name }});

	my ($virsh_definition, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." dumpxml --inactive ".$server_name, source => $THIS_FILE, line => __LINE__});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $virsh_definition, return_code => $return_code }});
	
	$anvil->Server->parse_definition({
		server     => $server_name,
		source     => "from_virsh", 
		definition => $virsh_definition, 
	});
	
	return($virsh_definition);
}

# This defines the server using the on-disk XML file. Effectively, this updates the 'inactive' XML definition
# in virsh.
sub redefine_server_from_disk
{
	my ($anvil, $server_name) = @_;
	
	my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { xml_file => $xml_file }});
	
	# Push the new definition into virsh (it won't take effect until a reboot likely, but it will update
	# the 'inactive' definition immediately.
	my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." defined ".$xml_file, source => $THIS_FILE, line => __LINE__});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output, return_code => $return_code }});
	
	# Now undefine the server again so it disappears when stopped. 
	($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." undefine ".$server_name, source => $THIS_FILE, line => __LINE__});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output, return_code => $return_code }});
	
	# Re-read and parse the new (inactive) definition
	my $virsh_definition = get_and_parse_virsh_definition($anvil, $server_name);
	
	return($virsh_definition);
}

# This takes the XML definition from virsh and sees if it's different from the version in the database or on
# disk, updating them if so.
sub update_definitions_from_virsh
{
	my ($anvil, $server_name, $server_uuid, $virsh_definition) = @_;
	
	my $problem = $anvil->Server->parse_definition({
		server     => $server_name,
		source     => "from_virsh", 
		definition => $virsh_definition, 
	});
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
	if (not $problem)
	{
		# Parsed successfully. Update the database (it'll check if something actually changed).
		my $server_definition_uuid = $anvil->Database->insert_or_update_server_definitions({
			debug                         => 2,
			server_definition_xml         => $virsh_definition,  
			server_definition_server_uuid => $server_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { server_definition_uuid => $server_definition_uuid }});
		
		# Reload the database definitions
		$anvil->Database->get_server_definitions();
		
		my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { xml_file => $xml_file }});
		
		# Just write out the file. If nothing else, it'll update the mtime.
		my $return = $anvil->Storage->write_file({
			body      => $virsh_definition,
			file      => $xml_file,
			overwrite => 1,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'return' => $return }});
	}
	
	return(0);
}

# This updates the database definition and parses the new definition.
sub update_database_definition
{
	my ($anvil, $server_name, $server_uuid, $new_definition) = @_;
	
	my $server_definition_uuid = $anvil->Database->insert_or_update_server_definitions({
		debug                         => 2,
		server_definition_xml         => $new_definition,  
		server_definition_server_uuid => $server_uuid, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { server_definition_uuid => $server_definition_uuid }});

	my $problem = $anvil->Server->parse_definition({
		server     => $server_name,
		source     => "new_definition", 
		definition => $new_definition, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
	
	return($new_definition);
}

# This writes an XML definition to the on-disk definition file, then loads the file stats and parses the 
# definition.
sub update_on_disk_definition
{
	my ($anvil, $server_name, $new_definition) = @_;
	
	my $xml_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { xml_file => $xml_file }});
	
	# Write the file
	my $return = $anvil->Storage->write_file({
		body        => $new_definition,
		file        => $xml_file,
		overwrite   => 1,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'return' => $return }});
	$anvil->Storage->get_file_stats({
		file_path => $xml_file,
	});
	
	# Read the file back in and parse it.
	my $on_disk_definition = get_and_parse_disk_definition($anvil, $server_name);
	
	return($on_disk_definition);
}