anvil/tools/anvil-manage-server-storage

#!/usr/bin/perl
# 
# This program will manage servers; Changing RAM, CPU cores, Growing virtual disks, adding virtual disks, 
# inserting and ejecting ISO images into virtual optical media.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = No database connection.
# 
# TODO: 
# 
# USAGE:
# - Show
#   - anvil-manage-server-storage --server srv01-fs37
# - ISO
#   - anvil-manage-server-storage --server srv01-fs37 --optical sda --insert /mnt/shared/files/CentOS-5.11-x86_64-bin-DVD-1of2.iso
#   - anvil-manage-server-storage --server srv01-fs37 --optical sda --eject
# - Disk
#   - anvil-manage-server-storage --server srv01-fs37 --disk vdb --grow {+10GiB,150GiB}
# 

use strict;
use warnings;
use Anvil::Tools;
require POSIX;
use Term::Cap;
use Data::Dumper;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

### TODO: Remove this before final release
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
##########################################

# Read switches (target ([user@]host[:port]) and the file with the target's password.
$anvil->Get->switches({list => [
	"add",
	"anvil", 
	"disk", 
	"eject", 
	"grow",
	"insert", 
	"optical", 
	"server",
	], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

# Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks
# is to setup the database server.
$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0305"});
	sleep 10;
	$anvil->nice_exit({exit_code => 1});
}

# If we don't have a job UUID, try to find one.
if (not $anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->data->{switches}{'job-uuid'} = $anvil->Job->get_job_uuid({program => $THIS_FILE});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }});
}

$anvil->Database->get_hosts();
$anvil->Database->get_anvils();
$anvil->Database->get_servers();

if ($anvil->data->{switches}{anvil})
{
	# Make sure they asked for a real anvil.
	$anvil->Get->anvil_from_switch({string => $anvil->data->{switches}{anvil}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"switches::anvil_name" => $anvil->data->{switches}{anvil_name},
		"switches::anvil_uuid" => $anvil->data->{switches}{anvil_uuid},
	}});
}

if (not $anvil->data->{switches}{server})
{
	# Show the list of servers.
	show_server_list($anvil);
	print "\nPlease specify which server you want to modify using '--server <name or uuid>'.\n\n";
	$anvil->nice_exit({exit_code => 0});
}

validate_server($anvil);

if ($anvil->data->{switches}{optical})
{
	manage_optical($anvil);
}
elsif ($anvil->data->{switches}{disk})
{
	manage_disk($anvil);
}
else
{
	show_server_details($anvil);
}

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

sub load_storage
{
	my ($anvil) = @_;
	
	# Host and server
	my $short_host_name = $anvil->Get->short_host_name;
	my $server_name     = $anvil->data->{switches}{server_name};
	
	# We need to know which volume groups the disks are on so that we can report how much free space is 
	# available for either growing or adding a disk.
	$anvil->Get->available_resources({
		debug      => 2,
		anvil_uuid => $anvil->data->{switches}{anvil_uuid},
	});
	
	# Load LVM and Storage Group data
	$anvil->Database->get_lvm_data({debug => 2});
	
	# Load DRBD resource data so that we can map DRBD resources / volumes to volume groups.
	my $drbd_resource = "";
	foreach my $device_path (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{device}})
	{
		my $on_lv         = $anvil->data->{server}{$short_host_name}{$server_name}{device}{$device_path}{on_lv};
		   $drbd_resource = $anvil->data->{server}{$short_host_name}{$server_name}{device}{$device_path}{resource};
		my $device_target = $anvil->data->{server}{$short_host_name}{$server_name}{device}{$device_path}{target};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:device_path'   => $device_path, 
			's2:on_lv'         => $on_lv,
			's3:drbd_resource' => $drbd_resource, 
			's4:device_target' => $device_target, 
		}});
	}
	# There should only ever be one, but it's not impossible for there to be multiple.
	foreach my $drbd_resource (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{drbd}{resource}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_resource => $drbd_resource }});
		
		# Get the DRBD volume data
		load_drbd_data($anvil, $drbd_resource);
	}
	
	return($drbd_resource);
}

sub manage_disk
{
	my ($anvil) = @_;
	
	my $short_host_name = $anvil->Get->short_host_name;
	my $server_name     = $anvil->data->{switches}{server_name};
	my $from_source     = get_definition_source($anvil);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		short_host_name => $short_host_name,
		server_name     => $server_name, 
		from_source     => $from_source, 
	}});
	if (not $anvil->data->{switches}{disk})
	{
		# If there's only one optical drive, select it automatically
		my $count = keys %{$anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{disk}{target}};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
		if ($count == 1)
		{
			foreach my $device_target (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{disk}{target}})
			{
				$anvil->data->{switches}{disk} = $device_target;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"switches::drive" => $anvil->data->{switches}{disk},
				}});
				last;
			}
		}
		
		if (not $anvil->data->{switches}{disk})
		{
			# Can't proceed.
			print "\n[ Error ] - Please indicate the disk drive to work with using '--disk --drive <target_device>'.\n\n";
			show_server_details($anvil);
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	my $drbd_resource = load_storage($anvil);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_resource => $drbd_resource }});
	foreach my $host_type ("node", "dr")
	{
		foreach my $short_host_name (sort {$a cmp $b} keys %{$anvil->data->{drbd_resource}{$drbd_resource}{host_type}{$host_type}{short_host_name}})
		{
			my $host_uuid = $anvil->data->{drbd_resource}{$drbd_resource}{host_type}{$host_type}{short_host_name}{$short_host_name}{host_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:short_host_name' => $short_host_name,
				's2:host_uuid'       => $host_uuid, 
			}});
			foreach my $volume_number (sort {$a cmp $b} keys %{$anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}})
			{
				my $device_path  = $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{device_path};
				my $device_minor = $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{device_minor};
				my $volume_size  = $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{volume_size};
				my $backing_disk = $anvil->data->{new}{resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{backing_disk};
				my $meta_disk    = $anvil->data->{new}{resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{meta_disk};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:volume_number' => $volume_number,
					's2:device_path'   => $device_path, 
					's3:device_minor'  => $device_minor, 
					's4:volume_size'   => $volume_size, 
					's5:backing_disk'  => $backing_disk, 
					's6:meta_disk'     => $meta_disk, 
				}});
				
				# Which volume group is the backing device in?
				foreach my $this_scan_lvm_lv_name (sort {$a cmp $b} keys %{$anvil->data->{lvm}{host_name}{$short_host_name}{lv}})
				{
					my $this_scan_lvm_lv_path  = $anvil->data->{lvm}{host_name}{$short_host_name}{lv}{$this_scan_lvm_lv_name}{scan_lvm_lv_path}; 
					my $this_scan_lvm_lv_on_vg = $anvil->data->{lvm}{host_name}{$short_host_name}{lv}{$this_scan_lvm_lv_name}{scan_lvm_lv_on_vg}; 
					my $this_scan_lvm_lv_uuid  = $anvil->data->{lvm}{host_name}{$short_host_name}{lv}{$this_scan_lvm_lv_name}{scan_lvm_lv_uuid};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:this_scan_lvm_lv_name'  => $this_scan_lvm_lv_name, 
						's2:this_scan_lvm_lv_path'  => $this_scan_lvm_lv_path,
						's3:this_scan_lvm_lv_on_vg' => $this_scan_lvm_lv_on_vg, 
						's4:this_scan_lvm_lv_uuid'  => $this_scan_lvm_lv_uuid, 
					}});
					
					if ($backing_disk eq $this_scan_lvm_lv_path)
					{
						# What's the VG's UUID? 
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							's1:short_host_name'        => $short_host_name, 
							's2:this_scan_lvm_lv_on_vg' => $this_scan_lvm_lv_on_vg, 
						}});
						if (exists $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$this_scan_lvm_lv_on_vg})
						{
							my $scan_lvm_vg_uuid   = $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$this_scan_lvm_lv_name}{scan_lvm_vg_uuid};
							my $storage_group_uuid = $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$this_scan_lvm_lv_name}{storage_group_uuid};
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:scan_lvm_vg_uuid'   => $scan_lvm_vg_uuid, 
								's2:storage_group_uuid' => $storage_group_uuid, 
							}});
						}
					}
				}
			}
		}
	}
	
	my $device_target = $anvil->data->{switches}{disk};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { device_target => $device_target }});

	my $device           = "disk";
	my $alias            = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{alias};
	my $boot_order       = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{boot_order};
	my $say_boot         = $boot_order eq "99" ? "--" : sprintf("%02d", $boot_order);
	my $type             = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{type};
	my $address_type     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{type};
	my $address_bus      = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{bus};
	my $driver_name      = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{name};
	my $device_bus       = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{device_bus};
	my $driver_type      = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{type};
	my $address_domain   = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{domain};
	my $address_slot     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{slot};
	my $address_function = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{function};
	my $device_path      = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{path};
	my $driver_io        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{io};
	my $driver_cache     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{cache};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's01:device_target'    => $device_target,
		's02:alias'            => $alias, 
		's03:boot_order'       => $boot_order,
		's04:say_boot'         => $say_boot, 
		's05:type'             => $type, 
		's06:address_type'     => $address_type, 
		's07:address_bus'      => $address_bus, 
		's08:driver_name'      => $driver_name, 
		's09:device_bus'       => $device_bus, 
		's10:driver_type'      => $driver_type, 
		's11:address_domain'   => $address_domain,
		's12:address_slot'     => $address_slot, 
		's13:address_function' => $address_function, 
		's14:device_path'      => $device_path, 
		's15:driver_io'        => $driver_io, 
		's16:driver_cache'     => $driver_cache, 
	}});
	print "- Target: [".$device_target."], boot: [".$say_boot."], path: [".$device_path."], cache: [".$driver_cache."], driver type: [".$driver_type."]\n";
	
	my $volume = "";
	
	print "Sub-Nodes:\n";
	foreach my $device_path (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{device}})
	{
		my $on_lv         = $anvil->data->{server}{$short_host_name}{$server_name}{device}{$device_path}{on_lv};
		   $drbd_resource = $anvil->data->{server}{$short_host_name}{$server_name}{device}{$device_path}{resource};
		my $device_target = $anvil->data->{server}{$short_host_name}{$server_name}{device}{$device_path}{target};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:device_path'   => $device_path, 
			's2:on_lv'         => $on_lv,
			's3:drbd_resource' => $drbd_resource, 
			's4:device_target' => $device_target, 
		}});
	}
	foreach my $drbd_resource (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{drbd}{resource}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_resource => $drbd_resource }});
		
		# Get the DRBD volume data
		load_drbd_data($anvil, $drbd_resource);
	}
	foreach my $host_type ("node", "dr")
	{
		foreach my $short_host_name (sort {$a cmp $b} keys %{$anvil->data->{drbd_resource}{$drbd_resource}{host_type}{$host_type}{short_host_name}})
		{
			my $host_uuid = $anvil->data->{drbd_resource}{$drbd_resource}{host_type}{$host_type}{short_host_name}{$short_host_name}{host_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:short_host_name' => $short_host_name,
				's2:host_uuid'       => $host_uuid, 
			}});
			print " |- Name: [".$short_host_name."], UUID: [".$host_uuid."]\n";
			foreach my $volume_number (sort {$a cmp $b} keys %{$anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}})
			{
				my $device_path  = $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{device_path};
				my $device_minor = $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{device_minor};
				my $volume_size  = $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{volume_size};
				my $backing_disk = $anvil->data->{new}{resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{backing_disk};
				my $meta_disk    = $anvil->data->{new}{resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{meta_disk};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:volume_number' => $volume_number,
					's2:device_path'   => $device_path, 
					's3:device_minor'  => $device_minor, 
					's4:volume_size'   => $volume_size, 
					's5:backing_disk'  => $backing_disk, 
					's6:meta_disk'     => $meta_disk, 
				}});
				print "   ^- Volume: [".$volume_number."], backing device: [".$backing_disk."], DRBD minor: [".$device_minor."], size: [".$anvil->Convert->bytes_to_human_readable({'bytes' => $volume_size})."]\n";
			}
		}
	}
	
	# What are we doing?
	if ($anvil->data->{switches}{grow})
	{
		### TODO: Make this work without the peer node being online.
		# The server is allowed to be running, but both nodes and any DR hosts this is replicating to
		# needs to be online.
		
		# Process the DRBD config to get the backing LVs.
	}
	
	return(0);
}

sub manage_optical
{
	my ($anvil) = @_;
	
	my $short_host_name = $anvil->Get->short_host_name;
	my $server_name     = $anvil->data->{switches}{server_name};
	my $from_source     = get_definition_source($anvil);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		short_host_name => $short_host_name,
		server_name     => $server_name, 
		from_source     => $from_source, 
	}});
	
	if (not $anvil->data->{switches}{optical})
	{
		# If there's only one optical drive, select it automatically
		my $count = keys %{$anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
		if ($count == 1)
		{
			foreach my $device_target (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}})
			{
				$anvil->data->{switches}{optical} = $device_target;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"switches::drive" => $anvil->data->{switches}{optical},
				}});
				last;
			}
		}
		
		if (not $anvil->data->{switches}{optical})
		{
			# Can't proceed.
			print "\n[ Error ] - Please indicate the optical drive to work with using '--optical <target_device>'.\n\n";
			show_server_details($anvil);
			$anvil->nice_exit({exit_code => 1});
		}
	}
	my $device_target = $anvil->data->{switches}{optical};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { device_target => $device_target }});
	
	my $eject_first        = 0;
	my $alias              = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{alias};
	my $boot_order         = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{boot_order};
	my $say_boot           = $boot_order eq "99" ? "--" : sprintf("%02d", $boot_order);
	my $type               = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{type};
	my $address_type       = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{address}{type};
	my $address_bus        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{address}{bus};
	my $driver_name        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{driver}{name};
	my $device_bus         = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{device_bus};
	my $driver_type        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{driver}{type};
	my $address_controller = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{address}{controller};
	my $address_unit       = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{address}{unit};
	my $address_target     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{address}{target};
	my $device_path        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{path};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's01:device_target'      => $device_target,
		's02:alias'              => $alias, 
		's03:boot_order'         => $boot_order,
		's04:say_boot'           => $say_boot, 
		's05:type'               => $type, 
		's06:address_type'       => $address_type, 
		's07:address_bus'        => $address_bus, 
		's08:driver_name'        => $driver_name, 
		's09:device_bus'         => $device_bus, 
		's10:driver_type'        => $driver_type, 
		's11:address_controller' => $address_controller,
		's12:address_unit'       => $address_unit, 
		's13:address_target'     => $address_target, 
		's14:device_path'        => $device_path, 
	}});
	
	# Without a --source, the result is an eject.
	my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." change-media ".$server_name." --path ".$device_target." --update --live --config";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	if ($anvil->data->{switches}{insert})
	{
		# Make sure the new target exists.
		my $iso = $anvil->data->{switches}{insert};
		if (not -f $iso)
		{
			print "[ Error ] - The target: [".$iso."] doesn't exist, can't insert it into the optical drive.\n";
			update_definition($anvil, "undefine");
			$anvil->nice_exit({exit_code => 1});
		}
		else
		{
			$shell_call .= " --source ".$anvil->data->{switches}{insert};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		}
	}
	
	# If the server is running, update the on-disk and in-DB definition.
	print "Defining the server: [".$server_name."] to prepare for 'virsh' modification of the server.\n";
	update_definition($anvil, "define");
	
	# Now we can modify the server using virsh.
	if ($anvil->data->{switches}{insert})
	{
		print "- Inserting: [".$anvil->data->{switches}{insert}."] into the drive: [".$device_target."].\n";
	}
	else
	{
		print "- Ejecting: [".$anvil->data->{switches}{insert}."] from: [".$device_target."].\n";
	}
	my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:output'      => $output,
		's2:return_code' => $return_code, 
	}});
	print "'virsh' Output: [".$output."]\n";
	
	print "Updating the stored definition and undefining the server now:\n";
	update_definition($anvil, "undefine");
	print "Done!\n";
	
	return(0);
}

sub get_definition_source
{
	my ($anvil) = @_;
	
	my $short_host_name  = $anvil->Get->short_host_name;
	my $server_name      = $anvil->data->{switches}{server_name};
	my $server_uuid      = $anvil->data->{switches}{server_uuid};
	my $server_state     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
	my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
	my $from_source      = "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		short_host_name => $short_host_name,
		server_name     => $server_name, 
		server_uuid     => $server_uuid, 
		server_state    => $server_state, 
		from_source     => $from_source, 
	}});
	
	$anvil->data->{server}{$short_host_name}{$server_name}{from_virsh} = "" if not exists $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh};
	$anvil->data->{server}{$short_host_name}{$server_name}{from_disk}  = "" if not exists $anvil->data->{server}{$short_host_name}{$server_name}{from_disk};
	$anvil->data->{server}{$short_host_name}{$server_name}{from_db}    = "" if not exists $anvil->data->{server}{$short_host_name}{$server_name}{from_db};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"server::${short_host_name}::${server_name}::from_virsh" => $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh},
		"server::${short_host_name}::${server_name}::from_disk"  => $anvil->data->{server}{$short_host_name}{$server_name}{from_disk},
		"server::${short_host_name}::${server_name}::from_db"    => $anvil->data->{server}{$short_host_name}{$server_name}{from_db},
	}});
	
	if (($server_state eq "running")                    && 
	    ($server_host_uuid eq $anvil->Get->host_uuid()) && 
	    (exists $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}))
	{
		$from_source = "from_virsh";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { from_source => $from_source }});
	}
	elsif (exists $anvil->data->{server}{$short_host_name}{$server_name}{from_disk})
	{
		$from_source = "from_disk";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { from_source => $from_source }});
	}
	else
	{
		$from_source = "from_db";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { from_source => $from_source }});
	}

	return($from_source);
}

sub show_server_details
{
	my ($anvil) = @_;
	
	my $short_host_name = $anvil->Get->short_host_name;
	my $host_uuid       = $anvil->Get->host_uuid;
	my $server_name     = $anvil->data->{switches}{server_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:short_host_name' => $short_host_name,
		's2:host_uuid'       => $host_uuid,
		's2:server_name'     => $server_name, 
	}});
	
	# Load storage data
	my $drbd_resource = load_storage($anvil);
	
	my $from_source = get_definition_source($anvil);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { from_source => $from_source }});
	foreach my $device ("disk", "cdrom")
	{
		if ($device eq "disk")
		{
			print "\nDisk Drives:\n";
		}
		else
		{
			print "\nOptical Drives:\n";
		}
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { device => $device }});
		next if $device ne "cdrom" &&  $device ne "disk";
		foreach my $device_target (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}})
		{
			my $alias        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{alias};
			my $boot_order   = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{boot_order};
			my $say_boot     = $boot_order eq "99" ? "--" : sprintf("%02d", $boot_order);
			my $type         = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{type};
			my $address_type = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{type};
			my $address_bus  = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{bus};
			my $driver_name  = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{name};
			my $device_bus   = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{device_bus};
			my $driver_type  = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{type};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's01:device_target' => $device_target,
				's02:alias'         => $alias, 
				's03:boot_order'    => $boot_order,
				's04:say_boot'      => $say_boot, 
				's05:type'          => $type, 
				's06:address_type'  => $address_type, 
				's07:address_bus'   => $address_bus, 
				's08:driver_name'   => $driver_name, 
				's09:device_bus'    => $device_bus, 
				's10:driver_type'   => $driver_type, 
			}});
			if ($device eq "disk")
			{
				my $address_domain   = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{domain};
				my $address_slot     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{slot};
				my $address_function = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{function};
				my $device_path      = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{path};
				my $driver_io        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{io};
				my $driver_cache     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{cache};
				my $on_lv            = $anvil->data->{server}{$short_host_name}{$server_name}{device}{$device_path}{on_lv};
				my $drbd_resource    = $anvil->data->{server}{$short_host_name}{$server_name}{device}{$device_path}{resource};
				my $drbd_volume      = $anvil->data->{lvm}{host_name}{$short_host_name}{lv_path}{$on_lv}{drbd}{volume};
				my $volume_size      = $anvil->data->{lvm}{host_name}{$short_host_name}{lv_path}{$on_lv}{drbd}{volume};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:address_domain'   => $address_domain,
					's2:address_slot'     => $address_slot, 
					's3:address_function' => $address_function, 
					's4:device_path'      => $device_path, 
					's5:driver_io'        => $driver_io, 
					's6:driver_cache'     => $driver_cache, 
					's7:on_lv'            => $on_lv, 
					's8:drbd_resource'    => $drbd_resource, 
					's9:drbd_volume'      => $drbd_volume, 
				}});
				print "- Target: [".$device_target."], boot: [".$say_boot."], replication resource: [".$drbd_resource."/".$drbd_volume."], size: [".$anvil->Convert->bytes_to_human_readable({'bytes' => $volume_size})."] (".$anvil->Convert->add_commas({number => $volume_size})." Bytes)\n";
				print "  - Replcation path: [".$device_path."], backed by: [".$on_lv."], cache: [".$driver_cache."]\n";
				
				# Get the backing LV from the DRBD resource.
				
				
				#if ($anvil->data->{storage_groups}{vg_uuid}{$vg_uuid}{storage_group_uuid})
			}
			else
			{
				my $address_controller = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{controller};
				my $address_unit       = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{unit};
				my $address_target     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{target};
				my $device_path        = "<empty>";
				if (defined $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{path})
				{
					$device_path = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{path};
				}
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:address_controller' => $address_controller,
					's2:address_unit'       => $address_unit, 
					's3:address_target'     => $address_target, 
					's4:device_path'        => $device_path, 
				}});
				print "- Target: [".$device_target."], boot: [".$say_boot."], ISO: [".$device_path."]\n";
			}
		}
	}
	print "\n";
	
	print "Sub-Nodes:\n";
	show_volume($anvil, $drbd_resource, "node");
	
	my $dr_count = keys %{$anvil->data->{drbd_resource}{$drbd_resource}{host_type}{dr}{short_host_name}};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_count => $dr_count }});
	if ($dr_count)
	{
		print "DR Hosts:\n";
		show_volume($anvil, $drbd_resource, "dr");
	}
	
	return(0);
}

sub load_drbd_data
{
	my ($anvil, $drbd_resource) = @_;
	
	my $query = "
SELECT 
    a.host_uuid, 
    b.scan_drbd_resource_xml, 
    c.scan_drbd_volume_number, 
    c.scan_drbd_volume_device_path, 
    c.scan_drbd_volume_device_minor, 
    c.scan_drbd_volume_size 
FROM 
    hosts a, 
    scan_drbd_resources b, 
    scan_drbd_volumes c 
WHERE 
    a.host_uuid = b.scan_drbd_resource_host_uuid 
AND 
    b.scan_drbd_resource_uuid = c.scan_drbd_volume_scan_drbd_resource_uuid 
AND 
    b.scan_drbd_resource_name = ".$anvil->Database->quote($drbd_resource)."
;";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
	
	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
	my $count   = @{$results};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		results => $results, 
		count   => $count, 
	}});
	foreach my $row (@{$results})
	{
		my $host_uuid               = $row->[0];
		my $short_host_name         = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name};
		my $host_type               = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type};
		my $scan_drbd_volume_number = $row->[2];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:host_uuid'               => $host_uuid, 
			's2:host_type'               => $host_type, 
			's3:short_host_name'         => $short_host_name, 
			's4:scan_drbd_volume_number' => $scan_drbd_volume_number, 
		}});
		
		$anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$scan_drbd_volume_number}{device_path}  = $row->[3];
		$anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$scan_drbd_volume_number}{device_minor} = $row->[4];
		$anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$scan_drbd_volume_number}{volume_size}  = $row->[5];
		$anvil->data->{drbd_resource}{$drbd_resource}{host_type}{$host_type}{short_host_name}{$short_host_name}{host_uuid}          = $host_uuid;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"s1:drbd_resource::${drbd_resource}::host_uuid::${host_uuid}::volume_number::${scan_drbd_volume_number}::device_path"  => $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$scan_drbd_volume_number}{device_path}, 
			"s2:drbd_resource::${drbd_resource}::host_uuid::${host_uuid}::volume_number::${scan_drbd_volume_number}::device_minor" => $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$scan_drbd_volume_number}{device_minor}, 
			"s3:drbd_resource::${drbd_resource}::host_uuid::${host_uuid}::volume_number::${scan_drbd_volume_number}::volume_size"  => $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$scan_drbd_volume_number}{volume_size}, 
			"s4:drbd_resource::${drbd_resource}::host_type::${host_type}::short_host_name::${short_host_name}::host_uuid"          => $anvil->data->{drbd_resource}{$drbd_resource}{host_type}{$host_type}{short_host_name}{$short_host_name}{host_uuid}, 
		}});
		
		if (not exists $anvil->data->{drbd_resource}{$drbd_resource}{xml})
		{
			$anvil->data->{drbd_resource}{$drbd_resource}{xml} = $row->[1];
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"drbd_resource::${drbd_resource}::xml" => $anvil->data->{drbd_resource}{$drbd_resource}{xml}, 
			}});
			$anvil->DRBD->parse_resource({
				debug => 2, 
				xml   => $anvil->data->{drbd_resource}{$drbd_resource}{xml}, 
			});
		}
	}
	
	return(0);
}

sub show_volume
{
	my ($anvil, $drbd_resource, $host_type) = @_;
	
	my $anvil_uuid = defined $anvil->data->{switches}{anvil_uuid} ? $anvil->data->{switches}{anvil_uuid} : "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
	foreach my $short_host_name (sort {$a cmp $b} keys %{$anvil->data->{drbd_resource}{$drbd_resource}{host_type}{$host_type}{short_host_name}})
	{
		my $host_uuid = $anvil->data->{drbd_resource}{$drbd_resource}{host_type}{$host_type}{short_host_name}{$short_host_name}{host_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:short_host_name' => $short_host_name,
			's2:host_uuid'       => $host_uuid, 
		}});
		print " |- Name: [".$short_host_name."], UUID: [".$host_uuid."]\n";
		foreach my $volume_number (sort {$a cmp $b} keys %{$anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}})
		{
			my $device_path        = $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{device_path};
			my $device_minor       = $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{device_minor};
			my $volume_size        = $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{volume_size};
			my $backing_disk       = $anvil->data->{new}{resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{backing_disk};
			my $meta_disk          = $anvil->data->{new}{resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{meta_disk};
			my $storage_group_uuid = $anvil->data->{new}{resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{storage_group_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:volume_number'      => $volume_number,
				's2:device_path'        => $device_path, 
				's3:device_minor'       => $device_minor, 
				's4:volume_size'        => $volume_size, 
				's5:backing_disk'       => $backing_disk, 
				's6:meta_disk'          => $meta_disk, 
				's7:storage_group_uuid' => $storage_group_uuid, 
			}});
			my $storage_group_name       = "<unknown>";
			my $storage_group_size       = 0;
			my $storage_group_free_space = 0;
			if (($storage_group_uuid) && ($anvil_uuid) && (exists $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}))
			{
				$storage_group_name       = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{storage_group_name};
				$storage_group_size       = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size};
				$storage_group_free_space = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
			}
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:storage_group_size'       => $storage_group_size,
				's2:storage_group_free_space' => $storage_group_free_space, 
			}});
			print "   |- Volume: [".$volume_number."], backing device: [".$backing_disk."], DRBD minor: [".$device_minor."], size: [".$anvil->Convert->bytes_to_human_readable({'bytes' => $volume_size})."]\n";
			print "   ^- In storage group: [".$storage_group_name."], size: [".$anvil->Convert->bytes_to_human_readable({'bytes' => $storage_group_size})."], free: [".$anvil->Convert->bytes_to_human_readable({'bytes' => $storage_group_free_space})."]\n";
			
			# How much space is available in the storage group this volume is in?
			foreach my $this_scan_lvm_lv_name (sort {$a cmp $b} keys %{$anvil->data->{lvm}{host_name}{$short_host_name}{lv}})
			{
				my $this_scan_lvm_lv_path  = $anvil->data->{lvm}{host_name}{$short_host_name}{lv}{$this_scan_lvm_lv_name}{scan_lvm_lv_path}; 
				my $this_scan_lvm_lv_on_vg = $anvil->data->{lvm}{host_name}{$short_host_name}{lv}{$this_scan_lvm_lv_name}{scan_lvm_lv_on_vg}; 
				my $this_scan_lvm_lv_uuid  = $anvil->data->{lvm}{host_name}{$short_host_name}{lv}{$this_scan_lvm_lv_name}{scan_lvm_lv_uuid};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:this_scan_lvm_lv_name'  => $this_scan_lvm_lv_name, 
					's2:this_scan_lvm_lv_path'  => $this_scan_lvm_lv_path,
					's3:this_scan_lvm_lv_on_vg' => $this_scan_lvm_lv_on_vg, 
					's4:this_scan_lvm_lv_uuid'  => $this_scan_lvm_lv_uuid, 
				}});
				if ($backing_disk eq $this_scan_lvm_lv_path)
				{
					# What's the VG's UUID? 
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:short_host_name'        => $short_host_name, 
						's2:this_scan_lvm_lv_on_vg' => $this_scan_lvm_lv_on_vg, 
					}});
					if (exists $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$this_scan_lvm_lv_on_vg})
					{
						my $scan_lvm_vg_internal_uuid = $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$this_scan_lvm_lv_on_vg}{scan_lvm_vg_internal_uuid};
						my $storage_group_uuid        = $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$this_scan_lvm_lv_on_vg}{storage_group_uuid};
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							's1:scan_lvm_vg_internal_uuid' => $scan_lvm_vg_internal_uuid, 
							's2:storage_group_uuid'        => $storage_group_uuid, 
						}});
						
						if ($storage_group_uuid)
						{
							my $vg_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size};
							my $free_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
						}
					}
				}
			}
		}
	}
	
	return(0);
}

sub validate_server
{
	my ($anvil) = @_;
	
	$anvil->Get->server_from_switch({
		debug      => 2, 
		string     => $anvil->data->{switches}{server},
		anvil_uuid => $anvil->data->{switches}{anvil_uuid},
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"switches::server_name" => $anvil->data->{switches}{server_name},
		"switches::server_uuid" => $anvil->data->{switches}{server_uuid},
	}});
	
	if (not $anvil->data->{switches}{server_uuid})
	{
		show_server_list($anvil);
		if ($anvil->data->{switches}{anvil_uuid})
		{
			# Not found on the requested Anvil! node.
			print "\nThe server: [".$anvil->data->{switches}{server}."] was not found on the Anvil! node: [".$anvil->data->{switches}{anvil_name}."]. Valid servers are above.\n\n";
		}
		else
		{
			# Not found at all.
			print "\nThe server: [".$anvil->data->{switches}{server}."] was not found. Valid servers are above.\n\n";
		}
		$anvil->nice_exit({exit_code => 1});
	}
	
	print "Working with the server: [".$anvil->data->{switches}{server_name}."], UUID: [".$anvil->data->{switches}{server_uuid}."]\n";
	my $short_host_name   = $anvil->Get->short_host_name;
	my $server_name       = $anvil->data->{switches}{server_name};
	my $server_uuid       = $anvil->data->{switches}{server_uuid};
	my $server_definition = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_definition_xml};
	my $server_host_uuid  = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
	my $server_state      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
	my $anvil_uuid        = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:short_host_name'   => $short_host_name, 
		's2:server_name'       => $server_name, 
		's3:server_uuid'       => $server_uuid, 
		's4:server_definition' => $server_definition,
		's5:server_host_uuid'  => $server_host_uuid, 
		's6:server_state'      => $server_state, 
		's7:anvil_uuid'        => $anvil_uuid, 
	}});
	if (not $anvil->data->{switches}{anvil_uuid})
	{
		$anvil->data->{switches}{anvil_uuid} = $anvil_uuid;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:switches::anvil_uuid' => $anvil->data->{switches}{anvil_uuid}, 
		}});
	}
	
	if ($server_state eq "DELETED")
	{
		print "The server: [".$server_name."] has been deleted.\n";
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Parse the definition.
	$anvil->Server->parse_definition({
		debug      => 3,
		host       => $short_host_name,
		server     => $server_name, 
		source     => "from_db",
		definition => $server_definition, 
	});
	
	# Can we read the XML definition?
	$anvil->Server->get_status({
		debug  => 2,
		server => $server_name,
		host   => $short_host_name,
	});
	
	if (not $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{xml})
	{
		# The server isn't actually running... Not here anyway.
		if ($server_state eq "running")
		{
			my $server_host_name = $anvil->Database->get_host_from_uuid({
				short     => 1,
				host_uuid => $server_host_uuid,
			});
			
			print "The server: [".$server_name."] appears to be running on: [".$server_host_name."], please run this on that host.\n";
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	return(0);
}

# Update the definition on disk and in the DB, and define or undefine if requested.
sub update_definition
{
	my ($anvil, $task) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { task => $task }});
	
	my $short_host_name     = $anvil->Get->short_host_name;
	my $server_name         = $anvil->data->{switches}{server_name};
	my $server_uuid         = $anvil->data->{switches}{server_uuid};
	my $server_definition   = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_definition_xml};
	my $server_host_uuid    = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
	my $server_state        = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
	my $definition_file     = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
	my $server_running_here = 0;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:short_host_name'   => $short_host_name, 
		's2:server_name'       => $server_name, 
		's3:server_uuid'       => $server_uuid, 
		's4:server_definition' => $server_definition,
		's5:server_host_uuid'  => $server_host_uuid, 
		's6:server_state'      => $server_state, 
		's7:definition_file'   => $definition_file, 
	}});
	
	# See if the server is running locally
	my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list --name";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	
	my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:output'      => $output,
		's2:return_code' => $return_code, 
	}});
	foreach my $this_server (split/\n/, $output)
	{
		$this_server = $anvil->Words->clean_spaces({string => $this_server});
		next if not $this_server;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { this_server => $this_server }});
		
		if ($this_server eq $server_name)
		{
			$server_running_here = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_running_here => $server_running_here }});
			last;
		}
	}
	
	my $disk_definition  = $anvil->data->{server}{$short_host_name}{$server_name}{from_disk}{xml}  ? $anvil->data->{server}{$short_host_name}{$server_name}{from_disk}{xml}  : "";
	my $virsh_definition = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{xml} ? $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{xml} : "";
	my $use_definition   = "";
	if (($server_running_here) or (($server_state eq "running") && ($virsh_definition)))
	{
		# Get the live definition
		if ($server_running_here)
		{
			my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." dumpxml --inactive ".$server_name;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			my ($live_virsh_definition, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				live_virsh_definition => $live_virsh_definition, 
				return_code           => $return_code,
			}});
			
			my ($problem) = $anvil->Server->parse_definition({
				server     => $server_name,
				source     => "from_virsh", 
				definition => $live_virsh_definition, 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
			if (not $problem)
			{
				$use_definition = $live_virsh_definition;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { use_definition => $use_definition }});
				
				$anvil->Server->parse_definition({
					debug      => 3,
					host       => $short_host_name,
					server     => $server_name, 
					source     => "from_virsh",
					definition => $live_virsh_definition, 
				});
			}
			else
			{
				$use_definition = $virsh_definition;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { use_definition => $use_definition }});
				
				$anvil->Server->parse_definition({
					debug      => 3,
					host       => $short_host_name,
					server     => $server_name, 
					source     => "from_virsh",
					definition => $virsh_definition, 
				});
			}
		}
		else
		{
			$use_definition = $virsh_definition;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { use_definition => $use_definition }});
			
			$anvil->Server->parse_definition({
				debug      => 3,
				host       => $short_host_name,
				server     => $server_name, 
				source     => "from_virsh",
				definition => $virsh_definition, 
			});
		}
	}
	else
	{
		$use_definition = $disk_definition;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { use_definition => $use_definition }});
		
		$anvil->Server->parse_definition({
			debug      => 3,
			host       => $short_host_name,
			server     => $server_name, 
			source     => "from_disk",
			definition => $virsh_definition, 
		});
	}
	
	if (not $use_definition)
	{
		# What?!
		print "[ Error ] - Failed to find an on disk or from virsh definition for the server: [".$server_name."]. Unable to proceed.\n";
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Update the stored definition
	$anvil->data->{servers}{server_uuid}{$server_uuid}{server_definition_xml} = $use_definition;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"servers::server_uuid::${server_uuid}::server_definition_xml" => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_definition_xml},
	}});
	
	# Update the on-disk definition
	my ($failed) = $anvil->Storage->write_file({
		secure    => 1, 
		file      => $definition_file, 
		body      => $use_definition, 
		overwrite => 1,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
	my $server_definition_uuid = $anvil->Database->insert_or_update_server_definitions({
		debug                         => 2,
		server_definition_xml         => $use_definition,  
		server_definition_server_uuid => $server_uuid, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { server_definition_uuid => $server_definition_uuid }});
	
	if ($task eq "define")
	{
		# Define the server.
		my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." define ".$definition_file,
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output, 
			return_code => $return_code,
		}});
	}
	elsif ($task eq "undefine")
	{
		# Undefine the server.
		my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." undefine ".$server_name,
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($output, $return_code) = $anvil->System->call({
			debug      => 2,
			shell_call => $shell_call, 
			source     => $THIS_FILE, 
			line       => __LINE__,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output, 
			return_code => $return_code,
		}});
	}
	
	return(0);
}

sub show_server_list
{
	my ($anvil) = @_;
	
	# Loop through all Anvil! nodes, then all server in that Anvil!
	foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
	{
		my $anvil_uuid        = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
		my $anvil_description = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			anvil_name        => $anvil_name, 
			anvil_uuid        => $anvil_uuid,
			anvil_description => $anvil_description,
		}});
		if (($anvil->data->{switches}{anvil_uuid}) && ($anvil->data->{switches}{anvil_uuid} ne $anvil_uuid))
		{
			next;
		}
		print "\nAnvil! Node: [".$anvil_name."], UUID: [".$anvil_uuid."] - Description: [".$anvil_description."]\n";
		
		my $server_count = 0;
		if (exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name})
		{
			$server_count = keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_count => $server_count }});
		}
		if (not $server_count)
		{
			print "- No servers on this node yet\n";
		}
		else
		{
			foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
			{
				my $server_uuid  = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
				my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					server_name  => $server_name, 
					server_uuid  => $server_uuid, 
					server_state => $server_state,
				}});
				next if $server_state eq "DELETED";
				print "^- Server: [".$server_name."], UUID: [".$server_uuid."]\n";
			}
		}
	}
	
	return(0);
}