anvil/tools/anvil-manage-dr

#!/usr/bin/perl
# 
# This manages if a server is backed up to a DR host or not. When enabled, it can start or stop replication.
# 
# NOTE: Unlike most jobs, this one will directly work on the peer node and the DR host using SSH connections.
#       This behaviour is likely to change later as it's not ideal.
# 
# TODO: --remove is not yet implemented, this needs to be done. Use anvil-delete-server for methods to delete.
# EXAMPLES:
# - anvil-manage-dr --link --dr-host vm-a01dr02 --anvil vm-anvil-01
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Any problem that causes an early exit.
# 

use strict;
use warnings;
use Anvil::Tools;
require POSIX;
use Term::Cap;
use Text::Diff;
use Data::Dumper;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

$anvil->Get->switches({list => [
	"anvil",
	"connect",
	"disconnect",
	"dr-host",
	"job-uuid",
	"license-file",
	"link",
	"protect",
	"protocol",
	"remove",
	"server",
	"show", 
	"unlink",
	"update",
	"Yes"], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0306"});
	sleep 10;
	$anvil->nice_exit({exit_code => 1});
}

# Are we linking or unlinking?
if (($anvil->data->{switches}{'link'}) or ($anvil->data->{switches}{'unlink'}))
{
	handle_links($anvil);
}

# If we've got a job UUID, load the job details.
if ($anvil->data->{switches}{'job-uuid'})
{
	load_job($anvil);
}
elsif ($anvil->data->{switches}{list})
{
	# If '--list' was used, show all Nodes, DR hosts and servers.
	show_list($anvil);
}

my $terminal = "";
if (not $anvil->data->{switches}{'job-uuid'})
{
	my $termios = new POSIX::Termios;
	$termios->getattr;
	my $ospeed   = $termios->getospeed;
	   $terminal = Tgetent Term::Cap { TERM => undef, OSPEED => $ospeed };
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { terminal => $terminal }});
	
	$terminal->Trequire(qw/ce ku kd/);
	print $terminal->Tputs('cl');
}

sanity_check($anvil, $terminal);


$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

sub show_list
{
	my ($anvil) = @_;
	
	# Show Anvil! nodes
	$anvil->Database->get_hosts();
	$anvil->Database->get_storage_group_data({debug => 2});
	$anvil->Database->get_dr_links({debug => 2});
	$anvil->Database->get_servers();
	
	# Show Anvil! nodes.
	print $anvil->Words->string({key => "header_0081"})."\n";
	my $anvil_count = keys %{$anvil->data->{anvils}{anvil_name}};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_count => $anvil_count }});
	if ($anvil_count)
	{
		foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
		{
			my $anvil_uuid        = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
			my $anvil_description = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description};
			my $dr_link_count     = keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				anvil_name        => $anvil_name,
				anvil_uuid        => $anvil_uuid, 
				anvil_description => $anvil_description, 
				dr_link_count     => $dr_link_count, 
			}});
			print $anvil->Words->string({key => "message_0364", variables => { 
				anvil_name  => $anvil_name,
				description => $anvil_description,
			}})."\n";
			if ($dr_link_count)
			{
				foreach my $dr_link_host_name (sort {$a cmp $b} keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}})
				{
					my $dr_link_uuid = $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}{$dr_link_host_name}{dr_link_uuid};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						dr_link_host_name => $dr_link_host_name,
						dr_link_uuid      => $dr_link_uuid, 
					}});
					print $anvil->Words->string({key => "message_0365", variables => { 
						host_name => $dr_link_host_name,
						link_uuid => $dr_link_uuid,
					}})."\n";
				}
			}
			else
			{
				# No DR hosts yet
				print $anvil->Words->string({key => "message_0366"})."\n";
			}
		}
	}
	else
	{
		# No nodes yet
		print $anvil->Words->string({key => "message_0367"})."\n";
	}
	
	# DR Hosts
	print "\n".$anvil->Words->string({key => "header_0115"})."\n";
	my $dr_host_count = keys %{$anvil->data->{sys}{hosts}{by_type}{dr}{host_name}};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_host_count => $dr_host_count }});
	if ($dr_host_count)
	{
		# TODO: Show free space of VGs, available RAM and CPU core count and type/speed
		foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_type}{dr}{host_name}})
		{
			print $anvil->Words->string({key => "message_0368", variables => { host_name => $host_name }})."\n";
		}
	}
	else
	{
		# No DR hosts yet
		print $anvil->Words->string({key => "message_0366"})."\n";
	}
	
	# DR Hosts
	print "\n".$anvil->Words->string({key => "header_0116"})."\n";
	my $server_count = keys %{$anvil->data->{servers}{server_uuid}};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_count => $server_count }});
	if ($server_count)
	{
		# If there's servers, there has to be Anvil! nodes.
		foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
		{
			# TODO: Show each server's disk(s) size, RAM and CPU core count
			my $anvil_uuid  = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
			foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
			{
				my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
				my $server_name = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
				print $anvil->Words->string({key => "message_0370", variables => { 
					server_name => $server_name,
					anvil_name  => $anvil_name,
				}})."\n";
			}
		}
	}
	else
	{
		# No servers yet exist
		print $anvil->Words->string({key => "message_0371"})."\n";
	}
	$anvil->nice_exit({exit_code => 0});
	
	return(0);
}

sub handle_links
{
	my ($anvil) = @_;

	# Get a list of DR hosts.
	$anvil->Database->get_dr_links();

	# Do we habe an Anvil! and a DR host?
	my $anvil_uuid = $anvil->Database->get_anvil_uuid_from_string({string => $anvil->data->{switches}{anvil}});
	my $host_uuid  = $anvil->Database->get_host_uuid_from_string({string => $anvil->data->{switches}{'dr-host'}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
		anvil_uuid => $anvil_uuid,
		host_uuid  => $host_uuid,
	}});

	my $problem = 0;
	if (not $anvil_uuid)
	{
		$problem = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		if ($anvil->data->{switches}{anvil})
		{
			print $anvil->Words->string({key => "error_0405", variables => { anvil => $anvil->data->{switches}{anvil} }})."\n";
		}
		else
		{
			print $anvil->Words->string({key => "error_0406"})."\n";
		}
		foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
		{
			my $anvil_uuid        = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
			my $anvil_description = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description};
			print "- ".$anvil_name." - ".$anvil_description."\n";
		}
	}

	my $show_hosts = 0;
	if ($host_uuid)
	{
		# Is it a DR host.
		my $host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
		if ($host_type ne "dr")
		{
			$show_hosts = 1;
			$problem    = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
				show_hosts => $show_hosts,
				problem    => $problem,
			}});
			print $anvil->Words->string({key => "error_0408", variables => { host => $anvil->data->{switches}{'dr-host'} }})."\n";
		}
	}
	else
	{
		$show_hosts = 1;
		$problem    = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
			show_hosts => $show_hosts,
			problem    => $problem,
		}});
		if ($anvil->data->{switches}{anvil})
		{
			print "\n".$anvil->Words->string({key => "error_0407", variables => { host => $anvil->data->{switches}{'dr-host'} }})."\n";
		}
		else
		{
			print "\n".$anvil->Words->string({key => "error_0409"})."\n";
		}
	}

	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { show_hosts => $show_hosts }});
	if ($show_hosts)
	{
		$anvil->Database->get_hosts();
		foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }});
			my $this_host_uuid  = $anvil->data->{sys}{hosts}{by_name}{$host_name};
			my $short_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
			my $host_type       = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{host_type};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
				this_host_uuid  => $this_host_uuid,
				host_name       => $host_name,
				short_host_name => $short_host_name,
				host_type       => $host_type,
			}});
			next if $host_type ne "dr";
			print "- ".$short_host_name."\n";
		}
	}

	if ($problem)
	{
		print "\n".$anvil->Words->string({key => "error_0410"})."\n";
		foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
		{
			my $this_anvil_uuid   = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
			my $anvil_description = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description};
			print "- ".$anvil_name." - ".$anvil_description."\n";

			foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$this_anvil_uuid}{dr_link_host_name}})
			{
				my $dr_link_uuid = $anvil->data->{dr_links}{by_anvil_uuid}{$this_anvil_uuid}{dr_link_host_name}{$host_name}{dr_link_uuid};
				print " ^-> ".$host_name."\n";
			}
		}
		print "\n";
		$anvil->nice_exit({exit_code => 1});
	}

	# Still alivee? Update!
	my $dr_link_uuid = "";
	if (exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$host_uuid})
	{
		$dr_link_uuid = $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$host_uuid}{dr_link_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_link_uuid => $dr_link_uuid }});
	}

	if ($anvil->data->{switches}{'link'})
	{
		my $returned_dr_link_uuid = $anvil->Database->insert_or_update_dr_links({
			debug => 2,
			dr_link_host_uuid  => $host_uuid,
			dr_link_anvil_uuid => $anvil_uuid,
			dr_link_note       => "user-created",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { returned_dr_link_uuid => $returned_dr_link_uuid }});
		
		print "\n".$anvil->Words->string({key => "log_0734", variables => {
			host  => $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name},
			anvil => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name},
		}})."\n";
	}
	elsif ($anvil->data->{switches}{'unlink'})
	{
		if ($dr_link_uuid)
		{
			# Delete
			$anvil->Database->insert_or_update_dr_links({
				debug        => 2,
				dr_link_uuid => $dr_link_uuid,
				'delete'     => 1,
			});
			print "\n".$anvil->Words->string({key => "log_0735", variables => {
				host  => $anvil->data->{hosts}{host_uuid}{$host_uuid}{shost_host_name},
				anvil => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name},
			}})."\n";
		}
		else
		{
			# Link didn't exist, nothing to unlink.
			print "\n".$anvil->Words->string({key => "log_0736", variables => {
				host  => $anvil->data->{hosts}{host_uuid}{$host_uuid}{shost_host_name},
				anvil => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name},
			}})."\n";
		}
	}
	$anvil->nice_exit({exit_code => 0});

	return(0);
}

sub sanity_check
{
	my ($anvil, $terminal) = @_;
	
	# Begin sanity checks
	$anvil->Job->update_progress({
		progress => 10,
		message  => "job_0358", 
	});

	# Somehow, duplicate entries are being created in dr_links. Look for and purge them.
	check_for_duplicate_dr_links($anvil);

	# Are we a node or DR?
	my $host_type  = $anvil->Get->host_type();
	my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		host_type  => $host_type,
		anvil_uuid => $anvil_uuid, 
	}});

	if ($host_type eq "striker")
	{
		# This has to be run on an Anvil! sub-node. (For now at least. We need to make a menu to let
		# the user select a VM interactively).
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0404"});
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0404",
			job_status => "failed",
		});
		$anvil->nice_exit({exit_code => 1});
	}

	# If we're protecting, removing, connecting, disconnecting, or update, we need to have a server.
	$anvil->Database->get_servers();
	my $server_name = "";
	my $server_uuid = "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
		'switches::connect'    => $anvil->data->{switches}{'connect'},
		'switches::disconnect' => $anvil->data->{switches}{'disconnect'},
		'switches::protect'    => $anvil->data->{switches}{protect},
		'switches::remove'     => $anvil->data->{switches}{remove},
		'switches::update'     => $anvil->data->{switches}{update},
	}});
	if (($anvil->data->{switches}{'connect'})    or
	    ($anvil->data->{switches}{'disconnect'}) or
	    ($anvil->data->{switches}{protect})      or
	    ($anvil->data->{switches}{remove})       or
	    ($anvil->data->{switches}{update}))
	{
		# Were we given a server?
		if (not $anvil->data->{switches}{server})
		{
			# Please specify the server to manager using '--server <name or uuid>'.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0340"});
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0340",
				job_status => "failed",
			});

			# Show the servers on this anvil and exit.
			show_servers($anvil, $anvil_uuid);
			$anvil->nice_exit({exit_code => 1});
		}
		$server_uuid = $anvil->Database->get_server_uuid_from_string({
			debug  => 2,
			string => $anvil->data->{switches}{server},
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_uuid => $server_uuid }});
		if (not $server_uuid)
		{
			my $variables = { server => $anvil->data->{switches}{server} };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0411", variables => $variables});
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0411",
				variables  => $variables,
				job_status => "failed",
			});

			# Show the servers on this anvil and exit.
			show_servers($anvil, $anvil_uuid);
			$anvil->nice_exit({exit_code => 1});
		}

		# Lastly, is this server on this Anvil! node?
		   $server_name     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
		my $host_anvil_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
			server_name     => $server_name,
			host_anvil_uuid => $host_anvil_uuid." (".$anvil->data->{anvils}{anvil_uuid}{$host_anvil_uuid}{anvil_name}.")",
		}});

		if ($anvil_uuid ne $host_anvil_uuid)
		{
			# Server isn't here.
			my $variables = {
				server     => $server_name,
				this_anvil => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name},
				on_anvil   => $anvil->data->{anvils}{anvil_uuid}{$host_anvil_uuid}{anvil_name},
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0412", variables => $variables});
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0412",
				variables  => $variables,
				job_status => "failed",
			});

			# Show the servers on this anvil and exit.
			show_servers($anvil, $anvil_uuid);
			$anvil->nice_exit({exit_code => 1});
		}
	}

	# If we're (dis}connecting, is the server being protected in the first place?
	if (($anvil->data->{switches}{'connect'}) or ($anvil->data->{switches}{'disconnect'}))
	{
		# Is this server configured to be protected?
		my $config_file =  $anvil->data->{path}{directories}{drbd_resources}."/".$server_name.".res";
		   $config_file =~ s/\/\//\//g;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { config_file => $config_file }});
		if (not -e $config_file)
		{
			# Can't connect an unprotected server.
			my $variables = {
				server      => $server_name, 
				config_file => $config_file, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0371", variables => $variables});
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0371", 
				variables  => $variables, 
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
	}

	# If we're doing a --protect or --remove, make sure we're a node, the cluster is up, and both nodes 
	# are ready.
	if (($anvil->data->{switches}{protect}) or ($anvil->data->{switches}{remove}))
	{
		# Make sure we're in an Anvil! if we're a node (DR link will be checked later).
		if (($host_type eq "node") && (not $anvil_uuid))
		{
			# This must be run on a node active in the cluster  hosting the server being managed.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0332"});
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0332", 
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# If we're a node, make sure we're in a cluster.
		if ($host_type eq "node")
		{
			# Can we parse the CIB?
			my ($problem) = $anvil->Cluster->parse_cib();
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
			if ($problem)
			{
				# No. Clear the job,
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0336"});
				$anvil->Job->update_progress({
					progress   => 0,
					message    => "error_0336", 
					job_status => "failed", 
				});
				$anvil->nice_exit({exit_code => 1});
			}
			
			# Both nodes need to be in the cluster, are they?
			if (not $anvil->data->{cib}{parsed}{'local'}{ready})
			{
				# We're not a full member of the cluster yet. Please try again once we're fully in. Exiting.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0337"});
				$anvil->Job->update_progress({
					progress   => 0,
					message    => "error_0337", 
					job_status => "failed", 
				});
				$anvil->nice_exit({exit_code => 1});
			}
		}
		
		if ($anvil->data->{switches}{protect})
		{
			if (not $anvil->data->{switches}{protocol})
			{
				$anvil->data->{switches}{protocol} = "short-throw";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					'switches::protocol' => $anvil->data->{switches}{protocol},
				}});
			}
			elsif (($anvil->data->{switches}{protocol} ne "sync")        && 
			       ($anvil->data->{switches}{protocol} ne "short-throw") && 
			       ($anvil->data->{switches}{protocol} ne "long-throw"))
			{
				# The protocol is invalid. Please use '--help' for more information.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0342", variables => { protocol => $anvil->data->{switches}{protocol} }});
				$anvil->Job->update_progress({
					progress   => 100,
					message    => "error_0341,!!protocol!".$anvil->data->{switches}{protocol}."!!", 
					job_status => "failed", 
				});
				$anvil->nice_exit({exit_code => 1});
			}
			
			if ($anvil->data->{switches}{protocol} eq "long-throw")
			{
				# If there isn't a license file, make sure it's being provided.
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					'path::config::drbd-proxy.license' => $anvil->data->{path}{configs}{'drbd-proxy.license'},
				}});
				if (not -e $anvil->data->{path}{configs}{'drbd-proxy.license'})
				{
					if (not $anvil->data->{switches}{'license-file'})
					{
						# Proxy wouldn't work.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0373"});
						$anvil->Job->update_progress({
							progress   => 100,
							message    => "error_0373", 
							job_status => "failed", 
						});
						$anvil->nice_exit({exit_code => 1});
					}
					elsif (not -f $anvil->data->{switches}{'license-file'})
					{
						# It was passed, but doesn't point to the file.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0374", variables => { file => $anvil->data->{switches}{'license-file'} }});
						$anvil->Job->update_progress({
							progress   => 100,
							message    => "error_0374,!!file!".$anvil->data->{switches}{'license-file'}."!!", 
							job_status => "failed", 
						});
						$anvil->nice_exit({exit_code => 1});
					}
					
					# Still here? Copy the license file.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0429", variables => { file => $anvil->data->{switches}{'license-file'} }});
					my $failed = $anvil->Storage->copy_file({
						source_file => $anvil->data->{switches}{'license-file'}, 
						target_file => $anvil->data->{path}{configs}{'drbd-proxy.license'},
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
				}
				
				# Lastly, read in the license file to make sure it _looks_ ok.
				my $problem = $anvil->DRBD->check_proxy_license({debug => 2});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
				if ($problem)
				{
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0375"});
					$anvil->Job->update_progress({
						progress   => 100,
						message    => "error_0375", 
						job_status => "failed", 
					});
					$anvil->nice_exit({exit_code => 1});
				}
			}
		}
	}
	
	# Get the Anvil! details.
	$anvil->Database->get_hosts();
	$anvil->Database->get_storage_group_data({debug => 2});
	$anvil->Database->get_dr_links({debug => 2});

	# Does this Anvil! have at least one DR node? If there's only one, use it. If more than one, we need
	# a '--dr-host' switch.
	my $dr_host_uuid = "";
	if ($anvil->data->{switches}{'dr-host'})
	{
		$dr_host_uuid = $anvil->Database->get_host_uuid_from_string({string => $anvil->data->{switches}{'dr-host'}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_host_uuid => $dr_host_uuid }});
		if (not $dr_host_uuid)
		{
			# Bad host.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0400", variables => {
				dr_host => $anvil->data->{switches}{'dr-host'},
			}});
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0400,!!dr_host!".$anvil->data->{switches}{'dr-host'}."!!",
				job_status => "failed",
			});
			$anvil->nice_exit({exit_code => 1});
		}
	}

	# If I don't have a dr_host_uuid yet, see which are available. If only one, use it. If two or more, 
	# and if the server is already being protected, determine which to use from it's config. Otherwise,
	# tell the user they need to specify which.
	if (not $dr_host_uuid)
	{
		my $dr_count = keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
			anvil_uuid => $anvil_uuid,
			dr_count   => $dr_count,
		}});

		if (not $dr_count)
		{
			print $anvil->Words->string({key => "error_0333"})."\n";
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0333",
				job_status => "failed",
			});
			$anvil->nice_exit({exit_code => 1});
		}
		if ($dr_count eq "1")
		{
			# Only one, use it.
			foreach my $host_uuid (keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}})
			{
				$dr_host_uuid = $host_uuid;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_host_uuid => $dr_host_uuid }});
			}
		}
		else
		{
			# Two or more. Is the server already protected? If so, try to find which DR it's 
			# using.
			if (($anvil->data->{switches}{'connect'}) or ($anvil->data->{switches}{'disconnect'}))
			{
				# Read the config.
				my $config_file =  $anvil->data->{path}{directories}{drbd_resources}."/".$server_name.".res";
				   $config_file =~ s/\/\//\//g;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { config_file => $config_file }});
				
				my $resource_config = $anvil->Storage->read_file({file => $config_file});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource_config => $resource_config }});
				foreach my $line (split/\n/, $resource_config)
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { lineg => $line }});
					
				}
			}
		}
	}

	# If I still don't have a DR host, fail out.
	if (not $dr_host_uuid)
	{
		print $anvil->Words->string({key => "error_0402"})."\n";
		foreach my $dr_link_host_name (sort {$a cmp $b} keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}})
		{
			my $dr_link_uuid      = $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}{$dr_link_host_name}{dr_link_uuid};
			my $dr_link_host_uuid = $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_host_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
				dr_link_host_name => $dr_link_host_name,
				dr_link_uuid      => $dr_link_uuid,
				dr_link_host_uuid => $dr_link_host_uuid,
			}});
			print $anvil->Words->string({key => "error_0403", variables => {
				host_name => $dr_link_host_name,
				host_uuid => $dr_link_host_uuid,
			}})."\n";
		}
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0402",
			job_status => "failed",
		});
		$anvil->nice_exit({exit_code => 1});
	}

	my $dr_host_name       = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{host_name};
	my $dr_short_host_name = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{short_host_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
		dr_host_name       => $dr_host_name,
		dr_short_host_name => $dr_short_host_name,
	}});

	# Last check, we've got a dr_host_uuid, is it associated with this Anvil! node?
	if (not exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$dr_host_uuid})
	{
		# Bad requested DR host.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0401", variables => {
			dr_host => $dr_host_name,
		}});
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0401,!!dr_host!".$dr_host_name."!!",
			job_status => "failed",
		});
		$anvil->nice_exit({exit_code => 1});
	}

	$anvil->data->{sys}{dr_host_uuid}       = $dr_host_uuid;
	$anvil->data->{sys}{dr_host_name}       = $dr_host_name;
	$anvil->data->{sys}{dr_short_host_name} = $dr_short_host_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
		"sys::dr_host_uuid"       => $anvil->data->{sys}{dr_host_uuid},
		"sys::dr_host_name"       => $anvil->data->{sys}{dr_host_name},
		"sys::dr_short_host_name" => $anvil->data->{sys}{dr_short_host_name},
	}});

	### TODO: Switch to: my $target_ip = $anvil->Network->find_target_ip({});
	# Can we access DR, if we're not the DR host?
	$anvil->Database->get_ip_addresses({debug => 2});
	my $password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $dr_ip    = $anvil->System->find_matching_ip({
		debug => 2,
		host  => $dr_host_name,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
		password => $anvil->Log->is_secure($password),
		dr_ip    => $dr_ip,
	}});
	if ($dr_host_uuid ne $anvil->Get->host_uuid)
	{
		if ((not $dr_ip) or ($dr_ip eq "!!error!!"))
		{
			# Failed to find an IP we can access the DR host. Has it been configured? Is it running? Exiting.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, secure => 0, key => "error_0334", variables => { host_name => $dr_host_name }});
			$anvil->Job->update_progress({
				progress   => 0,
				message    => "error_0334,!!host_name!".$dr_host_name."!!",
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Test access.
		my $access = $anvil->Remote->test_access({
			target   => $dr_ip,
			password => $password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
		if (not $access)
		{
			# Failed to access the DR host. Is it running? Exiting.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0335", variables => { 
				host_name  => $dr_host_name,
				ip_address => $dr_ip, 
			}});
			$anvil->Job->update_progress({
				progress   => 0,
				message    => "error_0335,!!host_name!".$dr_host_name."!!,!!ip_address!".$dr_ip."!!",
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		### TODO: We can queue a job to update the peer later, there's no real need, in the long run,
		###       for the peer to be online.
		# If we're protecting or removing a server from DR, the peer needs to be up.
		if ((($anvil->data->{switches}{protect})   or 
		     ($anvil->data->{switches}{remove})    or 
		     ($anvil->data->{switches}{protocol})) && 
		    (not $anvil->data->{cib}{parsed}{peer}{ready}))
		{
			if ($anvil->data->{switches}{protect})
			{
				# We can't setup a server to be protected unless both nodes are up, and the 
				# peer isn't at this time.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0338"});
				$anvil->Job->update_progress({
					progress   => 0,
					message    => "error_0338", 
					job_status => "failed", 
				});
			}
			else
			{
				# We can't remove a server from DR unless both nodes are up, and the peer 
				# isn't at this time.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0339"});
				$anvil->Job->update_progress({
					progress   => 0,
					message    => "error_0339", 
					job_status => "failed", 
				});
			}
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	# Verify we found the server.
	$anvil->data->{server}{'server-name'} = "";
	$anvil->data->{server}{'server-uuid'} = "";
	$anvil->data->{server}{'anvil-uuid'}  = $anvil_uuid;
	if (not $anvil->data->{switches}{server})
	{
	}
	else
	{
		my $server = $anvil->data->{switches}{server};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server => $server }});
		
		if (exists $anvil->data->{servers}{server_uuid}{$server})
		{
			$anvil->data->{server}{'server-uuid'} = $server;
			$anvil->data->{server}{'server-name'} = $anvil->data->{servers}{server_uuid}{$server}{server_name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				'server::server-uuid' => $anvil->data->{server}{'server-uuid'},
				'server::server-name' => $anvil->data->{server}{'server-name'},
			}});
		}
		if (exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server})
		{
			$anvil->data->{server}{'server-name'} = $server;
			$anvil->data->{server}{'server-uuid'} = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server}{server_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				'server::server-uuid' => $anvil->data->{server}{'server-uuid'},
				'server::server-name' => $anvil->data->{server}{'server-name'},
			}});
		}
	}
	
	# Get and parse the server's definition to find the DRBD devices.
	if ((not $anvil->data->{server}{'server-uuid'}) or (not $anvil->data->{server}{'server-name'}))
	{
		# Failed to find the server by name or UUID.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0341", variables => { server => $anvil->data->{switches}{server} }});
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0341,!!server!".$anvil->data->{switches}{server}."!!", 
			job_status => "failed", 
		});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Are we being asked to actuall do something?
	if (((not $anvil->data->{switches}{'connect'})  && 
	     (not $anvil->data->{switches}{disconnect}) && 
	     (not $anvil->data->{switches}{protect})    && 
	     (not $anvil->data->{switches}{remove})     && 
	     (not $anvil->data->{switches}{update}))    or 
	     ($anvil->data->{switches}{help})           or 
	     ($anvil->data->{switches}{h}))
	{
		# Show the man page.
		system($anvil->data->{path}{exe}{man}." ".$THIS_FILE);
		if (($anvil->data->{switches}{help}) or ($anvil->data->{switches}{h}))
		{
			$anvil->nice_exit({exit_code => 0});
		}
		else
		{
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	# Sanity checks complete!
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0359"});
	$anvil->Job->update_progress({
		progress => 20,
		message  => "job_0359", 
	});
	
	# If we've got a got job-uuid, wait for / make sure all machines are online.
	if ($anvil->data->{switches}{'job-uuid'})
	{
		# Loop until we have access to both the peer machines.
		my $waiting  = 1;
		my $wait_for = 10;
		while ($waiting)
		{
			# This will get set to 1 if we have to keep waiting.
			$waiting = 0;
			my $password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
			my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
			my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
			my $dr_host_uuid    = $anvil->data->{sys}{dr_host_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				password        => $anvil->Log->is_secure($password),
				node1_host_uuid => $node1_host_uuid,
				node2_host_uuid => $node2_host_uuid, 
				dr_host_uuid    => $dr_host_uuid,
			}});
			foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
			{
				next if $this_host_uuid eq $anvil->Get->host_uuid();
				my $this_host_name = $anvil->Get->host_name_from_uuid({host_uuid => $this_host_uuid});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					this_host_uuid => $this_host_uuid,
					this_host_name => $this_host_name,
				}});
				
				# We'll try the SN, then the BCN and finally the IFN to see which, if any, network we
				# can reach the peer on. This is needed because the DR host could be on a totally 
				# different network.
				$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip}      = "";
				$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_subnet}  = "";
				$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_network} = "";
				foreach my $check_network ("sn", "bcn", "mn", "ifn")
				{
					last if $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { check_network => $check_network }});
					foreach my $this_network (sort {$a cmp $b} keys %{$anvil->data->{hosts}{host_uuid}{$this_host_uuid}{network}})
					{
						next if $this_network !~ /^$check_network/;
						my $this_ip     = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{network}{$this_network}{ip_address};
						my $this_subnet = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{network}{$this_network}{subnet_mask};
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							's1:this_ip'      => $this_ip,
							's2:this_network' => $this_network, 
							's3:this_subnet'  => $this_subnet,
						}});
						
						# Test access.
						my $access = $anvil->Remote->test_access({
							target   => $this_ip,
							password => $password,
						});
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
						if ($access)
						{
							$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip}      = $this_ip;
							$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_subnet}  = $this_subnet;
							$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_network} = $this_network;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								"hosts::host_uuid::${this_host_uuid}::network::use_ip"      => $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip},
								"hosts::host_uuid::${this_host_uuid}::network::use_subnet"  => $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_subnet},
								"hosts::host_uuid::${this_host_uuid}::network::use_network" => $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_network},
							}});
						}
					}
				}
				
				if (not $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip})
				{
					# No access
					my $variables = {
						waiting   => $wait_for,
						host_name => $this_host_name,
					};
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0404", variables => $variables});
					$anvil->Job->update_progress({
						progress  => 12,
						message   => "job_0404", 
						variables => $variables, 
					});
					
					$waiting = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				}
			}
			
			# If we're waiting for a peer, record as such.
			sleep $wait_for;
		}
	}

	# If we're protecting, make sure there's enough space on the DR host.
	if ($anvil->data->{switches}{protect})
	{
		process_protect($anvil, $terminal);
	}
	elsif ($anvil->data->{switches}{remove})
	{
		process_remove($anvil, $terminal);
	}
	elsif ($anvil->data->{switches}{'connect'})
	{
		process_connect($anvil, $terminal);
	}
	elsif ($anvil->data->{switches}{disconnect})
	{
		process_disconnect($anvil, $terminal);
	}
	elsif ($anvil->data->{switches}{update})
	{
		process_update($anvil, $terminal);
	}
	
	return(0);
}

sub process_update
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr_host_uuid          = $anvil->data->{sys}{dr_host_uuid};
	my $dr_host_name          = $anvil->data->{sys}{dr_host_name};
	my $dr_short_host_name    = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr_host_uuid          => $dr_host_uuid,
		dr_host_name          => $dr_host_name,
		dr_short_host_name    => $dr_short_host_name,
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
	}});
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{Yes})
	{
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0395", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 25,
			message   => "job_0395", 
			variables => $variables, 
		});
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "update=1\n";
		
		# Register the job with the DR host.
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $dr_host_uuid,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	# If the resource is down, bring it up.
	my $variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0387", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 50,
		message   => "job_0387", 
		variables => $variables, 
	});
	
	# Bring up the connection locally, and then also bring up the connection on the nodes, in case the 
	# server is down.
	my $drbd_up_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." || ".$anvil->data->{path}{exe}{drbdadm}." up ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_up_call => $drbd_up_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_up_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0388", variables => $variables});
	$anvil->Job->update_progress({
		progress => 60,
		message  => "job_0388", 
	});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either/both nodes
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0389", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 70,
			message   => "job_0389", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $drbd_up_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Now watch until out volume(s) are UpToDate
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0390"});
	$anvil->Job->update_progress({
		progress => 70,
		message  => "job_0390", 
	});
	
	my $waiting = 1;
	while ($waiting)
	{
		$anvil->DRBD->gather_data({debug => 2});
		   $waiting      = 0;
		my $time_to_sync = "";
		my $sync_speed   = "";
		my $sync_source  = "";
		foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
			foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}})
			{
				my $replication_speed      = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{replication_speed};
				my $peer_role              = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_role};
				my $peer_disk_state        = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_disk_state};
				my $estimated_time_to_sync = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{estimated_time_to_sync};
				my $local_disk_state       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_disk_state};
				my $connection_state       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{connection_state};
				my $local_role             = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_role};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					peer                   => $peer,
					replication_speed      => $replication_speed, 
					peer_role              => $peer_role, 
					peer_disk_state        => $peer_disk_state, 
					estimated_time_to_sync => $estimated_time_to_sync, 
					local_disk_state       => $local_disk_state, 
					connection_state       => $connection_state, 
					local_role             => $local_role, 
				}});
				
				if ($connection_state eq "synctarget")
				{
					$sync_source = $peer;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sync_source => $sync_source }});
				}
				if ($replication_speed)
				{
					$sync_speed = $anvil->Convert->bytes_to_human_readable({'bytes' => $replication_speed});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sync_speed => $sync_speed }});
				}
				if ($estimated_time_to_sync)
				{
					$time_to_sync = $anvil->Convert->time({'time' => $estimated_time_to_sync, translate => 1});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { time_to_sync => $time_to_sync }});
				}
				
				if ($local_disk_state ne "uptodate")
				{
					$waiting = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				}
			}
		}
		if ($waiting)
		{
			if ($sync_source)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					sync_source  => $sync_source, 
					sync_speed   => $sync_speed, 
					time_to_sync => $time_to_sync, 
				}});
				my $variables = { 
					sync_source  => $sync_source,
					sync_speed   => $sync_speed, 
					time_to_sync => $time_to_sync, 
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0396", variables => $variables});
				$anvil->Job->update_progress({
					progress  => 80,
					message   => "job_0396", 
					variables => $variables, 
				});
			}
			else
			{
				$anvil->Job->update_progress({
					progress  => 75,
					message   => "job_0398", 
					variables => $variables, 
				});
			}
			sleep 5;
		}
	}
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0397", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 90,
		message   => "job_0397", 
		variables => $variables, 
	});
	
	# Bring down the connection.
	my $drbd_down_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." && ".$anvil->data->{path}{exe}{drbdadm}." down ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_down_call => $drbd_down_call }});
	($output, $return_code) = $anvil->System->call({shell_call => $drbd_down_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0391", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0391", 
		variables => $variables, 
	});
	
	return(0);
}

sub process_disconnect
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr_host_uuid          = $anvil->data->{sys}{dr_host_uuid};
	my $dr_host_name          = $anvil->data->{sys}{dr_host_name};
	my $dr_short_host_name    = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr_host_uuid          => $dr_host_uuid,
		dr_host_name          => $dr_host_name,
		dr_short_host_name    => $dr_short_host_name,
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
	}});
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{Yes})
	{
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0392", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 25,
			message   => "job_0392", 
			variables => $variables, 
		});
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "disconnect=1\n";
		
		# Register the job with the DR host.
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $dr_host_uuid,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	# If the resource is up, bring it down.
	my $variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0393", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 50,
		message   => "job_0393", 
		variables => $variables, 
	});
	
	# Bring up the connection locally, and then also bring up the connection on the nodes, in case the 
	# server is down.
	my $drbd_down_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." && ".$anvil->data->{path}{exe}{drbdadm}." down ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_down_call => $drbd_down_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_down_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0394", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0394", 
		variables => $variables, 
	});
	
	return(0);
}

sub process_connect
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr_host_uuid          = $anvil->data->{sys}{dr_host_uuid};
	my $dr_host_name          = $anvil->data->{sys}{dr_host_name};
	my $dr_short_host_name    = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr_host_uuid          => $dr_host_uuid,
		dr_host_name          => $dr_host_name,
		dr_short_host_name    => $dr_short_host_name,
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
	}});
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{Yes})
	{
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0386", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 25,
			message   => "job_0386", 
			variables => $variables, 
		});
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "connect=1\n";
		
		# Register the job with the DR host.
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $dr_host_uuid,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	# If the resource is down, bring it up.
	my $variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0387", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 50,
		message   => "job_0387", 
		variables => $variables, 
	});
	
	# Bring up the connection locally, and then also bring up the connection on the nodes, in case the 
	# server is down.
	my $drbd_up_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." || ".$anvil->data->{path}{exe}{drbdadm}." up ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_up_call => $drbd_up_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_up_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	
	# If this was an error, abort.
	if ($return_code)
	{
		# Failed.
		my $variables = {
			return_code => $return_code, 
			output      => $output, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0370", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 100,
			message   => "error_0370", 
			variables => $variables, 
		});
		$anvil->nice_exit({exit_code => 1});
	}
	
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0388", variables => $variables});
	$anvil->Job->update_progress({
		progress => 60,
		message  => "job_0388", 
	});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either/both nodes
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		$variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0389", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 70,
			message   => "job_0389", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $drbd_up_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Now watch until we connect to at least one peer.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0390"});
	$anvil->Job->update_progress({
		progress => 80,
		message  => "job_0390", 
	});
	my $waiting = 1;
	while($waiting)
	{
		sleep 5;
		$anvil->DRBD->gather_data({debug => 2});
		
		$waiting = 0;
		foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
			my $volume_up = 0;
			foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}})
			{
				my $replication_speed      = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{replication_speed};
				my $peer_role              = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_role};
				my $peer_disk_state        = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_disk_state};
				my $estimated_time_to_sync = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{estimated_time_to_sync};
				my $local_disk_state       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_disk_state};
				my $connection_state       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{connection_state};
				my $local_role             = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_role};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					peer                   => $peer,
					replication_speed      => $replication_speed, 
					peer_role              => $peer_role, 
					peer_disk_state        => $peer_disk_state, 
					estimated_time_to_sync => $estimated_time_to_sync, 
					local_disk_state       => $local_disk_state, 
					connection_state       => $connection_state, 
					local_role             => $local_role, 
				}});
				
				if ($connection_state eq "established")
				{
					$volume_up = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume_up => $volume_up }});
				}
				last if $volume_up;
			}
			
			if (not $volume_up)
			{
				$waiting = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
			}
		}
		
		if (not $waiting)
		{
			# We're ready.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0379"});
			$anvil->Job->update_progress({
				progress => 95,
				message  => "job_0379", 
			});
			$waiting = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
		}
	}
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0391", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0391", 
		variables => $variables, 
	});
	
	return(0);
}

sub process_remove
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr_host_uuid          = $anvil->data->{sys}{dr_host_uuid};
	my $dr_host_name          = $anvil->data->{sys}{dr_host_name};
	my $dr_short_host_name    = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	my $short_host_name       = $anvil->Get->short_host_name();
	my $server_definition_xml = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr_host_uuid          => $dr_host_uuid,
		dr_host_name          => $dr_host_name,
		dr_short_host_name    => $dr_short_host_name,
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
		server_definition_xml => $server_definition_xml, 
		short_host_name       => $short_host_name,
	}});
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{Yes})
	{
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0407", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 25,
			message   => "job_0407", 
			variables => $variables, 
		});
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "remove=1\n";
		
		# Register the job with the DR host.
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $dr_host_uuid,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	# Sanity checks complete!
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0406", variables => { server => $anvil->data->{server}{'server-name'} }});
	$anvil->Job->update_progress({
		progress => 30,
		message  => "job_0406,!!server!".$anvil->data->{server}{'server-name'}."!!", 
	});
	
	# Bring down the resource, if it's up.
	my $variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0393", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 40,
		message   => "job_0393", 
		variables => $variables, 
	});
	
	# Bring down the connection locally
	my $drbd_down_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." && ".$anvil->data->{path}{exe}{drbdadm}." down ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_down_call => $drbd_down_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_down_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	
	# Read through the local DRBD config file, and remove any LVs that are found to still exist.
	$anvil->DRBD->gather_data({debug => 2});
	
	my $config_file = $anvil->data->{new}{resource}{$server_name}{config_file} ? $anvil->data->{new}{resource}{$server_name}{config_file} : "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { config_file => $config_file }});
	
	# If we don't have a config file, we could be re-running and the file was wiped. We need this though,
	# so try to pull it from a node.
	if (not $config_file)
	{
		# Try to read it from node 1.
		$config_file = $anvil->data->{path}{directories}{drbd_resources}."/".$server_name.".res";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { config_file => $config_file }});
		
		my $variables = { 
			config_file => $config_file,
			source      => $node1_short_host_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0417", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 45,
			message   => "job_0417", 
			variables => $variables, 
		});
		
		# Rsync it over. 
		my $source = "root\@".$anvil->data->{lookup}{host_uuid}{$node1_host_uuid}{network}{use_ip}.":".$config_file;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { source => $source }});
		$anvil->Storage->rsync({
			source      => $source,
			destination => $anvil->data->{path}{directories}{drbd_resources}."/",
		});
		
		# Rescan DRBD
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0418"});
		$anvil->Job->update_progress({
			progress => 50,
			message  => "job_0418", 
		});
		$anvil->DRBD->gather_data({debug => 2});
	}
	
	# Read the old config file.
	my $old_resource_config = $anvil->Storage->read_file({file => $config_file});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_resource_config => $old_resource_config }});
	
	my $progress = "50";
	foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
	{
		my $local_lv = $anvil->data->{new}{resource}{$server_name}{host}{$dr_short_host_name}{volume}{$volume}{backing_disk};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			volume   => $volume,
			local_lv => $local_lv, 
		}});
		
		$progress++;
		my $variables = {
			server   => $server_name, 
			volume   => $volume,
			local_lv => $local_lv, 
		};
		if (-e $local_lv)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0408", variables => $variables});
			$anvil->Job->update_progress({
				progress  => $progress,
				message   => "job_0408", 
				variables => $variables, 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0591", variables => { device_path => $local_lv }});
			my $return_code = $anvil->DRBD->remove_backing_lv({
				debug        => 2, 
				backing_disk => $local_lv, 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
			if ($return_code)
			{
				# Should have been '0'
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0368", variables => $variables});
				$anvil->Job->update_progress({
					progress  => 100,
					message   => "error_0368", 
					variables => $variables, 
				});
				$anvil->nice_exit({exit_code => 1});
			}
			else
			{
				# Gone.
				$progress++;
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0410"});
				$anvil->Job->update_progress({
					progress  => $progress,
					message   => "job_0410", 
				});
			}
		}
		else
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0409", variables => $variables});
			$anvil->Job->update_progress({
				progress  => $progress,
				message   => "job_0409", 
				variables => $variables, 
			});
		}
	}
	
	# Tell the user we're generationg the new resource config now.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0411"});
	$anvil->Job->update_progress({
		progress  => 70,
		message   => "job_0411", 
	});
	
	### If we're here, we can now update the resource config file to remove the DR host.
	# Refresh the IP info (usually scrubbed by this point)
	$anvil->Database->get_ip_addresses();
	
	# First loop to build the node sections, then we'll loop to build the connections
	my $node1_volumes    = "";
	my $node2_volumes    = "";
	my $node1_ip_address = "";
	my $node1_tcp_port   = "";
	my $node2_ip_address = "";
	my $node2_tcp_port   = "";
	foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
	{
		# Find the node 1 backing device, drbd device and minor numbers.
		my $node1_device_path  = $anvil->data->{new}{resource}{$server_name}{host}{$node1_short_host_name}{volume}{$volume}{device_path};
		my $node1_backing_disk = $anvil->data->{new}{resource}{$server_name}{host}{$node1_short_host_name}{volume}{$volume}{backing_disk};
		my $node1_device_minor = $anvil->data->{new}{resource}{$server_name}{host}{$node1_short_host_name}{volume}{$volume}{device_minor};
		my $node1_meta_disk    = $anvil->data->{new}{resource}{$server_name}{host}{$node1_short_host_name}{volume}{$volume}{meta_disk};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:node1_device_path'  => $node1_device_path,
			's2:node1_backing_disk' => $node1_backing_disk, 
			's3:node1_device_minor' => $node1_device_minor, 
			's4:node1_meta_disk'    => $node1_meta_disk, 
		}});
		
		$node1_volumes .= $anvil->Words->string({key => "file_0004", variables => {
			volume    => $volume,
			drbd_path => $node1_device_path, 
			minor     => $node1_device_minor, 
			lv_path   => $node1_backing_disk, 
			meta_data => $node1_meta_disk, 
		}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_volumes => $node1_volumes }});
		
		my $node2_device_path  = $anvil->data->{new}{resource}{$server_name}{host}{$node2_short_host_name}{volume}{$volume}{device_path};
		my $node2_backing_disk = $anvil->data->{new}{resource}{$server_name}{host}{$node2_short_host_name}{volume}{$volume}{backing_disk};
		my $node2_device_minor = $anvil->data->{new}{resource}{$server_name}{host}{$node2_short_host_name}{volume}{$volume}{device_minor};
		my $node2_meta_disk    = $anvil->data->{new}{resource}{$server_name}{host}{$node2_short_host_name}{volume}{$volume}{meta_disk};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:node2_device_path'  => $node2_device_path,
			's2:node2_backing_disk' => $node2_backing_disk, 
			's3:node2_device_minor' => $node2_device_minor, 
			's4:node2_meta_disk'    => $node2_meta_disk, 
		}});
		
		$node2_volumes .= $anvil->Words->string({key => "file_0004", variables => {
			volume    => $volume,
			drbd_path => $node2_device_path, 
			minor     => $node2_device_minor, 
			lv_path   => $node2_backing_disk, 
			meta_data => $node2_meta_disk, 
		}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_volumes => $node2_volumes }});
		
		if (not $node1_ip_address)
		{
			$node1_ip_address = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$node1_short_host_name}{$node2_short_host_name}{host1_ip_address};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_ip_address => $node1_ip_address }});
		}
		if (not $node1_tcp_port)
		{
			$node1_tcp_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$node1_short_host_name}{$node2_short_host_name}{host1_tcp_port};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_tcp_port => $node1_tcp_port }});
		}
		if (not $node2_ip_address)
		{
			$node2_ip_address = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$node1_short_host_name}{$node2_short_host_name}{host2_ip_address};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_ip_address => $node2_ip_address }});
		}
		if (not $node2_tcp_port)
		{
			$node2_tcp_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$node1_short_host_name}{$node2_short_host_name}{host2_tcp_port};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_tcp_port => $node2_tcp_port }});
		}
	}

	# Put together the nodes section.
	my $hosts = $anvil->Words->string({key => "file_0003", variables => {
		short_host_name => $node1_short_host_name, 
		node_id         => "0",
		volumes         => $node1_volumes, 
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { hosts => $hosts }});

	$hosts .= $anvil->Words->string({key => "file_0003", variables => {
		short_host_name => $node2_short_host_name, 
		node_id         => "1",
		volumes         => $node2_volumes, 
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { hosts => $hosts }});

	# Build the connections now
	my $connections = $anvil->Words->string({key => "file_0005", variables => {
		host1_short_name => $node1_short_host_name,
		host1_ip         => $node1_ip_address,
		tcp_port         => $node1_tcp_port,
		host2_short_name => $node2_short_host_name,
		host2_ip         => $node2_ip_address,
		tcp_port         => $node2_tcp_port,
		'c-rate-maximum' => 500,
		protocol         => "C",
		fencing          => "resource-and-stonith"
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { connections => $connections }});

	my $new_resource_config = $anvil->Words->string({key => "file_0006", variables => {
		server      => $server_name,
		hosts       => $hosts,
		connections => $connections,
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_resource_config => $new_resource_config }});
	
	my $difference = diff \$old_resource_config, \$new_resource_config, { STYLE => 'Unified' };
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});

	# Write out a test file.
	my $test_file = $anvil->data->{path}{directories}{temp}."/test-".$server_name.".res";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }});
	my ($problem) = $anvil->Storage->write_file({
		debug     => 2,
		backup    => 0,
		overwrite => 1,
		file      => $test_file, 
		body      => $new_resource_config,
		user      => "root", 
		group     => "root", 
		mode      => "0644",
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	
	# Validate.
	my $shell_call = $anvil->data->{path}{exe}{drbdadm}." --config-to-test ".$test_file." --config-to-exclude ".$config_file." sh-nop";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	if ($return_code)
	{
		# Something went wrong.
		my $variables = {
			return_code => $return_code, 
			output      => $output, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0345", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 100,
			message   => "error_0345", 
			variables => $variables, 
		});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Remove the test file.
	unlink $test_file;
	
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0412"});
	$anvil->Job->update_progress({
		progress  => 72,
		message   => "job_0412", 
	});
	
	# Copy the new config to the peers.
	$progress = 72;
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid)
	{
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = {
			file       => $config_file, 
			host_name  => $peer_host_name, 
			ip_address => $peer_ip, 
		};
		$progress += 2;
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0369", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $progress,
			message   => "job_0369", 
			variables => $variables, 
		});
		my ($problem) = $anvil->Storage->write_file({
			debug     => 2,
			backup    => 1,
			overwrite => 1,
			file      => $config_file, 
			body      => $new_resource_config, 
			user      => "root", 
			group     => "root", 
			mode      => "0644",
			target    => $peer_ip, 
			password  => $anvil_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		# Make sure the file exists now.
		my $check_resource_config = $anvil->Storage->read_file({
			file     => $config_file,
			target   => $peer_ip, 
			password => $anvil_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { check_resource_config => $check_resource_config }});
		
		my $difference = diff \$new_resource_config, \$check_resource_config, { STYLE => 'Unified' };
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
		
		# Failed to write the file.
		if ($difference)
		{
			$variables = {
				host_name             => $peer_host_name, 
				file                  => $config_file, 
				difference            => $difference, 
				new_resource_config   => $new_resource_config, 
				check_resource_config => $check_resource_config, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0405", variables => $variables});
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "job_0405", 
				variables  => $variables, 
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	# Call 'drbdadm adjust <server>' to update the running configs.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0413"});
	$anvil->Job->update_progress({
		progress => 76,
		message  => "job_0413", 
	});
	
	my $drbdadm_call = $anvil->data->{path}{directories}{drbdadm}." adjust ".$server_name.$anvil->Log->switches();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbdadm_call => $drbdadm_call }});
	($output, $return_code) = $anvil->System->call({shell_call => $drbdadm_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0414", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 77,
			message   => "job_0414", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $drbdadm_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Now delete our resource config file.
	$variables = { config_file => $config_file };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0415", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 80,
		message   => "job_0415", 
		variables => $variables, 
	});
	
	if (-e $config_file)
	{
		my $problem = $anvil->Storage->delete_file({file => $config_file});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		if ($problem)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0369", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0369", 
				variables => $variables
			});
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0416", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0416", 
		variables => $variables, 
	});
	
	return(0);
}

sub process_protect
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr_host_uuid          = $anvil->data->{sys}{dr_host_uuid};
	my $dr_host_name          = $anvil->data->{sys}{dr_host_name};
	my $dr_short_host_name    = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	my $short_host_name       = $anvil->Get->short_host_name();
	my $server_definition_xml = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr_host_uuid          => $dr_host_uuid,
		dr_host_name          => $dr_host_name,
		dr_short_host_name    => $dr_short_host_name,
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
		server_definition_xml => $server_definition_xml, 
		short_host_name       => $short_host_name,
	}});
	
	# Sanity checks complete!
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0360", variables => { server => $anvil->data->{server}{'server-name'} }});
	$anvil->Job->update_progress({
		progress => 30,
		message  => "job_0360,!!server!".$anvil->data->{server}{'server-name'}."!!", 
	});
	
	$anvil->Server->parse_definition({
		debug      => 2,
		host       => $short_host_name,
		server     => $anvil->data->{server}{'server-name'}, 
		source     => "from_db",
		definition => $server_definition_xml, 
	});
	
	$anvil->DRBD->gather_data({debug => 2});
	
	my $server_ram = $anvil->data->{server}{$short_host_name}{$server_name}{'from_db'}{memory};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server_ram => $anvil->Convert->add_commas({number => $server_ram})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram}).")",
	}});
	
	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{drbd}{resource}})
	{
		next if $resource eq "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource => $resource }});
		
		foreach my $this_host_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{host}})
		{
			my $this_host_uuid = $anvil->Get->host_uuid_from_name({host_name => $this_host_name});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				this_host_name => $this_host_name,
				this_host_uuid => $this_host_uuid, 
			}});
			
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}})
			{
				# Always get the LV sizes, as that factors metadata. DRBD size is 
				# minus metadata, and 0 when down.
				my $device_path  = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_path};
				my $backing_disk = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{backing_disk};
				my $device_minor = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_minor};
				my $tcp_port     = $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:volume"       => $volume, 
					"s2:device_path"  => $device_path,
					"s3:backing_disk" => $backing_disk,
					"s4:device_minor" => $device_minor,
				}});
				
				my $this_size = $anvil->Storage->get_size_of_block_device({debug => 2, host_uuid => $this_host_uuid, path => $backing_disk});
				if ($this_size eq "")
				{
					# DRBD config file was updated, but LV not created yet.
					next;
				}
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					this_size => $anvil->Convert->add_commas({number => $this_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $this_size}).")", 
				}});
				
				if ((not exists $anvil->data->{server}{drbd}{$resource}{$volume}{size}) or (not $anvil->data->{server}{drbd}{$resource}{$volume}{size}))
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{size} = $this_size;
					
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::size" => $anvil->data->{server}{drbd}{$resource}{$volume}{size}, 
					}});
				}
				
				if (not exists $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid})
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid} = "";
				}
				
				### NOTE: This check make sense only under the assumption that the DRBD minor
				###       is common across both nodes. This should be the case, but doesn't 
				###       strictly have to be so.
				if ((not exists $anvil->data->{server}{drbd}{$resource}{$volume}{minor_number})  or 
				    (not defined $anvil->data->{server}{drbd}{$resource}{$volume}{minor_number}) or 
				    ($anvil->data->{server}{drbd}{$resource}{$volume}{minor_number} eq ""))
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{minor_number} = $device_minor;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::minor_number" => $anvil->data->{server}{drbd}{$resource}{$volume}{minor_number},
					}});
				}
				
				if ((not exists $anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port})  or 
				    (not defined $anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port}) or 
				    ($anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port} eq ""))
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port} = $tcp_port;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::tcp_port" => $anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port},
					}});
				}
				
				# What storage group does this belong to?
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"server::drbd::${resource}::${volume}::storage_group_uuid" => $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid}, 
				}});
				if (not $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid})
				{
					my $storage_key        = $resource."/".$volume;
					my $storage_group_uuid = $anvil->Storage->get_storage_group_from_path({
						debug      => 2,
						anvil_uuid => $anvil_uuid, 
						path       => $backing_disk,
					});
					my $storage_group_name = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						storage_key        => $storage_key, 
						storage_group_uuid => $storage_group_uuid, 
						storage_group_name => $storage_group_name, 
					}});
					
					# We'll need to sum up the volumes on each storage group, as 
					# it's possible the volumes are on different SGs.
					$anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid}               = $storage_group_uuid;
					$anvil->data->{server}{storage_groups}{$storage_group_name}{used_by}{$storage_key} = 1;
					$anvil->data->{server}{storage_groups}{$storage_group_name}{storage_group_uuid}    = $storage_group_uuid;
					
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::storage_group_uuid"               => $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid}, 
						"server::storage_groups::${storage_group_name}::used_by::${storage_key}" => $anvil->data->{server}{storage_groups}{$storage_group_name}{used_by}{$storage_key},
						"server::storage_groups::${storage_group_name}::storage_group_uuid"      => $anvil->data->{server}{storage_groups}{$storage_group_name}{storage_group_uuid}, 
					}});
				}
				
				if ($this_size > $anvil->data->{server}{drbd}{$resource}{$volume}{size})
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{size} = $this_size;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::size" => $anvil->Convert->add_commas({number => $anvil->data->{server}{drbd}{$resource}{$volume}{size}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{server}{drbd}{$resource}{$volume}{size}}).")", 
					}});
				}
			}
		}
	}
	
	# Make sure there is enough space on DR for the volumes under this VM.
	my $problem     = 0;
	my $config_file = "";
	foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{server}{storage_groups}})
	{
		my $storage_group_uuid = $anvil->data->{server}{storage_groups}{$storage_group_name}{storage_group_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			storage_group_name => $storage_group_name, 
			storage_group_uuid => $storage_group_uuid, 
		}});
		
		# First, is this SG on DR?
		if (not exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr_host_uuid})
		{
			# The DR host doesn't appear to be storage group.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0343", variables => { 
				host_name     => $dr_host_name,
				storage_group => $storage_group_name,
			}});
			$anvil->Job->update_progress({
				progress => 100,
				message  => "error_0343,!!host_name!".$dr_host_name."!!,!!storage_group!".$storage_group_name."!!",
			});
			$problem = 1;
		}
		
		my $space_needed = 0;
		foreach my $resource_key (sort {$a cmp $b} keys %{$anvil->data->{server}{storage_groups}{$storage_group_name}{used_by}})
		{
			my ($resource, $volume) =  ($resource_key =~ /^(.*)\/(\d+)$/);
			my $volume_size         =  $anvil->data->{server}{drbd}{$resource}{$volume}{size};
			   $space_needed        += $volume_size, 
			   $config_file         =  $anvil->data->{new}{resource}{$resource}{config_file};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				config_file  => $config_file, 
				resource_key => $resource_key, 
				resource     => $resource, 
				volume       => $volume, 
				volume_size  => $anvil->Convert->add_commas({number => $volume_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $volume_size}).")", 
				space_needed => $anvil->Convert->add_commas({number => $space_needed})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $space_needed}).")", 
			}});
			
		}
		
		# Is there enough space on DR?
		my $space_on_dr = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr_host_uuid}{vg_free};
		if ((not defined $space_on_dr) or ($space_on_dr !~ /^\d+$/))
		{
			# The DR host doesn't have a VG in the SG.
			my $variables = {
				space_needed       => $anvil->Convert->bytes_to_human_readable({'bytes' => $space_needed}), 
				space_needed_bytes => $anvil->Convert->add_commas({number => $space_needed}),
				storage_group      => $storage_group_name, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0468", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0344", 
				variables => $variables
			});
			$problem = 1;
		}
		else
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				space_on_dr  => $anvil->Convert->add_commas({number => $space_on_dr})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $space_on_dr}).")", 
				space_needed => $anvil->Convert->add_commas({number => $space_needed})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $space_needed}).")", 
			}});
			if ($space_needed > $space_on_dr)
			{
				my $variables = {
					space_needed       => $anvil->Convert->bytes_to_human_readable({'bytes' => $space_needed}), 
					space_needed_bytes => $anvil->Convert->add_commas({number => $space_needed}),
					storage_group      => $storage_group_name, 
					space_on_dr        => $anvil->Convert->bytes_to_human_readable({'bytes' => $space_on_dr}), 
					space_on_dr_bytes  => $anvil->Convert->add_commas({number => $space_on_dr}), 
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0344", variables => $variables});
				$anvil->Job->update_progress({
					progress  => 100,
					message   => "error_0344", 
					variables => $variables
				});
				$problem = 1;
			}
		}
	}
	if ($problem)
	{
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Have we already configured the DR? If so, what ports are already allocated.
	my $node1_to_dr_port         = "";
	my $node1_to_dr_port_inside  = "";
	my $node1_to_dr_port_outside = "";
	my $node2_to_dr_port         = "";
	my $node2_to_dr_port_inside  = "";
	my $node2_to_dr_port_outside = "";
	foreach my $host1_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{host1_to_host2}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host1_name => $host1_name }});
		foreach my $host2_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host2_name => $host2_name }});
			next if (($host1_name ne $dr_short_host_name) && ($host2_name ne $dr_short_host_name));
			
			if (($host1_name eq $node1_short_host_name) or ($host2_name eq $node1_short_host_name))
			{
				$node1_to_dr_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host1_tcp_port};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port => $node1_to_dr_port }});
				
				if (exists $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port})
				{
					$node1_to_dr_port_inside  = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port};
					$node1_to_dr_port_outside = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_tcp_port};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						node1_to_dr_port_inside  => $node1_to_dr_port_inside,
						node1_to_dr_port_outside => $node1_to_dr_port_outside, 
					}});
				}
			}
			else
			{
				$node2_to_dr_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host1_tcp_port};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port => $node2_to_dr_port }});
				
				if (exists $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port})
				{
					$node2_to_dr_port_inside  = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port};
					$node2_to_dr_port_outside = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_tcp_port};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						node2_to_dr_port_inside  => $node2_to_dr_port_inside,
						node2_to_dr_port_outside => $node2_to_dr_port_outside, 
					}});
				}
			}
		}
	}
	
	# Get net next pair of TCP ports, if needed.
	my $dr_tcp_ports     = 1;
	my $long_throw_ports = 0;
	if ($anvil->data->{switches}{protocol} eq "long-throw")
	{
		$dr_tcp_ports     = 0;
		$long_throw_ports = 1;
	}
	my (undef, $tcp_ports) = $anvil->DRBD->get_next_resource({
		debug            => 2,
		dr_tcp_ports     => $dr_tcp_ports, 
		long_throw_ports => $long_throw_ports, 
	});
	my @free_ports = split/,/, $tcp_ports;
	my $i          = 0;

	if ($node1_to_dr_port eq "")
	{
		$node1_to_dr_port = $free_ports[$i++];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port => $node1_to_dr_port }});
	}
	
	if (($long_throw_ports) && (not $node1_to_dr_port_inside))
	{
		$node1_to_dr_port_inside = $free_ports[$i++];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port_inside => $node1_to_dr_port_inside }});
	}
	if (($long_throw_ports) && (not $node1_to_dr_port_outside))
	{
		$node1_to_dr_port_outside = $free_ports[$i++];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port_outside => $node1_to_dr_port_outside }});
	}
	
	if ($node2_to_dr_port eq "")
	{
		$node2_to_dr_port = $free_ports[$i];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port => $node2_to_dr_port }});
	}
	if (($long_throw_ports) && (not $node2_to_dr_port_inside))
	{
		$node2_to_dr_port_inside = $free_ports[$i++];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port_inside => $node2_to_dr_port_inside }});
	}
	if (($long_throw_ports) && (not $node2_to_dr_port_outside))
	{
		$node2_to_dr_port_outside = $free_ports[$i++];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port_outside => $node2_to_dr_port_outside }});
	}
	
	# Show what we're doing
	my $variables = {
		protocol                 => $anvil->data->{switches}{protocol}, 
		node1_to_dr_port         => $node1_to_dr_port, 
		node1_to_dr_port_inside  => $node1_to_dr_port_inside, 
		node1_to_dr_port_outside => $node1_to_dr_port_outside, 
		node2_to_dr_port         => $node2_to_dr_port, 
		node2_to_dr_port_inside  => $node2_to_dr_port_inside, 
		node2_to_dr_port_outside => $node2_to_dr_port_outside, 
		config_file              => $config_file, 
	};
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0361", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 40,
		message   => "job_0361", 
		variables => $variables, 
	});
	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}})
	{
		foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}{$resource}})
		{
			my $variables = {
				resource => $resource, 
				volume   => $volume, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0362", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 43,
				message   => "job_0362", 
				variables => $variables, 
			});
			my $lv_size            = $anvil->data->{server}{drbd}{$resource}{$volume}{size};
			my $storage_group_uuid = $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid};
			my $dr_lv_name         = $resource."_".$volume;
			my $dr_vg_name         = $anvil->Storage->get_vg_name({
				debug              => 3, 
				storage_group_uuid => $storage_group_uuid, 
				host_uuid          => $dr_host_uuid,
			});
			my $dr_lv_path         = "/dev/".$dr_vg_name."/".$dr_lv_name;
			my $extent_size        = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr_host_uuid}{vg_extent_size};
			my $extent_count       = int($lv_size / $extent_size);
			my $shell_call         = $anvil->data->{path}{exe}{lvcreate}." -l ".$extent_count." -n ".$dr_lv_name." ".$dr_vg_name." -y";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"s01:resource"           => $resource, 
				"s02:volume"             => $volume, 
				"s03:lv_size"            => $anvil->Convert->add_commas({number => $lv_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $lv_size}).")", , 
				"s04:storage_group_uuid" => $storage_group_uuid, 
				"s05:dr_lv_name"         => $dr_lv_name, 
				"s06:dr_vg_name"         => $dr_vg_name, 
				"s07:dr_lv_path"         => $dr_lv_path, 
				"s08:extent_size"        => $anvil->Convert->add_commas({number => $extent_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $extent_size}).")", 
				"s09:extent_count"       => $extent_count, 
				"s10:shell_call"         => $shell_call, 
			}});
			
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lvcreate_call}      = $shell_call;
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lv_path}            = $dr_lv_path;
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{storage_group_uuid} = $storage_group_uuid;
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_tcp_port}      = $anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port}; 
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_minor}         = $anvil->data->{server}{drbd}{$resource}{$volume}{minor_number};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"server::dr::volumes::${resource}::${volume}::lvcreate_call"      => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lvcreate_call}, 
				"server::dr::volumes::${resource}::${volume}::lv_path"            => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lv_path}, 
				"server::dr::volumes::${resource}::${volume}::storage_group_uuid" => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{storage_group_uuid}, 
				"server::dr::volumes::${resource}::${volume}::drbd_tcp_port"      => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_tcp_port}, 
				"server::dr::volumes::${resource}::${volume}::drbd_minor"         => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_minor}, 
			}});
			
			# Get the VG name that this volume will be created on.
			$variables = {
				lv_path       => $dr_lv_path, 
				lv_size       => $anvil->Convert->bytes_to_human_readable({'bytes' => $lv_size}), 
				lv_size_bytes => $anvil->Convert->add_commas({number => $lv_size}), 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0363", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 47,
				message   => "job_0363", 
				variables => $variables, 
			});
		}
	}
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{Yes})
	{
		# If there's no config file, this might be being run from DR which doesn't have info about 
		# the resource yet.
		if ((not $config_file) && (not $anvil->data->{switches}{'job-uuid'}))
		{
			# Bail out.
			print $anvil->Words->string({key => "message_0308"})."\n";
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "protect=1\n";
		   $job_data .= "dr-host=".$anvil->data->{sys}{dr_host_name}."\n";
		   $job_data .= "protocol=".$anvil->data->{switches}{protocol}."\n";
		
		# We always record the job against node 1, as it has to use cluster info to run this, so we 
		# can't run it on the DR itself.
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_host_uuid => $node1_host_uuid }});
		
		# Register the job with this host
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $node1_host_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	### If we're still here, time to get started.
	# Read in the old config and update it.
	my $old_resource_config = $anvil->Storage->read_file({file => $config_file});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { old_resource_config => $old_resource_config }});
	
	# Pull the data out of the old file
	my $hosts          = "";
	my $nodes_tcp_port = "";
	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource => $resource }});
		my $dr_seen = 0;
		foreach my $this_host_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{host}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { this_host_name => $this_host_name }});
			my $node_id = "";
			if (($this_host_name eq $node1_short_host_name) or ($this_host_name eq $node1_host_name))
			{
				$node_id = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node_id => $node_id }});
				
				if ((not $nodes_tcp_port) && 
				    (exists $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}) && 
				    ($anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port}))
				{
					$nodes_tcp_port = $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodes_tcp_port => $nodes_tcp_port }});
				}
			}
			elsif (($this_host_name eq $node2_short_host_name) or ($this_host_name eq $node2_host_name))
			{
				$node_id = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node_id => $node_id }});
				
				if ((not $nodes_tcp_port) && 
				    (exists $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}) && 
				    ($anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port}))
				{
					$nodes_tcp_port = $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodes_tcp_port => $nodes_tcp_port }});
				}
			}
			elsif (($this_host_name eq $dr_short_host_name) or ($this_host_name eq $dr_host_name))
			{
				$node_id = 2;
				$dr_seen = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					node_id => $node_id,
					dr_seen => $dr_seen,
				}});
			}
			my $volumes = "";
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}{$resource}})
			{
				my $device_path  = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_path};
				my $backing_disk = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{backing_disk};
				my $device_minor = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_minor};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:device_path"  => $device_path,
					"s2:backing_disk" => $backing_disk,
					"s3:device_minor" => $device_minor,
				}});
				
				$volumes .= $anvil->Words->string({key => "file_0004", variables => {
					volume    => $volume,
					drbd_path => $device_path, 
					minor     => $device_minor, 
					lv_path   => $backing_disk, 
				        meta_data => "internal", 
				}});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volumes => $volumes }});
				
				# Record the DRBD device for adding DR.
				if (not exists $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{device_path})
				{
					$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{device_path} = $device_path;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::dr::volumes::${resource}::${volume}::device_path" => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{device_path},
					}});
				}
			}
			$hosts .= $anvil->Words->string({key => "file_0003", variables => {
				short_host_name => $this_host_name, 
				node_id         => $node_id, 
				volumes         => $volumes,
			}});
		}
		if (not $dr_seen)
		{
			# Inject the DR.
			my $volumes = "";
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{dr}{volumes}{$resource}})
			{
				my $device_path  = $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{device_path};
				my $backing_disk = $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lv_path};
				my $device_minor = $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_minor};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:device_path"  => $device_path,
					"s2:backing_disk" => $backing_disk,
					"s3:device_minor" => $device_minor,
				}});
				
				$volumes .= $anvil->Words->string({key => "file_0004", variables => {
					volume    => $volume,
					drbd_path => $device_path, 
					minor     => $device_minor, 
					lv_path   => $backing_disk, 
				        meta_data => "internal", 
				}});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volumes => $volumes }});
			}
			$hosts .= $anvil->Words->string({key => "file_0003", variables => {
				short_host_name => $dr_short_host_name,
				node_id         => "2", 
				volumes         => $volumes,
			}});
		}
	}
	
	# Refresh the IP info (usually scrubbed by this point)
	$anvil->Database->get_ip_addresses();
	
	# The connections. Node 1 to 2 always uses the BCN, Either node to DR needs 
	my $storage_network = "sn1";
	my $dr_network      = $anvil->data->{lookup}{host_uuid}{$dr_host_uuid}{network}{use_network};
	my $dr_ip           = $anvil->data->{lookup}{host_uuid}{$dr_host_uuid}{network}{use_ip};
	my $node1_sn_ip     = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{network}{$storage_network}{ip_address};
	my $node1_dr_ip     = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{network}{$dr_network}{ip_address};
	my $node2_sn_ip     = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{network}{$storage_network}{ip_address};
	my $node2_dr_ip     = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{network}{$dr_network}{ip_address};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's01:storage_network' => $storage_network,
		's02:dr_host_uuid'    => $dr_host_uuid,
		's03:dr_network'      => $dr_network, 
		's04:dr_ip'           => $dr_ip,
		's05:node1_host_uuid' => $node1_host_uuid, 
		's06:node1_sn_ip'     => $node1_sn_ip,
		's07:node1_dr_ip'     => $node1_dr_ip,
		's08:node2_host_uuid' => $node2_host_uuid, 
		's09:node2_sn_ip'     => $node2_sn_ip, 
		's10:node2_dr_ip'     => $node2_dr_ip, 
	}});
	
	# Choose the DR protocol
	my $use_drbd_proxy = 0;
	my $dr_protocol    = "A";
	if ($anvil->data->{switches}{protocol} eq "sync")
	{
		$dr_protocol = "C";
	}
	elsif ($anvil->data->{switches}{protocol} eq "long-throw")
	{
		$use_drbd_proxy = 1;
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		dr_protocol    => $dr_protocol,
		use_drbd_proxy => $use_drbd_proxy, 
	}});

	# Node 1 to Node 2 first
	my $proxy      = "";
	my $max_c_rate = 500;
	my $file_key   = "file_0005";
	if ($use_drbd_proxy)
	{
		$proxy      = $anvil->Words->string({key => "file_0008", variables => { memlimit => 256 }});
		$max_c_rate = 100;
		$file_key   = "file_0007";
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		proxy      => $proxy, 
		max_c_rate => $max_c_rate,
		file_key   => $file_key, 
	}});
	
	# Node 1 to 2
	my $connections = $anvil->Words->string({key => "file_0005", variables => {
		host1_short_name => $node1_short_host_name, 
		host1_ip         => $node1_sn_ip, 
		host2_short_name => $node2_short_host_name, 
		host2_ip         => $node2_sn_ip, 
		tcp_port         => $nodes_tcp_port, 
		'c-rate-maximum' => $max_c_rate,
		protocol         => "C",
		fencing          => "resource-and-stonith"
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { connections => $connections }});
	
	# Node 1 to DR
	$connections .= $anvil->Words->string({key => $file_key, variables => {
		host1_short_name => $node1_short_host_name, 
		host1_ip         => $node1_dr_ip, 
		host2_short_name => $dr_short_host_name,
		host2_ip         => $dr_ip,
		tcp_port         => $node1_to_dr_port, 
		inside_tcp_port  => $node1_to_dr_port_inside,
		outside_tcp_port => $node1_to_dr_port_outside,
		'c-rate-maximum' => $max_c_rate,
		protocol         => $dr_protocol,
		fencing          => "dont-care"
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { connections => $connections }});

	# Node 2 to DR
	$connections .= $anvil->Words->string({key => $file_key, variables => {
		host1_short_name => $node2_short_host_name, 
		host1_ip         => $node2_dr_ip, 
		host2_short_name => $dr_short_host_name,
		host2_ip         => $dr_ip,
		tcp_port         => $node2_to_dr_port, 
		inside_tcp_port  => $node2_to_dr_port_inside,
		outside_tcp_port => $node2_to_dr_port_outside,
		'c-rate-maximum' => $max_c_rate,
		protocol         => $dr_protocol,
		fencing          => "dont-care"
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { connections => $connections }});
	
	my $new_resource_config = $anvil->Words->string({key => "file_0006", variables => {
		server      => $server_name,
		proxy       => $proxy, 
		hosts       => $hosts,
		connections => $connections,
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_resource_config => $new_resource_config }});
	
	# Is the new res file the same as the old one?
	my $difference = diff \$old_resource_config, \$new_resource_config, { STYLE => 'Unified' };
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
	
	# Is there a difference on the local version? There might be a difference on peers, but we'll deel 
	# with that below.
	if (not $difference)
	{
		# The resource file doesn't need to be updated.
		my $variables = {
			file => $config_file, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0364", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 50,
			message   => "job_0364", 
			variables => $variables, 
		});
	}
	else
	{
		# Write out a test file.
		my $test_file = $anvil->data->{path}{directories}{temp}."/test-".$server_name.".res";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }});
		my ($problem) = $anvil->Storage->write_file({
			debug     => 2,
			backup    => 0,
			overwrite => 1,
			file      => $test_file, 
			body      => $new_resource_config,
			user      => "root", 
			group     => "root", 
			mode      => "0644",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		# Validate.
		my $shell_call = $anvil->data->{path}{exe}{drbdadm}." --config-to-test ".$test_file." --config-to-exclude ".$config_file." sh-nop";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
		if ($return_code)
		{
			# Something went wrong.
			my $variables = {
				return_code => $return_code, 
				output      => $output, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0345", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0345", 
				variables => $variables, 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Remove the test file.
		unlink $test_file;
		
		# Backup the res file so we can tell the user where the current config was backed up to in 
		# case they need to restore it.
		my ($backup_file) = $anvil->Storage->backup({file => $config_file});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});
		my $variables = { backup_file => $backup_file };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0365", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 60,
			message   => "job_0365", 
			variables => $variables, 
		});
		
		# Write out the new file.
		($problem) = $anvil->Storage->write_file({
			debug     => 2,
			backup    => 0,
			overwrite => 1,
			file      => $config_file, 
			body      => $new_resource_config,
			user      => "root", 
			group     => "root", 
			mode      => "0644",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0366"});
		$anvil->Job->update_progress({
			progress  => 65,
			message   => "job_0366", 
		});
		
		# Call 'drbdadm dump-xml' to check that it's OK.
		($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{drbdadm}." dump-xml"});
		if ($return_code)
		{
			# Something went wrong.
			my $variables = {
				return_code => $return_code, 
				output      => $output, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0345", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 70,
				message   => "error_0345", 
				variables => $variables, 
			});
			
			# Restoring the old config now.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0346"});
			$anvil->Job->update_progress({
				progress => 75,
				message  => "error_0346", 
			});
			
			# Backup the bad file and worn the user.
			my ($backup_file) = $anvil->Storage->backup({file => $config_file});
			$variables = { file => $backup_file };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0347", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 80,
				message   => "error_0347", 
				variables => $variables, 
			});
			
			# Write out the new file.
			my ($problem) = $anvil->Storage->write_file({
				debug     => 2,
				backup    => 1,
				overwrite => 1,
				file      => $config_file, 
				body      => $old_resource_config, 
				user      => "root", 
				group     => "root", 
				mode      => "0644",
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});

			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0348"});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0348", 
				variables => $variables, 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# New config is good! Update the file on the peers.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0367"});
		$anvil->Job->update_progress({
			progress => 70,
			message  => "job_0367", 
		});
	}
	
	# New config is good! Update the file on the peers.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0368"});
	$anvil->Job->update_progress({
		progress => 72,
		message  => "job_0368", 
	});
	
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host. It's not uncommon for the DR host to 
		# not have a connection over the SN or even the BCN. So we'll use the IFN1 to move files.
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = {
			file       => $config_file, 
			host_name  => $peer_host_name, 
			ip_address => $peer_ip, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0369", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 75,
			message   => "job_0369", 
			variables => $variables, 
		});
		my ($problem) = $anvil->Storage->write_file({
			debug     => 2,
			backup    => 1,
			overwrite => 1,
			file      => $config_file, 
			body      => $new_resource_config, 
			user      => "root", 
			group     => "root", 
			mode      => "0644",
			target    => $peer_ip, 
			password  => $anvil_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		# Make sure the file exists now.
		my $check_resource_config = $anvil->Storage->read_file({
			file     => $config_file,
			target   => $peer_ip, 
			password => $anvil_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { check_resource_config => $check_resource_config }});
		
		my $difference = diff \$new_resource_config, \$check_resource_config, { STYLE => 'Unified' };
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
		
		# Failed to write the file.
		if ($difference)
		{
			$variables = {
				host_name             => $peer_host_name, 
				file                  => $config_file, 
				difference            => $difference, 
				new_resource_config   => $new_resource_config, 
				check_resource_config => $check_resource_config, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0405", variables => $variables});
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "job_0405", 
				variables  => $variables, 
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	# Immediately call scan-drbd on all machines to ensure that if another run is about to happen for a 
	# different server, it knows the used ports list is updated.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0381"});
	$anvil->Job->update_progress({
		progress => 76,
		message  => "job_0381", 
	});
	
	my $scan_drbd_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd --force ".$anvil->Log->switches();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { scan_drbd_call => $scan_drbd_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $scan_drbd_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0382", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 77,
			message   => "job_0382", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $scan_drbd_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Create the LV and MD on DR.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0370"});
	$anvil->Job->update_progress({
		progress  => 78,
		message   => "job_0370", 
	});
	my $create_md = 0;
	foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}{$server_name}})
	{
		my $dr_ip   = $anvil->data->{lookup}{host_uuid}{$dr_host_uuid}{network}{use_ip};
		my $lv_path = $anvil->data->{server}{dr}{volumes}{$server_name}{$volume}{lv_path};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:volume'  => $volume,
			's2:dr_ip'   => $dr_ip,
			's3:lv_path' => $lv_path, 
		}});
		
		my $variables = { 
			volume  => $volume, 
			lv_path => $lv_path, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0371", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 80,
			message   => "job_0371", 
			variables => $variables, 
		});
		
		my $lv_check_call = "
if [ -e '".$lv_path."' ]; 
then 
    echo exists;
else 
    echo create;
fi";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { lv_check_call => $lv_check_call }});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $dr_ip,
			password   => $anvil_password,
			shell_call => $lv_check_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		if ($output eq "exists")
		{
			my $variables = { lv_path => $lv_path };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0372", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 80,
				message   => "job_0372", 
				variables => $variables, 
			});
			next;
		}
		
		# Create the LV.
		my $lvcreate_call = $anvil->data->{server}{dr}{volumes}{$server_name}{$volume}{lvcreate_call};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { lvcreate_call => $lvcreate_call }});
		($output, $error, $return_code) = $anvil->Remote->call({
			target     => $dr_ip,
			password   => $anvil_password,
			shell_call => $lvcreate_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		
		sleep 1;
		# Does it exist now?
		($output, $error, $return_code) = $anvil->Remote->call({
			target     => $dr_ip,
			password   => $anvil_password,
			shell_call => $lv_check_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		if ($output eq "create")
		{
			my $variables = { lv_path => $lv_path };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0349", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0349", 
				variables => $variables, 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Create the DRBD meta data now.
		$create_md = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_md => $create_md }});
	}
	
	if ($create_md)
	{
		my $dr_ip        = $anvil->data->{lookup}{host_uuid}{$dr_host_uuid}{network}{use_ip};
		my $drbd_md_call = $anvil->data->{path}{exe}{drbdadm}." --force create-md --max-peers=3 ".$server_name;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:dr_ip'        => $dr_ip,
			's2:drbd_md_call' => $drbd_md_call, 
		}});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $dr_ip,
			password   => $anvil_password,
			shell_call => $drbd_md_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Reload the config.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0373"});
	$anvil->Job->update_progress({
		progress => 85,
		message  => "job_0373", 
	});
	my $shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = { 
			server    => $server_name,
			host_name => $peer_host_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0374", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 85,
			message   => "job_0374", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $shell_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# If the resource is down, bring it up.
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0375", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 88,
		message   => "job_0375", 
		variables => $variables, 
	});
	my $drbd_up_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." || ".$anvil->data->{path}{exe}{drbdadm}." up ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_up_call => $drbd_up_call }});
	($output, $return_code) = $anvil->System->call({shell_call => $drbd_up_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		$variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0376", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 90,
			message   => "job_0376", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $drbd_up_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Now watch until the DR host shows up
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0377"});
	$anvil->Job->update_progress({
		progress => 92,
		message  => "job_0377", 
	});
	my $waiting = 1;
	while($waiting)
	{
		sleep 5;
		$anvil->DRBD->gather_data({debug => 2});
		
		my $dr_seen = 1;
		foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
			if (exists $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr_short_host_name})
			{
				my $local_role       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr_short_host_name}{local_role};
				my $local_disk_state = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr_short_host_name}{local_disk_state};
				my $peer_role        = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr_short_host_name}{peer_role};
				my $peer_disk_state  = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr_short_host_name}{peer_disk_state};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					local_role       => $local_role,
					local_disk_state => $local_disk_state, 
					peer_role        => $peer_role, 
					peer_disk_state  => $peer_disk_state, 
				}});
			}
			else
			{
				# Not up yet.
				my $next_check = $anvil->Get->date_and_time({offset => 5, time_only => 1});
				my $variables  = { next_check => $next_check };
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0378", variables => $variables});
				$anvil->Job->update_progress({
					progress  => 95,
					message   => "job_0378", 
					variables => $variables, 
				});
				$dr_seen = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_seen => $dr_seen }});
				last;
			}
		}
		
		if ($dr_seen)
		{
			# We're ready.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0379"});
			$anvil->Job->update_progress({
				progress => 98,
				message  => "job_0379", 
			});
			$waiting = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
		}
	}
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0380", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0380", 
		variables => $variables, 
	});
	
	return(0);
}

sub load_job
{
	my ($anvil) = @_;
	
	$anvil->Job->clear();
	$anvil->Job->get_job_details();
	$anvil->Job->update_progress({
		progress         => 1,
		job_picked_up_by => $$, 
		job_picked_up_at => time, 
		message          => "message_0267", 
	});
	
	print "Loading the job: [".$anvil->data->{switches}{'job-uuid'}."]...\n";
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"jobs::job_command"  => $anvil->data->{jobs}{job_command},
		"jobs::job_data"     => $anvil->data->{jobs}{job_data}, 
		"jobs::job_progress" => $anvil->data->{jobs}{job_progress}, 
		"jobs::job_status"   => $anvil->data->{jobs}{job_status}, 
	}});
	
	# Break up the job data into switches.
	$anvil->data->{switches}{Yes} = 1;
	print "- Set the switch: [--Yes] to true.\n";
	foreach my $line (split/\n/, $anvil->data->{jobs}{job_data})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
		if ($line =~ /(.*?)=(.*)$/)
		{
			my $key   = $1;
			my $value = $2;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				key   => $key,
				value => $value, 
			}});
			
			$anvil->data->{switches}{$key} = $value;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"switches::${key}" => $anvil->data->{switches}{$key}, 
			}});
			
			print "* Set the switch: [--".$key."] to: [".$value."]\n";
		}
	}
	print "Job loaded successfully.\n\n";
	
	return(0);
}

sub check_for_duplicate_dr_links
{
	my ($anvil) = @_;

	if (exists $anvil->data->{dr_links})
	{
		delete $anvil->data->{dr_links};
	}
	my $query = "
SELECT
    dr_link_uuid,
    dr_link_host_uuid,
    dr_link_anvil_uuid,
    dr_link_note
FROM
    dr_links
ORDER BY
    modified_date DESC;";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
	my $count   = @{$results};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
		results => $results,
		count   => $count,
	}});
	foreach my $row (@{$results})
	{
		my $dr_link_uuid       = $row->[0];
		my $dr_link_host_uuid  = $row->[1];
		my $dr_link_anvil_uuid = $row->[2];
		my $dr_link_note       = $row->[3];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
			dr_link_uuid       => $dr_link_uuid,
			dr_link_host_uuid  => $dr_link_host_uuid,
			dr_link_anvil_uuid => $dr_link_anvil_uuid,
			dr_link_note       => $dr_link_note,
		}});

		if (exists $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_note})
		{
			# Duplicate!
			my $previous_note = $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_note};
			my $this_note     = $dr_link_note;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
				previous_note => $previous_note,
				this_note     => $this_note,
			}});

			# If the previous note was "DELETED", delete it. Otherwise we'll delete this one.
			if ($previous_note eq "DELETED")
			{
				# Delete the previous one.
				my $old_dr_link_uuid = $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_uuid};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_dr_link_uuid => $old_dr_link_uuid }});

				my $queries = [];
				push @{$queries}, "DELETE FROM history.dr_links WHERE dr_link_uuid = ".$anvil->Database->quote($old_dr_link_uuid).";";
				push @{$queries}, "DELETE FROM dr_links WHERE dr_link_uuid = ".$anvil->Database->quote($old_dr_link_uuid).";";
				foreach my $query (@{$queries})
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
				}
				$anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__});

				# Update the local record to store this one.
				$anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_uuid} = $dr_link_uuid;
				$anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_note} = $dr_link_note;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
					"dr_links::host_uuid::${dr_link_host_uuid}::${dr_link_anvil_uuid}::dr_link_uuid" => $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_uuid},
					"dr_links::host_uuid::${dr_link_host_uuid}::${dr_link_anvil_uuid}::dr_link_note" => $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_note},
				}});
			}
			else
			{
				# Delete this one.
				my $queries = [];
				push @{$queries}, "DELETE FROM history.dr_links WHERE dr_link_uuid = ".$anvil->Database->quote($dr_link_uuid).";";
				push @{$queries}, "DELETE FROM dr_links WHERE dr_link_uuid = ".$anvil->Database->quote($dr_link_uuid).";";
				foreach my $query (@{$queries})
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
				}
				$anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__});
			}
		}
		else
		{
			$anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_uuid} = $dr_link_uuid;
			$anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_note} = $dr_link_note;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
				"dr_links::host_uuid::${dr_link_host_uuid}::${dr_link_anvil_uuid}::dr_link_uuid" => $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_uuid},
				"dr_links::host_uuid::${dr_link_host_uuid}::${dr_link_anvil_uuid}::dr_link_note" => $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_note},
			}});
		}
	}

	# Done, reload dr_links.
	delete $anvil->data->{dr_links};
	$anvil->Database->get_dr_links();
}

sub show_servers
{
	my ($anvil, $anvil_uuid) = @_;

	my $server_count = keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_count => $server_count }});
	if ($server_count > 0)
	{
		print $anvil->Words->string({key => "error_0413"})."\n";
		foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
		{
			my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
			print "- ".$server_name." (".$server_uuid.")\n";
		}
		print "\n";
	}
}