#!/usr/bin/perl
# 
# This manages if a server is backed up to a DR host or not. When enabled, it can start or stop replication.
# 
# NOTE: Unlike most jobs, this one will directly work on the peer node and the DR host using SSH connections.
#       This behaviour is likely to change later as it's not ideal.
# 
# TODO: --remove is not yet implemented, this needs to be done. Use anvil-delete-server for methods to delete.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Any problem that causes an early exit.
# 

use strict;
use warnings;
use Anvil::Tools;
require POSIX;
use Term::Cap;
use Text::Diff;
use Data::Dumper;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

$anvil->Get->switches({list => ["connect", "disconnect", "job-uuid", "protect", "protocol", "remove", "server", "update", "Yes"], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0306"});
	sleep 10;
	$anvil->nice_exit({exit_code => 1});
}

# If we've got a job UUID, load the job details.
if ($anvil->data->{switches}{'job-uuid'})
{
	load_job($anvil);
}

my $terminal = "";
if (not $anvil->data->{switches}{'job-uuid'})
{
	my $termios = new POSIX::Termios;
	$termios->getattr;
	my $ospeed   = $termios->getospeed;
	$terminal = Tgetent Term::Cap { TERM => undef, OSPEED => $ospeed };
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { terminal => $terminal }});
	
	$terminal->Trequire(qw/ce ku kd/);
	print $terminal->Tputs('cl');
}

sanity_check($anvil, $terminal);


$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

sub sanity_check
{
	my ($anvil, $terminal) = @_;
	
	# Begin sanity checks
	$anvil->Job->update_progress({
		progress => 10,
		message  => "job_0358", 
	});
	
	# Are we a node or DR?
	my $host_type  = $anvil->Get->host_type();
	my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		host_type  => $host_type,
		anvil_uuid => $anvil_uuid, 
	}});
	
	# If we're doing a --protect or --remove, make sure we're a node, the cluster is up, and both nodes 
	# are ready.
	if (($anvil->data->{switches}{protect}) or ($anvil->data->{switches}{remove}))
	{
		# Make sure we're in an Anvil! (Node or DR Host)
		if (not $anvil_uuid)
		{
			# This must be run on a node active in the cluster  hosting the server being managed.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0332"});
			$anvil->Job->update_progress({
				progress => 100,
				message  => "error_0332", 
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# If we're a node, make sure we're in a cluster.
		if ($host_type eq "node")
		{
			# Can we parse the CIB?
			my ($problem) = $anvil->Cluster->parse_cib();
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
			if ($problem)
			{
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0336"});
				$anvil->Job->update_progress({
					progress => 0,
					message  => "error_0336", 
					job_status => "failed", 
				});
				$anvil->nice_exit({exit_code => 1});
				$anvil->nice_exit({exit_code => 1});
			}
			
			# Both nodes need to be in the cluster, are they?
			if (not $anvil->data->{cib}{parsed}{'local'}{ready})
			{
				# We're not a full member of the cluster yet. Please try again once we're fully in. Exiting.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0337"});
				$anvil->Job->update_progress({
					progress => 0,
					message  => "error_0337", 
					job_status => "failed", 
				});
				$anvil->nice_exit({exit_code => 1});
			}
		}
		
		if ($anvil->data->{switches}{protect})
		{
			if (not $anvil->data->{switches}{protocol})
			{
				$anvil->data->{switches}{protocol} = "async";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					'switches::protocol' => $anvil->data->{switches}{protocol},
				}});
			}
			elsif (($anvil->data->{switches}{protocol} ne "sync")  && 
			       ($anvil->data->{switches}{protocol} ne "async") && 
			       ($anvil->data->{switches}{protocol} ne "long-throw"))
			{
				# The protocol is invalid. Please use '--help' for more information.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0342", variables => { protocol => $anvil->data->{switches}{protocol} }});
				$anvil->Job->update_progress({
					progress => 100,
					message  => "error_0341,!!protocol!".$anvil->data->{switches}{protocol}."!!", 
					job_status => "failed", 
				});
				$anvil->nice_exit({exit_code => 1});
			}
		}
	}
	
	# Get the Anvil! details.
	$anvil->Database->get_hosts();
	$anvil->Database->get_anvils();
	$anvil->Database->get_storage_group_data();
	
	# Does this Anvil! have a DR node?
	if (not $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid})
	{
		# This Anvil! does not seem to have a DR host. Exiting.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "error_0333"});
		$anvil->Job->update_progress({
			progress => 100,
			message  => "error_0333", 
			job_status => "failed", 
		});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Can we access DR, if we're not the DR host?
	$anvil->Database->get_ip_addresses({debug => 2});
	my $password      = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
	my $dr1_host_name = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name};
	my $dr_ip         = $anvil->System->find_matching_ip({
		debug => 2,
		host  => $dr1_host_name,
	});
	if ($dr1_host_uuid ne $anvil->Get->host_uuid)
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			password      => $anvil->Log->is_secure($password),
			dr1_host_uuid => $dr1_host_uuid,
			dr1_host_name => $dr1_host_name, 
			dr_ip         => $dr_ip, 
		}});
		if ((not $dr_ip) or ($dr_ip eq "!!error!!"))
		{
			# Failed to find an IP we can access the DR host. Has it been configured? Is it running? Exiting.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "error_0334", variables => { host_name => $dr1_host_name }});
			$anvil->Job->update_progress({
				progress => 0,
				message  => "error_0334,!!host_name!".$dr1_host_name."!!", 
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Test access.
		my $access = $anvil->Remote->test_access({
			target   => $dr_ip,
			password => $password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
		if (not $access)
		{
			# Failed to access the DR host. Is it running? Exiting.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0335", variables => { 
				host_name  => $dr1_host_name,
				ip_address => $dr_ip, 
			}});
			$anvil->Job->update_progress({
				progress => 0,
				message  => "error_0335,!!host_name!".$dr1_host_name."!!,!!ip_address!".$dr_ip."!!", 
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		### TODO: We can queue a job to update the peer later, there's no real need, in the long run, for the
		###       peer to be online.
		# If we're protecting or removing a server from DR, the peer needs to be up.
		if ((($anvil->data->{switches}{protect})   or 
		($anvil->data->{switches}{remove})    or 
		($anvil->data->{switches}{protocol})) && 
		(not $anvil->data->{cib}{parsed}{peer}{ready}))
		{
			if ($anvil->data->{switches}{protect})
			{
				# We can't setup a server to be protected unless both nodes are up, and the peer 
				# isn't at this time.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0338"});
				$anvil->Job->update_progress({
					progress => 0,
					message  => "error_0338", 
					job_status => "failed", 
				});
			}
			else
			{
				# We can't remove a server from DR unless both nodes are up, and the peer isn't at 
				# this time.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0339"});
				$anvil->Job->update_progress({
					progress => 0,
					message  => "error_0339", 
					job_status => "failed", 
				});
			}
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	# Verify we found the server.
	$anvil->data->{server}{'server-name'} = "";
	$anvil->data->{server}{'server-uuid'} = "";
	$anvil->data->{server}{'anvil-uuid'}  = $anvil_uuid;
	if (not $anvil->data->{switches}{server})
	{
		# Please specify the server to manager using '--server <name or uuid>'.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0340"});
		$anvil->Job->update_progress({
			progress => 100,
			message  => "error_0340", 
			job_status => "failed", 
		});
		$anvil->nice_exit({exit_code => 1});
	}
	else
	{
		my $server = $anvil->data->{switches}{server};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server => $server }});
		
		$anvil->Database->get_servers();
		if (exists $anvil->data->{servers}{server_uuid}{$server})
		{
			$anvil->data->{server}{'server-uuid'} = $server;
			$anvil->data->{server}{'server-name'} = $anvil->data->{servers}{server_uuid}{$server}{server_name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				'server::server-uuid' => $anvil->data->{server}{'server-uuid'},
				'server::server-name' => $anvil->data->{server}{'server-name'},
			}});
		}
		if (exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server})
		{
			$anvil->data->{server}{'server-name'} = $server;
			$anvil->data->{server}{'server-uuid'} = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server}{server_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				'server::server-uuid' => $anvil->data->{server}{'server-uuid'},
				'server::server-name' => $anvil->data->{server}{'server-name'},
			}});
		}
	}
	
	# Get and parse the server's definition to find the DRBD devices.
	if ((not $anvil->data->{server}{'server-uuid'}) or (not $anvil->data->{server}{'server-name'}))
	{
		# Failed to find the server by name or UUID.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0341", variables => { server => $anvil->data->{switches}{server} }});
		$anvil->Job->update_progress({
			progress => 100,
			message  => "error_0341,!!server!".$anvil->data->{switches}{server}."!!", 
			job_status => "failed", 
		});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Are we being asked to actuall do something?
	if (((not $anvil->data->{switches}{'connect'})  && 
	     (not $anvil->data->{switches}{disconnect}) && 
	     (not $anvil->data->{switches}{protect})    && 
	     (not $anvil->data->{switches}{remove})     && 
	     (not $anvil->data->{switches}{update}))    or 
	     ($anvil->data->{switches}{help})           or 
	     ($anvil->data->{switches}{h}))
	{
		# Show the man page.
		system($anvil->data->{path}{exe}{man}." ".$THIS_FILE);
		if (($anvil->data->{switches}{help}) or ($anvil->data->{switches}{h}))
		{
			$anvil->nice_exit({exit_code => 0});
		}
		else
		{
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	# Sanity checks complete!
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0359"});
	$anvil->Job->update_progress({
		progress => 20,
		message  => "job_0359", 
	});
	
	# If we've got a got job-uuid, wait for / make sure all machines are online.
	if ($anvil->data->{switches}{'job-uuid'})
	{
		# Loop until we have access to both the peer machines.
		my $waiting  = 1;
		my $wait_for = 10;
		while ($waiting)
		{
			# This will get set to 1 if we have to keep waiting.
			$waiting = 0;
			my $password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
			my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
			my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
			my $dr1_host_uuid   = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				password        => $anvil->Log->is_secure($password),
				node1_host_uuid => $node1_host_uuid,
				node2_host_uuid => $node2_host_uuid, 
				dr1_host_uuid   => $dr1_host_uuid, 
			}});
			foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid)
			{
				next if $this_host_uuid eq $anvil->Get->host_uuid();
				my $this_host_name = $anvil->Get->host_name_from_uuid({host_uuid => $this_host_uuid});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					this_host_uuid => $this_host_uuid,
					this_host_name => $this_host_name,
				}});
				
				# We'll try the SN, then the BCN and finally the IFN to see which, if any, network we
				# can reach the peer on. This is needed because the DR host could be on a totally 
				# different network.
				$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip}      = "";
				$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_subnet}  = "";
				$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_network} = "";
				foreach my $check_network ("sn", "bcn", "mn", "ifn")
				{
					last if $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { check_network => $check_network }});
					foreach my $this_network (sort {$a cmp $b} keys %{$anvil->data->{hosts}{host_uuid}{$this_host_uuid}{network}})
					{
						next if $this_network !~ /^$check_network/;
						my $this_ip     = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{network}{$this_network}{ip_address};
						my $this_subnet = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{network}{$this_network}{subnet_mask};
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							's1:this_ip'      => $this_ip,
							's2:this_network' => $this_network, 
							's3:this_subnet'  => $this_subnet,
						}});
						
						# Test access.
						my $access = $anvil->Remote->test_access({
							target   => $this_ip,
							password => $password,
						});
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
						if ($access)
						{
							$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip}      = $this_ip;
							$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_subnet}  = $this_subnet;
							$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_network} = $this_network;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								"hosts::host_uuid::${this_host_uuid}::network::use_ip"      => $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip},
								"hosts::host_uuid::${this_host_uuid}::network::use_subnet"  => $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_subnet},
								"hosts::host_uuid::${this_host_uuid}::network::use_network" => $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_network},
							}});
						}
					}
				}
				
				if (not $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip})
				{
					# No access
					my $variables = {
						waiting   => $wait_for,
						host_name => $this_host_name,
					};
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0404", variables => $variables});
					$anvil->Job->update_progress({
						progress  => 12,
						message   => "job_0404", 
						variables => $variables, 
					});
					
					$waiting = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				}
			}
			
			# If we're waiting for a peer, record as such.
			sleep $wait_for;
		}
	}

	# If we're protecting, make sure there's enough space on the DR host.
	if ($anvil->data->{switches}{protect})
	{
		process_protect($anvil, $terminal);
	}
	elsif ($anvil->data->{switches}{remove})
	{
		process_remove($anvil, $terminal);
	}
	elsif ($anvil->data->{switches}{'connect'})
	{
		process_connect($anvil, $terminal);
	}
	elsif ($anvil->data->{switches}{disconnect})
	{
		process_disconnect($anvil, $terminal);
	}
	elsif ($anvil->data->{switches}{update})
	{
		process_update($anvil, $terminal);
	}
	
	return(0);
}

sub process_update
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr1_host_uuid         = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
	my $dr1_host_name         = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name};
	my $dr1_short_host_name   = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr1_host_uuid         => $dr1_host_uuid, 
		dr1_host_name         => $dr1_host_name, 
		dr1_short_host_name   => $dr1_short_host_name, 
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
	}});
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{Yes})
	{
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0395", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 25,
			message   => "job_0395", 
			variables => $variables, 
		});
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "update=1\n";
		
		# Register the job with the DR host.
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $dr1_host_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	# If the resource is down, bring it up.
	my $variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0387", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 50,
		message   => "job_0387", 
		variables => $variables, 
	});
	
	# Bring up the connection locally, and then also bring up the connection on the nodes, in case the 
	# server is down.
	my $drbd_up_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." || ".$anvil->data->{path}{exe}{drbdadm}." up ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_up_call => $drbd_up_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_up_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0388", variables => $variables});
	$anvil->Job->update_progress({
		progress => 60,
		message  => "job_0388", 
	});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid)
	{
		# "Peer" in this context is either/both nodes
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0389", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 70,
			message   => "job_0389", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $drbd_up_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Now watch until out volume(s) are UpToDate
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0390"});
	$anvil->Job->update_progress({
		progress => 70,
		message  => "job_0390", 
	});
	
	my $waiting = 1;
	while ($waiting)
	{
		$anvil->DRBD->gather_data({debug => 2});
		   $waiting      = 0;
		my $time_to_sync = "";
		my $sync_speed   = "";
		my $sync_source  = "";
		foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
			foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}})
			{
				my $replication_speed      = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{replication_speed};
				my $peer_role              = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_role};
				my $peer_disk_state        = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_disk_state};
				my $estimated_time_to_sync = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{estimated_time_to_sync};
				my $local_disk_state       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_disk_state};
				my $connection_state       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{connection_state};
				my $local_role             = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_role};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					peer                   => $peer,
					replication_speed      => $replication_speed, 
					peer_role              => $peer_role, 
					peer_disk_state        => $peer_disk_state, 
					estimated_time_to_sync => $estimated_time_to_sync, 
					local_disk_state       => $local_disk_state, 
					connection_state       => $connection_state, 
					local_role             => $local_role, 
				}});
				
				if ($connection_state eq "synctarget")
				{
					$sync_source = $peer;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sync_source => $sync_source }});
				}
				if ($replication_speed)
				{
					$sync_speed = $anvil->Convert->bytes_to_human_readable({'bytes' => $replication_speed});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sync_speed => $sync_speed }});
				}
				if ($estimated_time_to_sync)
				{
					$time_to_sync = $anvil->Convert->time({'time' => $estimated_time_to_sync, translate => 1});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { time_to_sync => $time_to_sync }});
				}
				
				if ($local_disk_state ne "uptodate")
				{
					$waiting = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				}
			}
		}
		if ($waiting)
		{
			if ($sync_source)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					sync_source  => $sync_source, 
					sync_speed   => $sync_speed, 
					time_to_sync => $time_to_sync, 
				}});
				my $variables = { 
					sync_source  => $sync_source,
					sync_speed   => $sync_speed, 
					time_to_sync => $time_to_sync, 
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0396", variables => $variables});
				$anvil->Job->update_progress({
					progress  => 80,
					message   => "job_0396", 
					variables => $variables, 
				});
			}
			else
			{
				$anvil->Job->update_progress({
					progress  => 75,
					message   => "job_0398", 
					variables => $variables, 
				});
			}
			sleep 5;
		}
	}
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0397", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 90,
		message   => "job_0397", 
		variables => $variables, 
	});
	
	# Bring down the connection.
	my $drbd_down_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." && ".$anvil->data->{path}{exe}{drbdadm}." down ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_down_call => $drbd_down_call }});
	($output, $return_code) = $anvil->System->call({shell_call => $drbd_down_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0391", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0391", 
		variables => $variables, 
	});
	
	return(0);
}

sub process_disconnect
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr1_host_uuid         = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
	my $dr1_host_name         = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name};
	my $dr1_short_host_name   = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr1_host_uuid         => $dr1_host_uuid, 
		dr1_host_name         => $dr1_host_name, 
		dr1_short_host_name   => $dr1_short_host_name, 
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
	}});
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{Yes})
	{
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0392", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 25,
			message   => "job_0392", 
			variables => $variables, 
		});
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "disconnect=1\n";
		
		# Register the job with the DR host.
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $dr1_host_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	# If the resource is up, bring it down.
	my $variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0393", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 50,
		message   => "job_0393", 
		variables => $variables, 
	});
	
	# Bring up the connection locally, and then also bring up the connection on the nodes, in case the 
	# server is down.
	my $drbd_down_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." && ".$anvil->data->{path}{exe}{drbdadm}." down ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_down_call => $drbd_down_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_down_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0394", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0394", 
		variables => $variables, 
	});
	
	return(0);
}

sub process_connect
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr1_host_uuid         = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
	my $dr1_host_name         = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name};
	my $dr1_short_host_name   = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr1_host_uuid         => $dr1_host_uuid, 
		dr1_host_name         => $dr1_host_name, 
		dr1_short_host_name   => $dr1_short_host_name, 
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
	}});
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{Yes})
	{
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0386", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 25,
			message   => "job_0386", 
			variables => $variables, 
		});
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "connect=1\n";
		
		# Register the job with the DR host.
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $dr1_host_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	# If the resource is down, bring it up.
	my $variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0387", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 50,
		message   => "job_0387", 
		variables => $variables, 
	});
	
	# Bring up the connection locally, and then also bring up the connection on the nodes, in case the 
	# server is down.
	my $drbd_up_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." || ".$anvil->data->{path}{exe}{drbdadm}." up ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_up_call => $drbd_up_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_up_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0388", variables => $variables});
	$anvil->Job->update_progress({
		progress => 60,
		message  => "job_0388", 
	});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid)
	{
		# "Peer" in this context is either/both nodes
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		$variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0389", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 70,
			message   => "job_0389", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $drbd_up_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Now watch until we connect to at least one peer.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0390"});
	$anvil->Job->update_progress({
		progress => 80,
		message  => "job_0390", 
	});
	my $waiting = 1;
	while($waiting)
	{
		sleep 5;
		$anvil->DRBD->gather_data({debug => 2});
		
		$waiting = 0;
		foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
			my $volume_up = 0;
			foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}})
			{
				my $replication_speed      = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{replication_speed};
				my $peer_role              = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_role};
				my $peer_disk_state        = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_disk_state};
				my $estimated_time_to_sync = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{estimated_time_to_sync};
				my $local_disk_state       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_disk_state};
				my $connection_state       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{connection_state};
				my $local_role             = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_role};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					peer                   => $peer,
					replication_speed      => $replication_speed, 
					peer_role              => $peer_role, 
					peer_disk_state        => $peer_disk_state, 
					estimated_time_to_sync => $estimated_time_to_sync, 
					local_disk_state       => $local_disk_state, 
					connection_state       => $connection_state, 
					local_role             => $local_role, 
				}});
				
				if ($connection_state eq "established")
				{
					$volume_up = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume_up => $volume_up }});
				}
				last if $volume_up;
			}
			
			if (not $volume_up)
			{
				$waiting = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
			}
		}
		
		if (not $waiting)
		{
			# We're ready.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0379"});
			$anvil->Job->update_progress({
				progress => 95,
				message  => "job_0379", 
			});
			$waiting = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
		}
	}
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0391", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0391", 
		variables => $variables, 
	});
	
	return(0);
}

sub process_remove
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr1_host_uuid         = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
	my $dr1_host_name         = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name};
	my $dr1_short_host_name   = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	my $short_host_name       = $anvil->Get->short_host_name();
	my $server_definition_xml = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr1_host_uuid         => $dr1_host_uuid, 
		dr1_host_name         => $dr1_host_name, 
		dr1_short_host_name   => $dr1_short_host_name, 
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
		server_definition_xml => $server_definition_xml, 
		short_host_name       => $short_host_name,
	}});
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{Yes})
	{
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0407", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 25,
			message   => "job_0407", 
			variables => $variables, 
		});
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "remove=1\n";
		
		# Register the job with the DR host.
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $dr1_host_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	# Sanity checks complete!
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0406", variables => { server => $anvil->data->{server}{'server-name'} }});
	$anvil->Job->update_progress({
		progress => 30,
		message  => "job_0406,!!server!".$anvil->data->{server}{'server-name'}."!!", 
	});
	
	# Bring down the resource, if it's up.
	my $variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0393", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 40,
		message   => "job_0393", 
		variables => $variables, 
	});
	
	# Bring down the connection locally
	my $drbd_down_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." && ".$anvil->data->{path}{exe}{drbdadm}." down ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_down_call => $drbd_down_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_down_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	
	# Read through the local DRBD config file, and remove any LVs that are found to still exist.
	$anvil->DRBD->gather_data({debug => 2});
	
	my $config_file = $anvil->data->{new}{resource}{$server_name}{config_file} ? $anvil->data->{new}{resource}{$server_name}{config_file} : "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { config_file => $config_file }});
	
	# If we don't have a config file, we could be re-running and the file was wiped. We need this though,
	# so try to pull it from a node.
	if (not $config_file)
	{
		# Try to read it from node 1.
		$config_file = $anvil->data->{path}{directories}{drbd_resources}."/".$server_name.".res";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { config_file => $config_file }});
		
		my $variables = { 
			config_file => $config_file,
			source      => $node1_short_host_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0417", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 45,
			message   => "job_0417", 
			variables => $variables, 
		});
		
		# Rsync it over. 
		my $source = "root\@".$anvil->data->{lookup}{host_uuid}{$node1_host_uuid}{network}{use_ip}.":".$config_file;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { source => $source }});
		$anvil->Storage->rsync({
			source      => $source,
			destination => $anvil->data->{path}{directories}{drbd_resources}."/",
		});
		
		# Rescan DRBD
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0418"});
		$anvil->Job->update_progress({
			progress => 50,
			message  => "job_0418", 
		});
		$anvil->DRBD->gather_data({debug => 2});
	}
	
	# Read the old config file.
	my $old_resource_config = $anvil->Storage->read_file({file => $config_file});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_resource_config => $old_resource_config }});
	
	my $progress = "50";
	foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
	{
		my $local_lv = $anvil->data->{new}{resource}{$server_name}{host}{$dr1_short_host_name}{volume}{$volume}{backing_disk};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			volume   => $volume,
			local_lv => $local_lv, 
		}});
		
		$progress++;
		my $variables = {
			server   => $server_name, 
			volume   => $volume,
			local_lv => $local_lv, 
		};
		if (-e $local_lv)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0408", variables => $variables});
			$anvil->Job->update_progress({
				progress  => $progress,
				message   => "job_0408", 
				variables => $variables, 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0591", variables => { device_path => $local_lv }});
			my $return_code = $anvil->DRBD->remove_backing_lv({
				debug        => 2, 
				backing_disk => $local_lv, 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
			if ($return_code)
			{
				# Should have been '0'
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0368", variables => $variables});
				$anvil->Job->update_progress({
					progress  => 100,
					message   => "error_0368", 
					variables => $variables, 
				});
				$anvil->nice_exit({exit_code => 1});
			}
			else
			{
				# Gone.
				$progress++;
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0410"});
				$anvil->Job->update_progress({
					progress  => $progress,
					message   => "job_0410", 
				});
			}
		}
		else
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0409", variables => $variables});
			$anvil->Job->update_progress({
				progress  => $progress,
				message   => "job_0409", 
				variables => $variables, 
			});
		}
	}
	
	# Tell the user we're generationg the new resource config now.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0411"});
	$anvil->Job->update_progress({
		progress  => 70,
		message   => "job_0411", 
	});
	
	### If we're here, we can now update the resource config file to remove the DR host.
	# Refresh the IP info (usually scrubbed by this point)
	$anvil->Database->get_ip_addresses();
	
	# First loop to build the node sections, then we'll loop to build the connections
	my $node1_volumes    = "";
	my $node2_volumes    = "";
	my $node1_ip_address = "";
	my $node1_tcp_port   = "";
	my $node2_ip_address = "";
	my $node2_tcp_port   = "";
	foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
	{
		# Find the node 1 backing device, drbd device and minor numbers.
		my $node1_device_path  = $anvil->data->{new}{resource}{$server_name}{host}{$node1_short_host_name}{volume}{$volume}{device_path};
		my $node1_backing_disk = $anvil->data->{new}{resource}{$server_name}{host}{$node1_short_host_name}{volume}{$volume}{backing_disk};
		my $node1_device_minor = $anvil->data->{new}{resource}{$server_name}{host}{$node1_short_host_name}{volume}{$volume}{device_minor};
		my $node1_meta_disk    = $anvil->data->{new}{resource}{$server_name}{host}{$node1_short_host_name}{volume}{$volume}{meta_disk};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:node1_device_path'  => $node1_device_path,
			's2:node1_backing_disk' => $node1_backing_disk, 
			's3:node1_device_minor' => $node1_device_minor, 
			's4:node1_meta_disk'    => $node1_meta_disk, 
		}});
		
		$node1_volumes .= $anvil->Words->string({key => "file_0004", variables => {
			volume    => $volume,
			drbd_path => $node1_device_path, 
			minor     => $node1_device_minor, 
			lv_path   => $node1_backing_disk, 
			meta_data => $node1_meta_disk, 
		}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_volumes => $node1_volumes }});
		
		my $node2_device_path  = $anvil->data->{new}{resource}{$server_name}{host}{$node2_short_host_name}{volume}{$volume}{device_path};
		my $node2_backing_disk = $anvil->data->{new}{resource}{$server_name}{host}{$node2_short_host_name}{volume}{$volume}{backing_disk};
		my $node2_device_minor = $anvil->data->{new}{resource}{$server_name}{host}{$node2_short_host_name}{volume}{$volume}{device_minor};
		my $node2_meta_disk    = $anvil->data->{new}{resource}{$server_name}{host}{$node2_short_host_name}{volume}{$volume}{meta_disk};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:node2_device_path'  => $node2_device_path,
			's2:node2_backing_disk' => $node2_backing_disk, 
			's3:node2_device_minor' => $node2_device_minor, 
			's4:node2_meta_disk'    => $node2_meta_disk, 
		}});
		
		$node2_volumes .= $anvil->Words->string({key => "file_0004", variables => {
			volume    => $volume,
			drbd_path => $node2_device_path, 
			minor     => $node2_device_minor, 
			lv_path   => $node2_backing_disk, 
			meta_data => $node2_meta_disk, 
		}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_volumes => $node2_volumes }});
		
		if (not $node1_ip_address)
		{
			$node1_ip_address = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$node1_short_host_name}{$node2_short_host_name}{host1_ip_address};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_ip_address => $node1_ip_address }});
		}
		if (not $node1_tcp_port)
		{
			$node1_tcp_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$node1_short_host_name}{$node2_short_host_name}{host1_tcp_port};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_tcp_port => $node1_tcp_port }});
		}
		if (not $node2_ip_address)
		{
			$node2_ip_address = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$node1_short_host_name}{$node2_short_host_name}{host2_ip_address};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_ip_address => $node2_ip_address }});
		}
		if (not $node2_tcp_port)
		{
			$node2_tcp_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$node1_short_host_name}{$node2_short_host_name}{host2_tcp_port};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_tcp_port => $node2_tcp_port }});
		}
	}

	# Put together the nodes section.
	my $hosts = $anvil->Words->string({key => "file_0003", variables => {
		short_host_name => $node1_short_host_name, 
		node_id         => "0",
		volumes         => $node1_volumes, 
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { hosts => $hosts }});

	$hosts .= $anvil->Words->string({key => "file_0003", variables => {
		short_host_name => $node2_short_host_name, 
		node_id         => "1",
		volumes         => $node2_volumes, 
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { hosts => $hosts }});

	# Build the connections now
	my $connections = $anvil->Words->string({key => "file_0005", variables => {
		host1_short_name => $node1_short_host_name,
		host1_ip         => $node1_ip_address,
		tcp_port         => $node1_tcp_port,
		host2_short_name => $node2_short_host_name,
		host2_ip         => $node2_ip_address,
		tcp_port         => $node2_tcp_port,
		'c-rate-maximum' => 500,
		protocol         => "C",
		fencing          => "resource-and-stonith"
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { connections => $connections }});

	my $new_resource_config = $anvil->Words->string({key => "file_0006", variables => {
		server      => $server_name,
		hosts       => $hosts,
		connections => $connections,
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_resource_config => $new_resource_config }});
	
	my $difference = diff \$old_resource_config, \$new_resource_config, { STYLE => 'Unified' };
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});

	# Write out a test file.
	my $test_file = $anvil->data->{path}{directories}{temp}."/test-".$server_name.".res";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }});
	my ($problem) = $anvil->Storage->write_file({
		debug     => 2,
		backup    => 0,
		overwrite => 1,
		file      => $test_file, 
		body      => $new_resource_config,
		user      => "root", 
		group     => "root", 
		mode      => "0644",
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	
	# Validate.
	my $shell_call = $anvil->data->{path}{exe}{drbdadm}." --config-to-test ".$test_file." --config-to-exclude ".$config_file." sh-nop";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	if ($return_code)
	{
		# Something went wrong.
		my $variables = {
			return_code => $return_code, 
			output      => $output, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0345", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 100,
			message   => "error_0345", 
			variables => $variables, 
		});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Remove the test file.
	unlink $test_file;
	
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0412"});
	$anvil->Job->update_progress({
		progress  => 72,
		message   => "job_0412", 
	});
	
	# Copy the new config to the peers.
	$progress = 72;
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid)
	{
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = {
			file       => $config_file, 
			host_name  => $peer_host_name, 
			ip_address => $peer_ip, 
		};
		$progress += 2;
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0369", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $progress,
			message   => "job_0369", 
			variables => $variables, 
		});
		my ($problem) = $anvil->Storage->write_file({
			debug     => 2,
			backup    => 1,
			overwrite => 1,
			file      => $config_file, 
			body      => $new_resource_config, 
			user      => "root", 
			group     => "root", 
			mode      => "0644",
			target    => $peer_ip, 
			password  => $anvil_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		# Make sure the file exists now.
		my $check_resource_config = $anvil->Storage->read_file({
			file     => $config_file,
			target   => $peer_ip, 
			password => $anvil_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { check_resource_config => $check_resource_config }});
		
		my $difference = diff \$new_resource_config, \$check_resource_config, { STYLE => 'Unified' };
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
		
		# Failed to write the file.
		if ($difference)
		{
			$variables = {
				host_name             => $peer_host_name, 
				file                  => $config_file, 
				difference            => $difference, 
				new_resource_config   => $new_resource_config, 
				check_resource_config => $check_resource_config, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0405", variables => $variables});
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "job_0405", 
				variables  => $variables, 
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	# Call 'drbdadm adjust <server>' to update the running configs.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0413"});
	$anvil->Job->update_progress({
		progress => 76,
		message  => "job_0413", 
	});
	
	my $drbdadm_call = $anvil->data->{path}{directories}{drbdadm}." adjust ".$server_name.$anvil->Log->switches();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbdadm_call => $drbdadm_call }});
	($output, $return_code) = $anvil->System->call({shell_call => $drbdadm_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0414", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 77,
			message   => "job_0414", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $drbdadm_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Now delete our resource config file.
	$variables = { config_file => $config_file };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0415", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 80,
		message   => "job_0415", 
		variables => $variables, 
	});
	
	if (-e $config_file)
	{
		my $problem = $anvil->Storage->delete_file({file => $config_file});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		if ($problem)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0369", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0369", 
				variables => $variables
			});
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0416", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0416", 
		variables => $variables, 
	});
	
	return(0);
}

sub process_protect
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr1_host_uuid         = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
	my $dr1_host_name         = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name};
	my $dr1_short_host_name   = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	my $short_host_name       = $anvil->Get->short_host_name();
	my $server_definition_xml = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr1_host_uuid         => $dr1_host_uuid, 
		dr1_host_name         => $dr1_host_name, 
		dr1_short_host_name   => $dr1_short_host_name, 
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
		server_definition_xml => $server_definition_xml, 
		short_host_name       => $short_host_name,
	}});
	
	# Sanity checks complete!
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0360", variables => { server => $anvil->data->{server}{'server-name'} }});
	$anvil->Job->update_progress({
		progress => 30,
		message  => "job_0360,!!server!".$anvil->data->{server}{'server-name'}."!!", 
	});
	
	$anvil->Server->parse_definition({
		debug      => 2,
		host       => $short_host_name,
		server     => $anvil->data->{server}{'server-name'}, 
		source     => "from_db",
		definition => $server_definition_xml, 
	});
	
	$anvil->DRBD->gather_data({debug => 2});

	my $server_ram = $anvil->data->{server}{$short_host_name}{$server_name}{'from_db'}{memory};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server_ram => $anvil->Convert->add_commas({number => $server_ram})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram}).")",
	}});
	
	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{drbd}{resource}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource => $resource }});
		
		foreach my $this_host_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{host}})
		{
			my $this_host_uuid = $anvil->Get->host_uuid_from_name({host_name => $this_host_name});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				this_host_name => $this_host_name,
				this_host_uuid => $this_host_uuid, 
			}});
			
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}})
			{
				# Always get the LV sizes, as that factors metadata. DRBD size is 
				# minus metadata, and 0 when down.
				my $device_path  = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_path};
				my $backing_disk = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{backing_disk};
				my $device_minor = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_minor};
				my $tcp_port     = $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:volume"       => $volume, 
					"s2:device_path"  => $device_path,
					"s3:backing_disk" => $backing_disk,
					"s4:device_minor" => $device_minor,
				}});
				
				my $this_size = $anvil->Storage->get_size_of_block_device({debug => 2, host_uuid => $this_host_uuid, path => $backing_disk});
				if ($this_size eq "")
				{
					# DRBD config file was updated, but LV not created yet.
					next;
				}
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					this_size => $anvil->Convert->add_commas({number => $this_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $this_size}).")", 
				}});
				
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					this_size => $anvil->Convert->add_commas({number => $this_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $this_size}).")", 
				}});
				
				if ((not exists $anvil->data->{server}{drbd}{$resource}{$volume}{size}) or (not $anvil->data->{server}{drbd}{$resource}{$volume}{size}))
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{size} = $this_size;
					
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::size" => $anvil->data->{server}{drbd}{$resource}{$volume}{size}, 
					}});
				}
				
				if (not exists $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid})
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid} = "";
				}
				
				### NOTE: This check make sense only under the assumption that the DRBD minor
				###       is common across both nodes. This should be the case, but doesn't 
				###       strictly have to be so.
				if ((not exists $anvil->data->{server}{drbd}{$resource}{$volume}{minor_number})  or 
				    (not defined $anvil->data->{server}{drbd}{$resource}{$volume}{minor_number}) or 
				    ($anvil->data->{server}{drbd}{$resource}{$volume}{minor_number} eq ""))
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{minor_number} = $device_minor;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::minor_number" => $anvil->data->{server}{drbd}{$resource}{$volume}{minor_number},
					}});
				}
				
				if ((not exists $anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port})  or 
				    (not defined $anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port}) or 
				    ($anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port} eq ""))
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port} = $tcp_port;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::tcp_port" => $anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port},
					}});
				}
				
				# What storage group does this belong to?
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"server::drbd::${resource}::${volume}::storage_group_uuid" => $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid}, 
				}});
				if (not $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid})
				{
					my $storage_key        = $resource."/".$volume;
					my $storage_group_uuid = $anvil->Storage->get_storage_group_from_path({
						debug      => 2,
						anvil_uuid => $anvil_uuid, 
						path       => $backing_disk,
					});
					my $storage_group_name = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						storage_key        => $storage_key, 
						storage_group_uuid => $storage_group_uuid, 
						storage_group_name => $storage_group_name, 
					}});
					
					# We'll need to sum up the volumes on each storage group, as 
					# it's possible the volumes are on different SGs.
					$anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid}               = $storage_group_uuid;
					$anvil->data->{server}{storage_groups}{$storage_group_name}{used_by}{$storage_key} = 1;
					$anvil->data->{server}{storage_groups}{$storage_group_name}{storage_group_uuid}    = $storage_group_uuid;
					
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::storage_group_uuid"               => $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid}, 
						"server::storage_groups::${storage_group_name}::used_by::${storage_key}" => $anvil->data->{server}{storage_groups}{$storage_group_name}{used_by}{$storage_key},
						"server::storage_groups::${storage_group_name}::storage_group_uuid"      => $anvil->data->{server}{storage_groups}{$storage_group_name}{storage_group_uuid}, 
					}});
				}
				
				if ($this_size > $anvil->data->{server}{drbd}{$resource}{$volume}{size})
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{size} = $this_size;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::size" => $anvil->Convert->add_commas({number => $anvil->data->{server}{drbd}{$resource}{$volume}{size}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{server}{drbd}{$resource}{$volume}{size}}).")", 
					}});
				}
			}
		}
	}
	
	# Make sure there is enough space on DR for the volumes under this VM.
	my $problem     = 0;
	my $config_file = "";
	foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{server}{storage_groups}})
	{
		my $storage_group_uuid = $anvil->data->{server}{storage_groups}{$storage_group_name}{storage_group_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			storage_group_name => $storage_group_name, 
			storage_group_uuid => $storage_group_uuid, 
		}});
		
		# First, is this SG on DR?
		if (not exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr1_host_uuid})
		{
			# The DR host doesn't appear to be storage group.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0343", variables => { 
				host_name     => $dr1_host_name,
				storage_group => $$storage_group_name, 
			}});
			$anvil->Job->update_progress({
				progress => 100,
				message  => "error_0343,!!host_name!".$dr1_host_name."!!,!!storage_group!".$$storage_group_name."!!", 
			});
			$problem = 1;
		}
		
		my $space_needed = 0;
		foreach my $resource_key (sort {$a cmp $b} keys %{$anvil->data->{server}{storage_groups}{$storage_group_name}{used_by}})
		{
			my ($resource, $volume) =  ($resource_key =~ /^(.*)\/(\d+)$/);
			my $volume_size         =  $anvil->data->{server}{drbd}{$resource}{$volume}{size};
			   $space_needed        += $volume_size, 
			   $config_file         =  $anvil->data->{new}{resource}{$resource}{config_file};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				config_file  => $config_file, 
				resource_key => $resource_key, 
				resource     => $resource, 
				volume       => $volume, 
				volume_size  => $anvil->Convert->add_commas({number => $volume_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $volume_size}).")", 
				space_needed => $anvil->Convert->add_commas({number => $space_needed})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $space_needed}).")", 
			}});
			
		}
		
		# Is there enough space on DR?
		my $space_on_dr = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr1_host_uuid}{vg_free};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			space_on_dr  => $anvil->Convert->add_commas({number => $space_on_dr})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $space_on_dr}).")", 
			space_needed => $anvil->Convert->add_commas({number => $space_needed})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $space_needed}).")", 
		}});
		if ($space_needed > $space_on_dr)
		{
			my $variables = {
				space_needed       => $anvil->Convert->bytes_to_human_readable({'bytes' => $space_needed}), 
				space_needed_bytes => $anvil->Convert->add_commas({number => $space_needed}),
				storage_group      => $storage_group_name, 
				space_on_dr        => $anvil->Convert->bytes_to_human_readable({'bytes' => $space_on_dr}), 
				space_on_dr_bytes  => $anvil->Convert->add_commas({number => $space_on_dr}), 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0344", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0344", 
				variables => $variables
			});
			$problem = 1;
		}
	}
	if ($problem)
	{
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Have we already configured the DR? If so, what ports are already allocated.
	my $node1_to_dr_port = "";
	my $node2_to_dr_port = "";
	foreach my $host1_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{host1_to_host2}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host1_name => $host1_name }});
		foreach my $host2_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host2_name => $host2_name }});
			next if (($host1_name ne $dr1_short_host_name) && ($host2_name ne $dr1_short_host_name));
			if (($host1_name eq $node1_short_host_name) or ($host2_name eq $node1_short_host_name))
			{
				$node1_to_dr_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host1_tcp_port};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port => $node1_to_dr_port }});
			}
			else
			{
				$node2_to_dr_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host1_tcp_port};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port => $node2_to_dr_port }});
			}
		}
	}
	
	# Get net next pair of TCP ports, if needed.
	my (undef, $tcp_ports) = $anvil->DRBD->get_next_resource({
		debug        => 2,
		dr_tcp_ports => 1, 
	});
	my ($first_port, $second_port) = split/,/, $tcp_ports;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"s01:first_port"  => $first_port, 
		"s02:second_port" => $second_port, 
	}});
	if ($node1_to_dr_port eq "")
	{
		$node1_to_dr_port = $first_port;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port => $node1_to_dr_port }});
	}
	if ($node2_to_dr_port eq "")
	{
		$node2_to_dr_port = $second_port;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port => $node2_to_dr_port }});
	}
	
	# Show what we're doing
	my $variables = {
		protocol         => $anvil->data->{switches}{protocol}, 
		node1_to_dr_port => $node1_to_dr_port, 
		node2_to_dr_port => $node2_to_dr_port, 
		config_file      => $config_file, 
	};
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0361", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 40,
		message   => "job_0361", 
		variables => $variables, 
	});
	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}})
	{
		foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}{$resource}})
		{
			my $variables = {
				resource => $resource, 
				volume   => $volume, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0362", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 43,
				message   => "job_0362", 
				variables => $variables, 
			});
			my $lv_size            = $anvil->data->{server}{drbd}{$resource}{$volume}{size};
			my $storage_group_uuid = $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid};
			my $dr_lv_name         = $resource."_".$volume;
			my $dr_vg_name         = $anvil->Storage->get_vg_name({
				debug              => 3, 
				storage_group_uuid => $storage_group_uuid, 
				host_uuid          => $dr1_host_uuid,
			});
			my $dr_lv_path         = "/dev/".$dr_vg_name."/".$dr_lv_name;
			my $extent_size        = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr1_host_uuid}{vg_extent_size};
			my $extent_count       = int($lv_size / $extent_size);
			my $shell_call         = $anvil->data->{path}{exe}{lvcreate}." -l ".$extent_count." -n ".$dr_lv_name." ".$dr_vg_name." -y";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"s01:resource"           => $resource, 
				"s02:volume"             => $volume, 
				"s03:lv_size"            => $anvil->Convert->add_commas({number => $lv_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $lv_size}).")", , 
				"s04:storage_group_uuid" => $storage_group_uuid, 
				"s05:dr_lv_name"         => $dr_lv_name, 
				"s06:dr_vg_name"         => $dr_vg_name, 
				"s07:dr_lv_path"         => $dr_lv_path, 
				"s08:extent_size"        => $anvil->Convert->add_commas({number => $extent_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $extent_size}).")", 
				"s09:extent_count"       => $extent_count, 
				"s10:shell_call"         => $shell_call, 
			}});
			
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lvcreate_call}      = $shell_call;
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lv_path}            = $dr_lv_path;
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{storage_group_uuid} = $storage_group_uuid;
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_tcp_port}      = $anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port}; 
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_minor}         = $anvil->data->{server}{drbd}{$resource}{$volume}{minor_number};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"server::dr::volumes::${resource}::${volume}::lvcreate_call"      => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lvcreate_call}, 
				"server::dr::volumes::${resource}::${volume}::lv_path"            => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lv_path}, 
				"server::dr::volumes::${resource}::${volume}::storage_group_uuid" => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{storage_group_uuid}, 
				"server::dr::volumes::${resource}::${volume}::drbd_tcp_port"      => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_tcp_port}, 
				"server::dr::volumes::${resource}::${volume}::drbd_minor"         => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_minor}, 
			}});
			
			# Get the VG name that this volume will be created on.
			$variables = {
				lv_path       => $dr_lv_path, 
				lv_size       => $anvil->Convert->bytes_to_human_readable({'bytes' => $lv_size}), 
				lv_size_bytes => $anvil->Convert->add_commas({number => $lv_size}), 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0363", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 47,
				message   => "job_0363", 
				variables => $variables, 
			});
		}
	}
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{Yes})
	{
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "protect=1\n";
		   $job_data .= "protocol=".$anvil->data->{switches}{protocol}."\n";
		
		# We always record the job against node 1, as it has to use cluster info to run this, so we 
		# can't run it on the DR itself.
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_host_uuid => $node1_host_uuid }});
		
		# Register the job with this host
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $node1_host_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	### If we're still here, time to get started.
	# Read in the old config and update it.
	my $old_resource_config = $anvil->Storage->read_file({file => $config_file});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { old_resource_config => $old_resource_config }});
	
	# Pull the data out of the old file
	my $hosts          = "";
	my $nodes_tcp_port = "";
	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource => $resource }});
		my $dr1_seen = 0;
		foreach my $this_host_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{host}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { this_host_name => $this_host_name }});
			my $node_id = "";
			if (($this_host_name eq $node1_short_host_name) or ($this_host_name eq $node1_host_name))
			{
				$node_id = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node_id => $node_id }});
				
				if ((not $nodes_tcp_port) && 
				    (exists $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}) && 
				    ($anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port}))
				{
					$nodes_tcp_port = $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodes_tcp_port => $nodes_tcp_port }});
				}
			}
			elsif (($this_host_name eq $node2_short_host_name) or ($this_host_name eq $node2_host_name))
			{
				$node_id = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node_id => $node_id }});
				
				if ((not $nodes_tcp_port) && 
				    (exists $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}) && 
				    ($anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port}))
				{
					$nodes_tcp_port = $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodes_tcp_port => $nodes_tcp_port }});
				}
			}
			elsif (($this_host_name eq $dr1_short_host_name) or ($this_host_name eq $dr1_host_name))
			{
				$node_id  = 2;
				$dr1_seen = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					node_id  => $node_id,
					dr1_seen => $dr1_seen, 
				}});
			}
			my $volumes = "";
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}{$resource}})
			{
				my $device_path  = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_path};
				my $backing_disk = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{backing_disk};
				my $device_minor = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_minor};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:device_path"  => $device_path,
					"s2:backing_disk" => $backing_disk,
					"s3:device_minor" => $device_minor,
				}});
				
				$volumes .= $anvil->Words->string({key => "file_0004", variables => {
					volume    => $volume,
					drbd_path => $device_path, 
					minor     => $device_minor, 
					lv_path   => $backing_disk, 
				        meta_data => "internal", 
				}});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volumes => $volumes }});
				
				# Record the DRBD device for adding DR.
				if (not exists $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{device_path})
				{
					$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{device_path} = $device_path;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::dr::volumes::${resource}::${volume}::device_path" => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{device_path},
					}});
				}
			}
			$hosts .= $anvil->Words->string({key => "file_0003", variables => {
				short_host_name => $this_host_name, 
				node_id         => $node_id, 
				volumes         => $volumes,
			}});
		}
		if (not $dr1_seen)
		{
			# Inject the DR.
			my $volumes = "";
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{dr}{volumes}{$resource}})
			{
				my $device_path  = $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{device_path};
				my $backing_disk = $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lv_path};
				my $device_minor = $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_minor};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:device_path"  => $device_path,
					"s2:backing_disk" => $backing_disk,
					"s3:device_minor" => $device_minor,
				}});
				
				$volumes .= $anvil->Words->string({key => "file_0004", variables => {
					volume    => $volume,
					drbd_path => $device_path, 
					minor     => $device_minor, 
					lv_path   => $backing_disk, 
				        meta_data => "internal", 
				}});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volumes => $volumes }});
			}
			$hosts .= $anvil->Words->string({key => "file_0003", variables => {
				short_host_name => $dr1_short_host_name, 
				node_id         => "2", 
				volumes         => $volumes,
			}});
		}
	}
	
	# Refresh the IP info (usually scrubbed by this point)
	$anvil->Database->get_ip_addresses();
	
	# The connections. Node 1 to 2 always uses the BCN, Either node to DR needs 
	my $storage_network = "sn1";
	my $dr_network      = $anvil->data->{lookup}{host_uuid}{$dr1_host_uuid}{network}{use_network};
	my $dr1_ip          = $anvil->data->{lookup}{host_uuid}{$dr1_host_uuid}{network}{use_ip};
	my $node1_sn_ip     = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{network}{$storage_network}{ip_address};
	my $node1_dr_ip     = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{network}{$dr_network}{ip_address};
	my $node2_sn_ip     = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{network}{$storage_network}{ip_address};
	my $node2_dr_ip     = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{network}{$dr_network}{ip_address};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's01:storage_network' => $storage_network,
		's02:dr1_host_uuid'   => $dr1_host_uuid, 
		's03:dr_network'      => $dr_network, 
		's04:dr1_ip'          => $dr1_ip, 
		's05:node1_host_uuid' => $node1_host_uuid, 
		's06:node1_sn_ip'     => $node1_sn_ip,
		's07:node1_dr_ip'     => $node1_dr_ip,
		's08:node2_host_uuid' => $node2_host_uuid, 
		's09:node2_sn_ip'     => $node2_sn_ip, 
		's10:node2_dr_ip'     => $node2_dr_ip, 
	}});
	
	# Choose the DR protocol
	my $dr_protocol = "A";
	if ($anvil->data->{switches}{protocol} eq "sync")
	{
		$dr_protocol = "C";
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_protocol => $dr_protocol }});

	# Node 1 to Node 2 first
	my $connections = $anvil->Words->string({key => "file_0005", variables => {
		host1_short_name => $node1_short_host_name, 
		host1_ip         => $node1_sn_ip, 
		host2_short_name => $node2_short_host_name, 
		host2_ip         => $node2_sn_ip, 
		tcp_port         => $nodes_tcp_port, 
		'c-rate-maximum' => 500,
		protocol         => "C",
                fencing          => "resource-and-stonith"
	}});
	
	# Node 1 to DR
	$connections .= $anvil->Words->string({key => "file_0005", variables => {
		host1_short_name => $node1_short_host_name, 
		host1_ip         => $node1_dr_ip, 
		host2_short_name => $dr1_short_host_name, 
		host2_ip         => $dr1_ip, 
		tcp_port         => $node1_to_dr_port, 
		'c-rate-maximum' => 500,
		protocol         => $dr_protocol,
                fencing          => "dont-care"
	}});

	# Node 2 to DR
	$connections .= $anvil->Words->string({key => "file_0005", variables => {
		host1_short_name => $node2_short_host_name, 
		host1_ip         => $node2_dr_ip, 
		host2_short_name => $dr1_short_host_name, 
		host2_ip         => $dr1_ip, 
		tcp_port         => $node2_to_dr_port, 
		'c-rate-maximum' => 500,
		protocol         => $dr_protocol,
                fencing          => "dont-care"
	}});
	
	my $new_resource_config = $anvil->Words->string({key => "file_0006", variables => {
		server      => $server_name,
		hosts       => $hosts,
		connections => $connections,
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_resource_config => $new_resource_config }});
	
	# Is the new res file the same as the old one?
	my $difference = diff \$old_resource_config, \$new_resource_config, { STYLE => 'Unified' };
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
	
	# Is there a difference on the local version? There might be a difference on peers, but we'll deel 
	# with that below.
	if (not $difference)
	{
		# The resource file doesn't need to be updated.
		my $variables = {
			file => $config_file, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0364", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 50,
			message   => "job_0364", 
			variables => $variables, 
		});
	}
	else
	{
		# Write out a test file.
		my $test_file = $anvil->data->{path}{directories}{temp}."/test-".$server_name.".res";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }});
		my ($problem) = $anvil->Storage->write_file({
			debug     => 2,
			backup    => 0,
			overwrite => 1,
			file      => $test_file, 
			body      => $new_resource_config,
			user      => "root", 
			group     => "root", 
			mode      => "0644",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		# Validate.
		my $shell_call = $anvil->data->{path}{exe}{drbdadm}." --config-to-test ".$test_file." --config-to-exclude ".$config_file." sh-nop";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
		if ($return_code)
		{
			# Something went wrong.
			my $variables = {
				return_code => $return_code, 
				output      => $output, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0345", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0345", 
				variables => $variables, 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Remove the test file.
		unlink $test_file;
		
		# Backup the res file so we can tell the user where the current config was backed up to in 
		# case they need to restore it.
		my ($backup_file) = $anvil->Storage->backup({file => $config_file});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});
		my $variables = { backup_file => $backup_file };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0365", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 60,
			message   => "job_0365", 
			variables => $variables, 
		});
		
		# Write out the new file.
		($problem) = $anvil->Storage->write_file({
			debug     => 2,
			backup    => 0,
			overwrite => 1,
			file      => $config_file, 
			body      => $new_resource_config,
			user      => "root", 
			group     => "root", 
			mode      => "0644",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0366"});
		$anvil->Job->update_progress({
			progress  => 65,
			message   => "job_0366", 
		});
		
		# Call 'drbdadm dump-xml' to check that it's OK.
		($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{drbdadm}." dump-xml"});
		if ($return_code)
		{
			# Something went wrong.
			my $variables = {
				return_code => $return_code, 
				output      => $output, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0345", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 70,
				message   => "error_0345", 
				variables => $variables, 
			});
			
			# Restoring the old config now.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0346"});
			$anvil->Job->update_progress({
				progress => 75,
				message  => "error_0346", 
			});
			
			# Backup the bad file and worn the user.
			my ($backup_file) = $anvil->Storage->backup({file => $config_file});
			$variables = { file => $backup_file };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0347", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 80,
				message   => "error_0347", 
				variables => $variables, 
			});
			
			# Write out the new file.
			my ($problem) = $anvil->Storage->write_file({
				debug     => 2,
				backup    => 1,
				overwrite => 1,
				file      => $config_file, 
				body      => $old_resource_config, 
				user      => "root", 
				group     => "root", 
				mode      => "0644",
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});

			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0348"});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0348", 
				variables => $variables, 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# New config is good! Update the file on the peers.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0367"});
		$anvil->Job->update_progress({
			progress => 70,
			message  => "job_0367", 
		});
	}
	
	# New config is good! Update the file on the peers.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0368"});
	$anvil->Job->update_progress({
		progress => 72,
		message  => "job_0368", 
	});
	
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host. It's not uncommon for the DR host to 
		# not have a connection over the SN or even the BCN. So we'll use the IFN1 to move files.
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = {
			file       => $config_file, 
			host_name  => $peer_host_name, 
			ip_address => $peer_ip, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0369", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 75,
			message   => "job_0369", 
			variables => $variables, 
		});
		my ($problem) = $anvil->Storage->write_file({
			debug     => 2,
			backup    => 1,
			overwrite => 1,
			file      => $config_file, 
			body      => $new_resource_config, 
			user      => "root", 
			group     => "root", 
			mode      => "0644",
			target    => $peer_ip, 
			password  => $anvil_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		# Make sure the file exists now.
		my $check_resource_config = $anvil->Storage->read_file({
			file     => $config_file,
			target   => $peer_ip, 
			password => $anvil_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { check_resource_config => $check_resource_config }});
		
		my $difference = diff \$new_resource_config, \$check_resource_config, { STYLE => 'Unified' };
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
		
		# Failed to write the file.
		if ($difference)
		{
			$variables = {
				host_name             => $peer_host_name, 
				file                  => $config_file, 
				difference            => $difference, 
				new_resource_config   => $new_resource_config, 
				check_resource_config => $check_resource_config, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0405", variables => $variables});
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "job_0405", 
				variables  => $variables, 
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	# Immediately call scan-drbd on all machines to ensure that if another run is about to happen for a 
	# different server, it knows the used ports list is updated.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0381"});
	$anvil->Job->update_progress({
		progress => 76,
		message  => "job_0381", 
	});
	
	my $scan_drbd_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd --force ".$anvil->Log->switches();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { scan_drbd_call => $scan_drbd_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $scan_drbd_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0382", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 77,
			message   => "job_0382", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $scan_drbd_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Create the LV and MD on DR.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0370"});
	$anvil->Job->update_progress({
		progress  => 78,
		message   => "job_0370", 
	});
	my $create_md = 0;
	foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}{$server_name}})
	{
		my $dr1_ip  = $anvil->data->{lookup}{host_uuid}{$dr1_host_uuid}{network}{use_ip};
		my $lv_path = $anvil->data->{server}{dr}{volumes}{$server_name}{$volume}{lv_path};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:volume'  => $volume,
			's2:dr1_ip'  => $dr1_ip, 
			's3:lv_path' => $lv_path, 
		}});
		
		my $variables = { 
			volume  => $volume, 
			lv_path => $lv_path, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0371", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 80,
			message   => "job_0371", 
			variables => $variables, 
		});
		
		my $lv_check_call = "
if [ -e '".$lv_path."' ]; 
then 
    echo exists;
else 
    echo create;
fi";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { lv_check_call => $lv_check_call }});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $dr1_ip, 
			password   => $anvil_password,
			shell_call => $lv_check_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		if ($output eq "exists")
		{
			my $variables = { lv_path => $lv_path };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0372", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 80,
				message   => "job_0372", 
				variables => $variables, 
			});
			next;
		}
		
		# Create the LV.
		my $lvcreate_call = $anvil->data->{server}{dr}{volumes}{$server_name}{$volume}{lvcreate_call};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { lvcreate_call => $lvcreate_call }});
		($output, $error, $return_code) = $anvil->Remote->call({
			target     => $dr1_ip, 
			password   => $anvil_password,
			shell_call => $lvcreate_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		
		sleep 1;
		# Does it exist now?
		($output, $error, $return_code) = $anvil->Remote->call({
			target     => $dr1_ip, 
			password   => $anvil_password,
			shell_call => $lv_check_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		if ($output eq "create")
		{
			my $variables = { lv_path => $lv_path };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0349", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0349", 
				variables => $variables, 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Create the DRBD meta data now.
		$create_md = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_md => $create_md }});
	}
	
	if ($create_md)
	{
		my $dr1_ip       = $anvil->data->{lookup}{host_uuid}{$dr1_host_uuid}{network}{use_ip};
		my $drbd_md_call = $anvil->data->{path}{exe}{drbdadm}." --force create-md --max-peers=3 ".$server_name;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:dr1_ip'       => $dr1_ip, 
			's2:drbd_md_call' => $drbd_md_call, 
		}});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $dr1_ip, 
			password   => $anvil_password,
			shell_call => $drbd_md_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Reload the config.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0373"});
	$anvil->Job->update_progress({
		progress => 85,
		message  => "job_0373", 
	});
	my $shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = { 
			server    => $server_name,
			host_name => $peer_host_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0374", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 85,
			message   => "job_0374", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $shell_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# If the resource is down, bring it up.
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0375", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 88,
		message   => "job_0375", 
		variables => $variables, 
	});
	my $drbd_up_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." || ".$anvil->data->{path}{exe}{drbdadm}." up ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_up_call => $drbd_up_call }});
	($output, $return_code) = $anvil->System->call({shell_call => $drbd_up_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		$variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0376", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 90,
			message   => "job_0376", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $drbd_up_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Now watch until the DR host shows up
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0377"});
	$anvil->Job->update_progress({
		progress => 92,
		message  => "job_0377", 
	});
	my $waiting = 1;
	while($waiting)
	{
		sleep 5;
		$anvil->DRBD->gather_data({debug => 2});
		
		my $dr_seen = 1;
		foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
			if (exists $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr1_short_host_name})
			{
				my $local_role       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr1_short_host_name}{local_role};
				my $local_disk_state = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr1_short_host_name}{local_disk_state};
				my $peer_role        = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr1_short_host_name}{peer_role};
				my $peer_disk_state  = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr1_short_host_name}{peer_disk_state}; 
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					local_role       => $local_role,
					local_disk_state => $local_disk_state, 
					peer_role        => $peer_role, 
					peer_disk_state  => $peer_disk_state, 
				}});
			}
			else
			{
				# Not up yet.
				my $next_check = $anvil->Get->date_and_time({offset => 5, time_only => 1});
				my $variables  = { next_check => $next_check };
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0378", variables => $variables});
				$anvil->Job->update_progress({
					progress  => 95,
					message   => "job_0378", 
					variables => $variables, 
				});
				$dr_seen = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_seen => $dr_seen }});
				last;
			}
		}
		
		if ($dr_seen)
		{
			# We're ready.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0379"});
			$anvil->Job->update_progress({
				progress => 98,
				message  => "job_0379", 
			});
			$waiting = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
		}
	}
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0380", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0380", 
		variables => $variables, 
	});
	
	return(0);
}

sub load_job
{
	my ($anvil) = @_;
	
	$anvil->Job->clear();
	$anvil->Job->get_job_details();
	$anvil->Job->update_progress({
		progress         => 1,
		job_picked_up_by => $$, 
		job_picked_up_at => time, 
		message          => "message_0267", 
	});
	
	print "Loading the job: [".$anvil->data->{switches}{'job-uuid'}."]...\n";
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"jobs::job_command"  => $anvil->data->{jobs}{job_command},
		"jobs::job_data"     => $anvil->data->{jobs}{job_data}, 
		"jobs::job_progress" => $anvil->data->{jobs}{job_progress}, 
		"jobs::job_status"   => $anvil->data->{jobs}{job_status}, 
	}});
	
	# Break up the job data into switches.
	$anvil->data->{switches}{Yes} = 1;
	print "- Set the switch: [--Yes] to true.\n";
	foreach my $line (split/\n/, $anvil->data->{jobs}{job_data})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
		if ($line =~ /(.*?)=(.*)$/)
		{
			my $key   = $1;
			my $value = $2;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				key   => $key,
				value => $value, 
			}});
			
			$anvil->data->{switches}{$key} = $value;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"switches::${key}" => $anvil->data->{switches}{$key}, 
			}});
			
			print "* Set the switch: [--".$key."] to: [".$value."]\n";
		}
	}
	print "Job loaded successfully.\n\n";
	
	return(0);
}