anvil/tools/anvil-delete-server

#!/usr/bin/perl
# 
# This provisions a new virtual machine server. It handles creating the logical volumes, DRBD resources, 
# verifies the needed files are ready, creates the provision script, begins the installation, and adds the 
# new server to pacemaker.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Any problem that causes an early exit.
# 

use strict;
use warnings;
use Anvil::Tools;
require POSIX;
use Term::Cap;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

# Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is 
# passed directly, it will be used. Otherwise, the password will be read from the database.
$anvil->Get->switches({list => [
	"force",
	"server",
	"job-uuid"], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});

$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, 'print' => 1, level => 0, priority => "err", key => "error_0218"});
	sleep 10;
	$anvil->nice_exit({exit_code => 1});
}

### NOTE: This must be run on a Node or DR host and will only delete servers on the same Anvil!. 
# This job is loaded on the node hosting the VM, or the primary node is the server isn't running. The first
# node to get this job will shut the server down and remove it from the cluster using 'pcs'. Once off and
# removed from the cluster, the server is marked as 'DELETED' and then a job is registered with the peer node
# and, if available, the DR host. At this point, the job acts the same regardless of the host. The DRBD 
# resource will stopped and then have it's metadata wiped, The LV backing the device will be deleted next. 

$anvil->data->{sys}{anvil_uuid} = $anvil->Cluster->get_anvil_uuid();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::anvil_uuid' => $anvil->data->{sys}{anvil_uuid} }});

# If we still don't have a job-uuit, go into interactive mode.
if ($anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->Job->clear();
	$anvil->Job->get_job_details();
	$anvil->Job->update_progress({
		progress         => 1,
		job_picked_up_by => $$, 
		job_picked_up_at => time, 
		message          => "message_0217", 
	});

	# Are we in an Anvil! system or are we a DR host?
	my $host_type = $anvil->Get->host_type();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
	if ((not $anvil->data->{sys}{anvil_uuid}) && ($host_type ne "dr"))
	{
		# We're not in an Anvil!.
		if ($anvil->data->{switches}{'job-uuid'})
		{
			# Mark the job as failed.
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0217", 
				job_status => "failed", 
			});
		}
		
		# Log an exit.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, 'print' => 1, level => 0, priority => 'err', key => "error_0217"});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Job data will be in $anvil->data->{jobs}{job_data}
	run_jobs($anvil);
}
elsif ($anvil->data->{switches}{server})
{
	# User specified what they want deleted.
	confirm_server_delete($anvil);
}
else
{
	if (not $anvil->data->{sys}{anvil_uuid})
	{
		# We can't do anything, exit.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, 'print' => 1, level => 0, priority => 'err', key => "error_0217"});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Interactive!
	ask_for_server($anvil);
}

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This actually provisions a VM.
sub run_jobs
{
	my ($anvil) = @_;
	
	# This parses the jobs::job_data intp variables.
	parse_job_data($anvil);
	
	my $host_type   = $anvil->Get->host_type();
	my $server_uuid = $anvil->data->{job}{server_uuid};
	my $server_name = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		host_type   => $host_type, 
		server_uuid => $server_uuid,
		server_name => $server_name, 
	}});
	
	# Before we start, we need to know if this server is on DR hosts. To do this, we'll parse the DRBD 
	# config file and look for DR hosts.
	$anvil->Database->get_hosts({debug => 2});
	$anvil->DRBD->gather_data({debug => 2});
	foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{peer}})
	{
		my $peer_host_uuid = $anvil->Database->get_host_uuid_from_string({debug => 2, string => $peer});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			peer           => $peer,
			peer_host_uuid => $peer_host_uuid, 
		}});
		if (($peer_host_uuid) && (exists $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}))
		{
			my $host_type = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{host_type};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
			
			if ($host_type eq "dr")
			{
				$anvil->data->{dr_hosts}{$peer_host_uuid} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"dr_hosts::${peer_host_uuid}" => $anvil->data->{dr_hosts}{$peer_host_uuid},
				}});
			}
		}
	}
	
	if (not $anvil->data->{job}{peer_mode})
	{
		# Remove the server from pacemaker (stopping it, if necessary).
		remove_from_pacemaker($anvil);
	}
	
	$anvil->Job->update_progress({
		progress => 25,
		message  => "job_0222,!!server_name!".$server_name."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0222", variables => { server_name => $server_name }});
	
	$anvil->Job->update_progress({
		progress => 50,
		message  => "job_0213", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0213"});
	
	### NOTE: If we're a DR host, and the server wasn't used here, this is expected to fail
	# Delete the DRBD resource and backing storage
	my $problem = $anvil->DRBD->delete_resource({debug => 2, resource => $server_name});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	if (($problem) && ($host_type eq "node"))
	{
		# Something went wrong
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0228,!!resource!".$server_name."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0228", variables => { resource => $server_name }});
	}
	
	$anvil->Job->update_progress({
		progress => 60,
		message  => "job_0214", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0214"});
	
	# Make sure the server is flagged as DELETEd.
	$anvil->Database->get_servers();
	my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_state => $server_state }});
	if ($server_state ne "DELETED")
	{
		my $query = "
UPDATE 
    servers 
SET 
    server_state  = 'DELETED', 
    modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." 
WHERE 
    server_uuid   = ".$anvil->Database->quote($server_uuid)."
;";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { query => $query }});
		$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
		
		$anvil->Job->update_progress({
			progress => 70,
			message  => "job_0215", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0215"});
	}
	
	# Delete the XML definition file.
	my $resource_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource_file => $resource_file }});
	if (-f $resource_file)
	{
		# Remove it.
		$anvil->Job->update_progress({
			progress => 80,
			message  => "job_0220,!!file!".$resource_file."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0220", variables => { file => $resource_file }});
		unlink $resource_file;
	}
	
	# Call scan-cluster, scan-network and scan-server to make sure the databases are updated.
	$anvil->Job->update_progress({
		progress => 90,
		message  => "job_0464", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0464"});
	foreach my $agent ("scan-cluster", "scan-network", "scan-server")
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0740", variables => { agent => $agent }});
		
		my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/".$agent."/".$agent.$anvil->Log->switches();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Make sure drbd-fence attributes are deleted for this server.
	clear_cib($anvil);
	
	# We're done
	$anvil->Job->update_progress({
		progress => 100,
		message  => "job_0216", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0216"});

	return(0);
}

# Make sure drbd-fence attributes are deleted for this server.
sub clear_cib
{
	my ($anvil) = @_;
	
	my $problem = $anvil->Cluster->parse_cib({debug => 2});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	if ($problem)
	{
		# Not in a cluster
		return(0);
	}
	
	my $server_uuid     = $anvil->data->{job}{server_uuid};
	my $server_name     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
	my $host_name       = $anvil->Get->host_name;
	my $short_host_name = $anvil->Get->short_host_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server_uuid     => $server_uuid,
		server_name     => $server_name, 
		host_name       => $host_name, 
		short_host_name => $short_host_name, 
	}});
	
	# Find attributes
	foreach my $node_id (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{cib}{node_state}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node_id => $node_id }});
		foreach my $attribute_id (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{cib}{node_state}{$node_id}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { attribute_id => $attribute_id }});
			if ($attribute_id =~ /^drbd-fenced_(.*)$/)
			{
				my $this_server_name = $1;
				my $state            = $anvil->data->{cib}{parsed}{cib}{node_state}{$node_id}{$attribute_id};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					this_server_name => $server_name,
					'state'          => $state, 
				}});
				
				if ($this_server_name eq $server_name)
				{
					# Stale attribute, remove it!
					my $node_name = $anvil->data->{cib}{parsed}{configuration}{nodes}{$node_id}{uname};
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0329", variables => { 
						attribute => $attribute_id,
						node_name => $node_name, 
						node_id   => $node_id, 
					}});
					
					my $shell_call = $anvil->data->{path}{exe}{pcs}." node attribute ".$node_name." ".$attribute_id."=";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
				}
			}
		}
	}
	
	return(0);
}

# This checks to see if the server is running and, if so, stops it. Once stopped, the resource is deleted.
sub remove_from_pacemaker
{
	my ($anvil) = @_;
	
	my $server_uuid = $anvil->data->{job}{server_uuid};
	my $server_name = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server_uuid => $server_uuid,
		server_name => $server_name, 
	}});
	
	# Sanity checks passed
	$anvil->Job->update_progress({
		progress => 10,
		message  => "job_0210,!!server_name!".$server_name."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0210", variables => { server_name => $server_name }});
	
	if (not $anvil->data->{cib}{parsed}{data}{server}{$server_name})
	{
		# Server is already out of the cluster.
		$anvil->Job->update_progress({
			progress => 20,
			message  => "job_0221,!!server_name!".$server_name."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0211", variables => { server_name => $server_name }});
		
		# Force the server off now, just in case it's running outside the cluster
		$anvil->Job->update_progress({
			progress => 25,
			message  => "job_0223,!!server_name!".$server_name."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0223", variables => { server_name => $server_name }});
		my $success = $anvil->Server->shutdown_virsh({
			debug  => 2, 
			force  => 1,
			server => $server_name,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { success => $success }});
		if (not $success)
		{
			# Failed to stop
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0223,!!server_name!".$server_name."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0223", variables => { server_name => $server_name }});
			$anvil->nice_exit({exit_code => 1});
		}
	}
	else
	{
		# As we're going to delete the server, we won't wait. We'll come back here and destroy the 
		# server if it's still running.
		if ($anvil->data->{cib}{parsed}{data}{server}{$server_name}{status} ne "off")
		{
			my $problem = $anvil->Cluster->shutdown_server({
				debug  => 2, 
				server => $server_name, 
				'wait' => 0, 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
			if ($problem)
			{
				# Failed to stop.
				$anvil->Job->update_progress({
					progress   => 100,
					message    => "error_0223,!!server_name!".$server_name."!!", 
					job_status => "failed", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0223", variables => { server_name => $server_name }});
				$anvil->nice_exit({exit_code => 1});
			}
			
			# Force the server off now.
			$anvil->Job->update_progress({
				progress => 20,
				message  => "job_0223,!!server_name!".$server_name."!!", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0223", variables => { server_name => $server_name }});
			my $success = $anvil->Server->shutdown_virsh({
				debug  => 2, 
				force  => 1,
				server => $server_name,
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { success => $success }});
			if (not $success)
			{
				# Failed to stop
				$anvil->Job->update_progress({
					progress   => 100,
					message    => "error_0223,!!server_name!".$server_name."!!", 
					job_status => "failed", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0223", variables => { server_name => $server_name }});
				$anvil->nice_exit({exit_code => 1});
			}
			
			# Server is off now.
			$anvil->Job->update_progress({
				progress => 25,
				message  => "job_0211,!!server_name!".$server_name."!!", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0211", variables => { server_name => $server_name }});
		}
		
		# Delete the resource.
		my $problem = $anvil->Cluster->delete_server({debug => 2, server_name => $server_name});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		if ($problem)
		{
			# Something went wrong 
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0227,!!server_name!".$server_name."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0227", variables => { server_name => $server_name }});
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	# Register the job with the peers. 
	my $anvil_uuid = $anvil->data->{sys}{anvil_uuid};
	my $peers      = [];
	if ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid} eq $anvil->Get->host_uuid)
	{
		# We're node 1
		push @{$peers}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}
	}
	else
	{
		# We're node 2
		push @{$peers}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}
	}
	
	if (exists $anvil->data->{dr_hosts})
	{
		foreach my $peer_host_uuid (keys %{$anvil->data->{dr_hosts}})
		{
			push @{$peers}, $peer_host_uuid;
		}
	}
	
	my $progress = 30;
	foreach my $host_uuid (@{$peers})
	{
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-delete-server'}.$anvil->Log->switches, 
			job_data        => "server_uuid=".$server_uuid."\npeer_mode=true", 
			job_name        => "server::delete", 
			job_title       => "job_0208", 
			job_description => "job_0209", 
			job_progress    => 0,
			job_host_uuid   => $host_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		my $host_name = $anvil->Get->host_name_from_uuid({host_uuid => $host_uuid});
		$anvil->Job->update_progress({
			progress => $progress,
			message  => "job_0212,!!host_name!".$host_name."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0212", variables => { host_name => $host_name }});
		$progress += 10;
	}
	
	return(0);
}

# This parses and verifies the job data
sub parse_job_data
{
	my ($anvil) = @_;
	
	$anvil->data->{job}{server_uuid} = "";
	$anvil->data->{job}{peer_mode}   = 0;
	foreach my $line (split/\n/, $anvil->data->{jobs}{job_data})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
		if ($line =~ /server_uuid=(.*)$/)
		{
			$anvil->data->{job}{server_uuid} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::server_uuid' => $anvil->data->{job}{server_uuid} }});
		}
		if ($line =~ /peer_mode=true/)
		{
			$anvil->data->{job}{peer_mode} = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::peer_mode' => $anvil->data->{job}{peer_mode} }});
		}
	}
	
	# Did we get a server UUID?
	if (not $anvil->data->{job}{server_uuid})
	{
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0219,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0219", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Does the server UUID match to a server?
	$anvil->Database->get_servers();
	my $server_uuid = $anvil->data->{job}{server_uuid};
	if (not exists $anvil->data->{servers}{server_uuid}{$server_uuid})
	{
		# Server UUID is invalid
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0220,!!server_uuid!".$server_uuid."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0220", variables => { server_uuid => $server_uuid }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	my $host_type = $anvil->Get->host_type();
	if ($host_type eq "node")
	{
		my $problem = $anvil->Cluster->parse_cib({debug => 2});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		if ($problem)
		{
			# The cluster isn't running, sleep and exit.
			$anvil->Job->update_progress({
				progress => 10,
				message  => "error_0222", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0222"});
			sleep 10;
			$anvil->nice_exit({exit_code => 1});
		}
		elsif (not $anvil->data->{cib}{parsed}{'local'}{ready})
		{
			# We're not a full member (yet)
			$anvil->Job->update_progress({
				progress => 10,
				message  => "error_0238", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0238"});
			sleep 10;
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	return(0);
}

# This allows a user to specify the server they want deleted without going through the meny system.
sub confirm_server_delete
{
	my ($anvil) = @_;
	
	$anvil->Get->server_from_switch({
		debug      => 2,
		server     => $anvil->data->{switches}{server}, 
		anvil_uuid => $anvil->data->{sys}{anvil_uuid},
	});
	
	if (not $anvil->data->{switches}{server_name})
	{
		# Not found.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0277", variables => { server => $anvil->data->{switches}{server} }});
		$anvil->nice_exit({exit_code => 1});
		
	}
	
	# Ask the user to confirm, if needed. Note that this method requires '--force', not '-y' or '--Yes'. 
	if (not $anvil->data->{switches}{force})
	{
		get_confirmation($anvil);
	}
	else
	{
		# They're forcing, save.
		save_job($anvil);
	}
	
	return(0);
}

# This will ask the user to select a server.
sub ask_for_server
{
	my ($anvil) = @_;
	
	my $termios = new POSIX::Termios;
	$termios->getattr;
	my $ospeed = $termios->getospeed;
	
	my $terminal = Tgetent Term::Cap { TERM => undef, OSPEED => $ospeed };
	$terminal->Trequire(qw/ce ku kd/);
	
	my $anvil_uuid  = $anvil->data->{sys}{anvil_uuid};
	my $anvil_name  = $anvil->Cluster->get_anvil_name({anvil_uuid => $anvil_uuid});
	my $retry       = 0;
	my $delete_uuid = "";
	while(1)
	{
		print $terminal->Tputs('cl');
		
		$anvil->Database->get_servers();
		my $servers     = [];
		my $position    = 0;
		my $server_list = "";
		
		print $anvil->Words->string({key => "message_0208", variables => { anvil_name => $anvil_name }})."\n";
		foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
		{
			my $server_uuid  = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
			my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:server_name'  => $server_name,
				's2:server_uuid'  => $server_uuid, 
				's3:server_state' => $server_state, 
			}});
			next if $server_state eq "DELETED";
			
			# We want to start the list at '1', so we'll bump the position before generating an entry, 
			# and subtract 1 from the user's answer later.
			$servers->[$position] = $server_uuid; 
			$position++;

			$server_list .= $position.") ".$server_name."\n";
		}
		if (not $position)
		{
			# No servers on this Anvil!.
			print $anvil->Words->string({key => "message_0209"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
		print $server_list."\n";;
		if ($retry)
		{
			print $anvil->Words->string({key => "message_0211"})."\n\n";
		}
		print $anvil->Words->string({key => "message_0210"})." ";
		
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /\D/)
		{
			# Did the user type the name?
			if ((exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$answer}) && ($anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$answer}{server_uuid}))
			{
				# Specified by name.
				$delete_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$answer}{server_uuid};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { delete_uuid => $delete_uuid }});
			}
		}
		elsif ($answer =~ /^\d+$/)
		{
			my $index = $answer - 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'index' => $index }});
			if ((exists $servers->[$index]) && ($servers->[$index]))
			{
				$delete_uuid = $servers->[$index];
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { delete_uuid => $delete_uuid }});
			}
		}
		if ($delete_uuid)
		{
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	$anvil->Get->server_from_switch({
		debug      => 2,
		server     => $delete_uuid, 
		anvil_uuid => $anvil->data->{sys}{anvil_uuid},
	});
	
	# Ask the user to confirm.
	get_confirmation($anvil);

	return(0);
}

sub get_confirmation
{
	my ($anvil) = @_;
	
	my $delete_uuid = $anvil->data->{switches}{server_uuid};
	my $server_name = $anvil->data->{servers}{server_uuid}{$delete_uuid}{server_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		delete_uuid => $delete_uuid,
		server_name => $server_name, 
	}});
	print "\n".$anvil->Words->string({key => "message_0212", variables => { server_name => $server_name }})." ";
	
	my $answer = <STDIN>;
	chomp $answer;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
	
	if ($answer eq "Yes")
	{
		### Save the job!
		save_job($anvil);
	}
	else
	{
		# Abort.
		print $anvil->Words->string({key => "message_0022"})."\n";
		$anvil->nice_exit({exit_code => 0});
	}
	
	return(0);
}

sub save_job
{
	my ($anvil) = @_;
	
	# Is the server running?
	print $anvil->Words->string({key => "message_0213"})."\n";
	
	$anvil->Database->get_anvils();
	my $hosts       = [];
	my $anvil_uuid  = $anvil->data->{sys}{anvil_uuid};;
	my $password    = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $server_name = $anvil->data->{switches}{server_name};
	my $server_uuid = $anvil->data->{switches}{server_uuid};
	my $delete_uuid = $server_uuid;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid  => $anvil_uuid,
		server_name => $server_name, 
		server_uuid => $server_uuid, 
		delete_uuid => $delete_uuid, 
	}});
	
	if ((not $server_name) && (exists $anvil->data->{servers}{server_uuid}{$server_uuid}) && ($anvil->data->{servers}{server_uuid}{$server_uuid}{server_name}))
	{
		$server_name = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_name => $server_name }});
	}
	
	# Find the server on hosts.
	my $server_host_name = $anvil->Server->locate({
		debug       => 2, 
		server_name => $server_name, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_host_name => $server_host_name }});
	
	foreach my $short_host_name (sort {$a cmp $b} keys %{$anvil->data->{server_location}{host}})
	{
		my $host_uuid = $anvil->Database->get_host_uuid_from_string({string => $short_host_name});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			short_host_name => $short_host_name, 
			host_uuid       => $host_uuid, 
		}});
		
		my $exists = 0;
		if (($anvil->data->{server_location}{host}{$short_host_name}{server}{$server_name}{file_definition}) or ($anvil->data->{server_location}{host}{$short_host_name}{server}{$server_name}{drbd_config}))
		{
			$exists = 1;
		}
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'exists' => $exists }});
		
		if (($exists)                                                                              or 
		    ($host_uuid eq $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}) or 
		    ($host_uuid eq $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}))
		{
			push @{$hosts}, $host_uuid;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_uuid => $host_uuid }});
		}
	}
	
	# If the server was found to be running, the host will be returned.
	my $server_host_uuid = "";
	if ($server_host_name)
	{
		$server_host_uuid = $anvil->Get->host_uuid_from_name({host_name => $server_host_name});
	}
	
	# Now, we'll do the delete, unless we see the server running elsewhere. 
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_host_uuid => $server_host_uuid }});
	
	my $job_host_uuid = "";
	if ($server_host_uuid)
	{
		$job_host_uuid = $server_host_uuid;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_host_uuid => $job_host_uuid }});
		if ($server_host_uuid eq $anvil->Get->host_uuid)
		{
			# Running here
			print $anvil->Words->string({key => "message_0216"})."\n";
		}
		else
		{
			# Running on a peer.
			print $anvil->Words->string({key => "message_0214", variables => { host_name => $server_host_name }})."\n";
		}
	}
	else
	{
		$job_host_uuid = $anvil->Get->host_uuid();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_host_uuid => $job_host_uuid }});
		print $anvil->Words->string({key => "message_0215"})."\n";
	}
	
	my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
		debug           => 2, 
		job_command     => $anvil->data->{path}{exe}{'anvil-delete-server'}.$anvil->Log->switches, 
		job_data        => "server_uuid=".$delete_uuid, 
		job_name        => "server::delete", 
		job_title       => "job_0208", 
		job_description => "job_0209", 
		job_progress    => 0,
		job_host_uuid   => $job_host_uuid, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
	
	$anvil->nice_exit({exit_code => 0});
	
	return(0);
}