anvil/tools/anvil-provision-server

#!/usr/bin/perl
# 
# This provisions a new virtual machine server. It handles creating the logical volumes, DRBD resources, 
# verifies the needed files are ready, creates the provision script, begins the installation, and adds the 
# new server to pacemaker.
# 
# Exit codes;
# 0 = Normal exit.

use strict;
use warnings;
use Anvil::Tools;
require POSIX;
use Term::Cap;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});

# Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is 
# passed directly, it will be used. Otherwise, the password will be read from the database.
$anvil->data->{switches}{'anvil-uuid'}    = "";
$anvil->data->{switches}{cpu}             = "";
$anvil->data->{switches}{'job-uuid'}      = "";
$anvil->data->{switches}{name}            = "";
$anvil->data->{switches}{ram}             = "";
$anvil->data->{switches}{'storage-group'} = "";
$anvil->data->{switches}{'storage-size'}  = "";
$anvil->Get->switches;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
	'switches::anvil-uuid'    => $anvil->data->{switches}{'anvil-uuid'},
	'switches::cpu'           => $anvil->data->{switches}{cpu},
	'switches::job-uuid'      => $anvil->data->{switches}{'job-uuid'},
	'switches::name'          => $anvil->data->{switches}{name},
	'switches::ram'           => $anvil->data->{switches}{ram},
	'switches::storage-group' => $anvil->data->{switches}{'storage-group'},
	'switches::storage-size'  => $anvil->data->{switches}{'storage-size'},
}});

$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0077"});
	sleep 10;
	$anvil->nice_exit({exit_code => 1});
}

# If we don't have a job UUID, try to find one.
if (not $anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->data->{switches}{'job-uuid'} = $anvil->Job->get_job_uuid({program => $THIS_FILE});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }});
}

# If we still don't have a job-uuit, go into interactive mode.
if ($anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->Job->clear();
	$anvil->Job->get_job_details();
	$anvil->Job->update_progress({
		progress         => 1,
		job_picked_up_by => $$, 
		job_picked_up_at => time, 
		message          => "message_0190", 
	});
	
	# Job data will be in $anvil->data->{jobs}{job_data}
	run_jobs($anvil);
}
else
{
	# Interactive!
	interactive_question($anvil);
	
}

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This actually provisions a VM.
sub run_jobs
{
	my ($anvil) = @_;
	
	# If we're here, the job has been assigned to us, so we use our own anvil! uuid.
	my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
	$anvil->Get->available_resources({
		debug      => 2,
		anvil_uuid => $anvil_uuid,
	});
	
	$anvil->Database->get_files();
	$anvil->Database->get_file_locations();
	
	my $server_name        = "";
	my $cpu_cores          = "";
	my $ram                = "";
	my $storage_group_uuid = "";
	my $storage_size       = "";
	my $install_iso_uuid   = "";
	my $driver_iso_uuid    = "";
	my $drbd_minor         = "";
	my $drbd_tcp_port      = "";
	my $peer_mode          = 0;
	foreach my $line (split/\n/, $anvil->data->{jobs}{job_data})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
		if ($line =~ /server_name=(.*)$/)
		{
			$server_name = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_name => $server_name }});
		}
		if ($line =~ /cpu_cores=(.*)$/)
		{
			$cpu_cores = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cpu_cores => $cpu_cores }});
		}
		if ($line =~ /ram=(.*)$/)
		{
			$ram = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cpu_cores => $cpu_cores }});
		}
		if ($line =~ /storage_group_uuid=(.*)$/)
		{
			$storage_group_uuid = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { storage_group_uuid => $storage_group_uuid }});
		}
		if ($line =~ /storage_size=(.*)$/)
		{
			$storage_size = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { storage_size => $storage_size }});
		}
		if ($line =~ /install_iso=(.*)$/)
		{
			$install_iso_uuid = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { install_iso_uuid => $install_iso_uuid }});
		}
		if ($line =~ /driver_iso=(.*)$/)
		{
			$driver_iso_uuid = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { driver_iso_uuid => $driver_iso_uuid }});
		}
		if ($line =~ /peer_mode=true$/)
		{
			# We'll ONLY setup our DRBD resource, nothing else.
			$peer_mode = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { peer_mode => $peer_mode }});
		}
		if ($line =~ /drbd_minor=(.*)$/)
		{
			$drbd_minor = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_minor => $drbd_minor }});
		}
		if ($line =~ /drbd_tcp_port=(.*)$/)
		{
			$drbd_tcp_port = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_tcp_port => $drbd_tcp_port }});
		}
	}
	
	# We need a server name and storage group UUID regardless of which mode we're in.
	if (not $server_name)
	{
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0187,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0187", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }});
		$anvil->nice_exit({exit_code => 1});
	}
	if (not $storage_group_uuid)
	{
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0192,!!server_name!".$server_name."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0192", variables => { 
			server_name => $server_name, 
			job_uuid    => $anvil->data->{switches}{'job-uuid'},
		}});
		$anvil->nice_exit({exit_code => 1});
	}
	if (not $storage_size)
	{
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0193,!!server_name!".$server_name."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0193", variables => { 
			server_name => $server_name, 
			job_uuid    => $anvil->data->{switches}{'job-uuid'},
		}});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# If we're the peer, there's only a few things we need to check.
	if ($peer_mode)
	{
		# The server should already exist on the peer. All we need to do is create our LV, create the
		# DRBD resource, bring the resource up, and save the XML definition file.
	}
	else
	{
		# Is the server name unique?
		$anvil->Database->get_servers();
		if (exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name})
		{
			# Duplicate name
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0198,!!server_name!".$server_name."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0198", variables => { server_name => $server_name }});
			$anvil->nice_exit({exit_code => 1});
		}

		if (not $cpu_cores)
		{
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0188,!!server_name!".$server_name."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0188", variables => { 
				server_name => $server_name, 
				job_uuid    => $anvil->data->{switches}{'job-uuid'},
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		elsif ($cpu_cores > $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads})
		{
			# Too many cores requested
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0189,!!server_name!".$server_name."!!,!!available_cores!".$anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}."!!,!!requested_cores!".$cpu_cores."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0189", variables => { 
				server_name     => $server_name, 
				available_cores => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}, 
				requested_cores => $cpu_cores, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		if (not $ram)
		{
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0190,!!server_name!".$server_name."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0190", variables => { 
				server_name => $server_name, 
				job_uuid    => $anvil->data->{switches}{'job-uuid'},
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		elsif ($ram > $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available})
		{
			# Too many cores requested
			my $say_available_ram = $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}})." (".$anvil->Convert->add_commas({number => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")";
			my $say_requested_ram = $anvil->Convert->bytes_to_human_readable({'bytes' => $ram})." (".$anvil->Convert->add_commas({number => $ram}).")"; 
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0191,!!server_name!".$server_name."!!,!!available_ram!".$say_available_ram."!!,!!requested_ram!".$say_requested_ram."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0191", variables => { 
				server_name   => $server_name, 
				available_ram => $say_available_ram, 
				requested_ram => $say_requested_ram, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		
		if (not $install_iso_uuid)
		{
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0195,!!server_name!".$server_name."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0195", variables => { 
				server_name => $server_name, 
				job_uuid    => $anvil->data->{switches}{'job-uuid'},
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		my $install_iso = $anvil->data->{files}{file_uuid}{$install_iso_uuid}{file_directory}."/".$anvil->data->{files}{file_uuid}{$install_iso_uuid}{file_name};
		if (not -e $install_iso)
		{
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0196,!!server_name!".$server_name."!!,!!install_iso!".$install_iso."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0196", variables => { 
				server_name => $server_name, 
				install_iso => $install_iso,
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		# Driver disc is optional.
		if ($driver_iso_uuid)
		{
			my $driver_iso = $anvil->data->{files}{file_uuid}{$driver_iso_uuid}{file_directory}."/".$anvil->data->{files}{file_uuid}{$driver_iso_uuid}{file_name};
			if (not -e $driver_iso)
			{
				$anvil->Job->update_progress({
					progress   => 100,
					message    => "error_0197,!!server_name!".$server_name."!!,!!driver_iso!".$driver_iso."!!", 
					job_status => "failed", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0197", variables => { 
					server_name => $server_name, 
					driver_iso => $driver_iso,
				}});
				$anvil->nice_exit({exit_code => 1});
			}
		}
	}

	# Read in VG information;
	my $problem = $anvil->Storage->get_storage_group_details({
		debug              => 2, 
		storage_group_uuid => $storage_group_uuid, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	if ($problem)
	{
		# Failed to collect storage group data.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0199,!!storage_group_uuid!".$storage_group_uuid."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0197", variables => { storage_group_uuid => $storage_group_uuid }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	if ((($drbd_minor eq "") or ($drbd_tcp_port eq "")) && (not $peer_mode))
	{
		my ($free_minor, $free_port) = $anvil->DRBD->get_next_resource({anvil_uuid => $anvil_uuid});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			free_minor => $free_minor,
			free_port  => $free_port, 
		}});
		
		if ($drbd_minor eq "")
		{
			$drbd_minor = $free_minor;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_minor => $drbd_minor }});
		}
		if ($drbd_tcp_port eq "")
		{
			$drbd_tcp_port = $free_port;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_tcp_port => $drbd_tcp_port }});
		}
	}
	
	# If we don't have a DRBD minor or TCP port, we're stuck.
	if (($drbd_minor eq "") or ($drbd_tcp_port eq ""))
	{
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0200", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0200"});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Sanity checks passed
	$anvil->Job->update_progress({
		progress   => 10,
		message    => "job_0185", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0185"});
	
	# Who is our peer?
	$anvil->Database->get_anvils();
	my $host_uuid       = $anvil->Get->host_uuid();
	my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $peer_host_uuid  = $host_uuid eq $node1_host_uuid ? $node2_host_uuid : $node1_host_uuid;
	my $peer_name       = $anvil->Get->host_name_from_uuid({host_uuid => $peer_host_uuid});
	my $short_host_name = $anvil->Get->short_host_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		host_uuid       => $host_uuid,
		node1_host_uuid => $node1_host_uuid, 
		node2_host_uuid => $node2_host_uuid, 
		peer_host_uuid  => $peer_host_uuid, 
		peer_name       => $peer_name, 
		short_host_name => $short_host_name, 
	}});
	
	# If we're not the peer, create the peer's job.
	if (not $peer_mode)
	{
		my $job_data   =  $anvil->data->{jobs}{job_data}."\n";
		   $job_data   .= "peer_mode=true\n";
		   $job_data   .= "drbd_minor=".$drbd_minor."\n";
		   $job_data   .= "drbd_tcp_port=".$drbd_tcp_port."\n";
		my ($job_uuid) =  $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-provision-server'}, 
			job_data        => $job_data, 
			job_name        => "server::provision", 
			job_title       => "job_0147", 
			job_description => "job_0148", 
			job_progress    => 0,
			job_host_uuid   => $peer_host_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		$anvil->Job->update_progress({
			progress => 20,
			message  => "job_0188,!!job_uuid!".$job_uuid."!!,!!peer_name!".$peer_name."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0188", variables => {
			job_uuid  => $job_uuid, 
			peer_name => $peer_name, 
		}});
	}
	
	# Report the DRBD minor and TCP port that we'll use.
	$anvil->Job->update_progress({
		progress => 30,
		message  => "job_0186,!!minor!".$drbd_minor."!!,!!port!".$drbd_tcp_port."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0186", variables => {
		minor => $drbd_minor, 
		port  => $drbd_tcp_port, 
	}});
	
	# We convert to extents as it ensure clean boundaries and, being based on bytes in both cases, gets 
	# us as close as we can to what the user asked for. The 'int()' always rounds down, so we don't ever 
	# ask for one more extent than is available by accident.
	my $vg_name      = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_name};
	my $extent_size  = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_extent_size};
	my $extent_count = int($storage_size / $extent_size);
	my $lv_path      = "/dev/".$vg_name."/".$server_name."_0";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		vg_name      => $vg_name, 
		extent_size  => $extent_size, 
		extent_count => $extent_count, 
		lv_path      => $lv_path, 
	}});
	
	# We need the peer's LVM path for the DRBD config.
	my $peer_vg_name = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$peer_host_uuid}{vg_name};
	my $peer_lv_path = "/dev/".$peer_vg_name."/".$server_name."_0";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { peer_lv_path => $peer_lv_path }});
	
	# If the LV exists, we won't initialize the DRBD resource. If the LV exists but isn't a DRBD backing 
	# device, this will cause the resource to not come up, but it's better than risking data loss.
	my $initialize_drbd = 1;
	if (-e $lv_path)
	{
		# Don't create and don't initialize.
		$initialize_drbd = 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { initialize_drbd => $initialize_drbd }});
		
		# Tell the user this might be an issue.
		$anvil->Job->update_progress({
			progress => 40,
			message  => "job_0187,!!lv_path!".$lv_path."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0187", variables => { lv_path => $lv_path }});
	}
	else
	{
		# Make sure we have enough space.
		if ($storage_size > $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size})
		{
			my $say_available_size = $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}})." (".$anvil->Convert->add_commas({number => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}}).")";
			my $say_requested_size = $anvil->Convert->bytes_to_human_readable({'bytes' => $storage_size})." (".$anvil->Convert->add_commas({number => $storage_size}).")";
			my $say_storage_group  = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{group_name};
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0194,!!server_name!".$server_name."!!,!!available_size!".$say_available_size."!!,!!requested_size!".$say_requested_size."!!,!!storage_group!".$say_storage_group."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0194", variables => { 
				server_name    => $server_name, 
				available_size => $say_available_size, 
				requested_size => $say_requested_size, 
				storage_group  => $say_storage_group, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Create the LV.
		my $shell_call = $anvil->data->{path}{exe}{lvcreate}." -l ".$extent_count." -n ".$server_name."_0 ".$vg_name;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code,
		}});
		
		if (not -e $lv_path)
		{
			# Something went wrong.
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0201,!!lv_path!".$lv_path."!!,!!lv_create!".$shell_call."!!,!!return_code!".$return_code."!!,!!output!".$output."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0201", variables => { 
				lv_path     => $lv_path, 
				lv_create   => $shell_call, 
				return_code => $return_code, 
				output      => $output, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Report 
		$anvil->Job->update_progress({
			progress => 35,
			message  => "job_0189,!!lv_path!".$lv_path."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0189", variables => { lv_path => $lv_path }});
	}
	
	# Create the resource config file now.
	$anvil->Database->get_hosts();
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		node1_short_host_name => $node1_short_host_name,
		node2_short_host_name => $node2_short_host_name,
	}});
	
	# Load the IPs so we can find which SN network and IPs to use 
	my $node1_sn_ip = "";
	my $node2_sn_ip = "";
	my $sn_network  = "";
	$anvil->Network->load_ips({host => $node1_short_host_name, host_uuid => $node1_host_uuid});
	$anvil->Network->load_ips({host => $node2_short_host_name, host_uuid => $node2_host_uuid});
	my $match = $anvil->Network->find_matches({
		first  => $node1_short_host_name,
		second => $node2_short_host_name, 
	});
	my $matched_ips = keys %{$match};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { matched_ips => $matched_ips }});
	foreach my $interface (sort {$a cmp $b} keys %{$match->{$node1_short_host_name}})
	{
		if ($interface =~ /sn/)
		{
			### TODO: This always chooses SN1 at this time, we need to support (later) VM 
			###       build-time SN selection when 2+ SNs exist.
			# Found an SN.
			$sn_network  = uc(($interface =~ /^(sn\d+)_/)[0]);
			$node1_sn_ip = $match->{$node1_short_host_name}{$interface}{ip};
			$node2_sn_ip = $match->{$node2_short_host_name}{$interface}{ip};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				sn_network  => $sn_network, 
				node1_sn_ip => $node1_sn_ip, 
				node2_sn_ip => $node2_sn_ip, 
			}});
		}
	}
	
	# Node 1 and Node 2 are set independent of which node we have.
	my $node1_lv_path = "";
	my $node2_lv_path = "";
	if ($host_uuid eq $node1_host_uuid)
	{
		# We're node 1
		$node1_lv_path = $lv_path;
		$node2_lv_path = $peer_lv_path;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			node1_lv_path => $node1_lv_path,
			node2_lv_path => $node2_lv_path,
		}});
	}
	else
	{
		# We're node 2
		$node1_lv_path = $peer_lv_path;
		$node2_lv_path = $lv_path;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			node1_lv_path => $node1_lv_path,
			node2_lv_path => $node2_lv_path,
		}});
	}
	
	my $drbd_res_body = $anvil->Words->string({key => "file_0001", variables => {
		server           => $server_name, 
		minor            => $drbd_minor, 
		tcp_port         => $drbd_tcp_port, 
		sn_network       => $sn_network,
		node1_sn_ip      => $node1_sn_ip,
		node2_sn_ip      => $node2_sn_ip, 
		node1_short_name => $node1_short_host_name, 
		node2_short_name => $node2_short_host_name, 
		node1_lv_path    => $node1_lv_path, 
		node2_lv_path    => $node2_lv_path, 
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_res_body => $drbd_res_body }});
	
	# Write the file, then call drbdadm to validate it.
	my $drbd_res_file =  $anvil->data->{path}{directories}{drbd_resources}."/".$server_name.".res";
	   $drbd_res_file =~ s/\/\//\//g;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_res_file => $drbd_res_file }});
	
	$problem = $anvil->Storage->write_file({
		overwrite => 1, 
		backup    => 1,
		file      => $drbd_res_file, 
		body      => $drbd_res_body, 
		mode      => "0644",
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	
	if (not -e $drbd_res_file)
	{
		# Failed to write the config file.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0202,!!drbd_res_file!".$drbd_res_file."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0202", variables => { drbd_res_file => $drbd_res_file }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Call drbdadm and verify it's good.
	$problem = $anvil->DRBD->gather_data({debug => 2});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});

	if ((not exists $anvil->data->{new}{resource}{$server_name}) or ($anvil->data->{new}{resource}{$server_name}{config_file} ne $drbd_res_file))
	{
		# Failed to load the resource.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0203,!!drbd_res_file!".$drbd_res_file."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0203", variables => { drbd_res_file => $drbd_res_file }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { initialize_drbd => $initialize_drbd }});
	if ($initialize_drbd)
	{
		# Create the DRBD metadata
		my $shell_call = $anvil->data->{path}{exe}{drbdadm}." -- --force create-md ".$server_name;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code,
		}});
		
		### Return codes
		# 0 == Success
		# 3 == Configuration not found.
		if ($return_code eq "3")
		{
			# Metadata creation failed.
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0204,!!return_code!".$return_code."!!,!!output!".$output."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0204", variables => { 
				return_code => $return_code,
				output      => $output, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0579", variables => { resource => $server_name }});
	}
	
	# Check to see if the resource needs to be started. It almost always will.
	$anvil->DRBD->get_status();
	my $startup_needed = 1;
	
	# Is the current resource up locally already? If it is, we're done. 
	my $role = defined $anvil->data->{drbd}{status}{$short_host_name}{resource}{$server_name}{role} ? $anvil->data->{drbd}{status}{$short_host_name}{resource}{$server_name}{role} : "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server => $server_name, 
		role   => $role,
	}});
	
	if ((lc($role) ne "secondary") && (lc($role) ne "primary"))
	{
		$startup_needed = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { startup_needed => $startup_needed }});
	}
	else
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0434", variables => { 
			resource => $server_name,
			role     => $role, 
		}});
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { startup_needed => $startup_needed }});
	if (not $startup_needed)
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0431"});
	}
	else
	{
		# Bring the drbd resource up.
		$anvil->DRBD->manage_resource({
			debug    => 2,
			resource => $server_name, 
			task     => "up",
		});
		
		my $waiting = 1;
		while($waiting)
		{
			# Reload the DRBD data 
			$anvil->DRBD->get_status();
			
			my $all_ready = 1;
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{drbd}{status}{$short_host_name}{resource}{$server_name}{devices}{volume}})
			{
				my $disk_state = $anvil->data->{drbd}{status}{$short_host_name}{resource}{$server_name}{devices}{volume}{$volume}{'disk-state'};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:short_host_name' => $short_host_name, 
					's2:server_name'     => $server_name, 
					's3:volume'          => $volume, 
					's4:disk_state'      => $disk_state,
				}});
				
				# Is the peer isn't connected (directly or by being in Sync), or this volume 
				# isn't UpToDate, we need to keep waiting.
				if ((lc($disk_state) ne "uptodate")   && 
				    (lc($disk_state) ne "consistent") && 
				    (lc($disk_state) ne "outdated")   && 
				    (lc($disk_state) ne "inconsistent"))
				{
					$all_ready = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_ready => $all_ready }});
				}
				
				# If we're not initializing DRBD, we'll wait until we're uptodate.
				if (not $initialize_drbd)
				{
					if (lc($disk_state) eq "diskless")
					{
						# This happens when the LV existed but there's no meta-data.
						$anvil->Job->update_progress({
							progress   => 100,
							message    => "error_0206,!!resource!".$server_name."!!", 
							job_status => "failed", 
						});
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0206", variables => { resource => $server_name }});
						$anvil->nice_exit({exit_code => 1});
					}
					if (lc($disk_state) ne "uptodate")
					{
						# Log why we're waiting.
						$all_ready = 0;
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0578", variables => { resource => $server_name }});
					}
				}
				
				# If we're initializing and the resource is 'Inconsistent', force the resource to primary
				if (($initialize_drbd) && (lc($disk_state) eq "inconsistent"))
				{
					# Create the DRBD metadata
					my $shell_call = $anvil->data->{path}{exe}{drbdadm}." primary ".$server_name." --force";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code,
					}});
					# Return code of '0' is success.
					if ($return_code)
					{
						# Failed to promote
						$anvil->Job->update_progress({
							progress   => 100,
							message    => "error_0205,!!return_code!".$return_code."!!,!!output!".$output."!!", 
							job_status => "failed", 
						});
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0205", variables => { 
							return_code => $return_code,
							output      => $output, 
						}});
						$anvil->nice_exit({exit_code => 1});
					}
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0580", variables => { resource => $server_name }});
					
					### TODO: Should we demote it again? For now, no.
				}
			}
			
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_ready => $all_ready }});
			if ($all_ready)
			{
				$waiting = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
			}
			
			if ($waiting)
			{
				sleep 1;
			}
		}
	}
	
	# If we're here, we can finally craft the 'virt-install' call!.

	return(0);
}

# Go through a series of questions to ask the user how they want to build their server.
sub interactive_question
{
	my ($anvil) = @_;
	
	# Do we know or can we find the Anvil! UUID?
	$anvil->data->{new_server}{anvil_uuid} = $anvil->data->{switches}{'anvil-uuid'} ? $anvil->data->{switches}{'anvil-uuid'} : "";
	$anvil->data->{new_server}{anvil_name} = $anvil->data->{switches}{'anvil-name'} ? $anvil->data->{switches}{'anvil-name'} : "";
	
	if ((not $anvil->data->{new_server}{anvil_uuid}) && (not $anvil->data->{new_server}{anvil_name}))
	{
		# Nothing given. Is this host a node, perhaps?
		my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
		
		if ($anvil_uuid)
		{
			$anvil->data->{new_server}{anvil_uuid} = $anvil_uuid;
			$anvil->data->{new_server}{anvil_name} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::anvil_name" => $anvil->data->{new_server}{anvil_name},
				"new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid},
			}});
		}
	}
	elsif (not $anvil->data->{new_server}{anvil_uuid})
	{
		$anvil->data->{new_server}{anvil_uuid} = $anvil->Cluster->get_anvil_uuid({anvil_name => $anvil->data->{new_server}{anvil_name}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid} }});
	}
	elsif (not $anvil->data->{new_server}{anvil_name})
	{
		$anvil->data->{new_server}{anvil_name} = $anvil->Cluster->get_anvil_name({anvil_uuid => $anvil->data->{new_server}{anvil_uuid}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "new_server::anvil_name" => $anvil->data->{new_server}{anvil_name} }});
	}
	
	$anvil->data->{new_server}{name} = $anvil->data->{switches}{name} ? $anvil->data->{switches}{name} : "";
	
	# If this is a node, load the anvil_uuid automatically.
	
	my $termios = new POSIX::Termios;
	$termios->getattr;
	my $ospeed = $termios->getospeed;
	
	my $terminal = Tgetent Term::Cap { TERM => undef, OSPEED => $ospeed };
	$terminal->Trequire(qw/ce ku kd/);
	
	interactive_ask_anvil_name($anvil, $terminal);
	interactive_ask_server_name($anvil, $terminal);
	interactive_ask_server_cpu($anvil, $terminal);
	interactive_ask_server_ram($anvil, $terminal);
	interactive_ask_server_storage_group($anvil, $terminal);
	interactive_ask_server_storage_size($anvil, $terminal);
	interactive_ask_server_install_media($anvil, $terminal);
	interactive_ask_server_driver_disc($anvil, $terminal);
	interactive_ask_server_confirm($anvil, $terminal);
	
	return(0);
}

sub interactive_ask_anvil_name
{
	my ($anvil, $terminal) = @_;
	
	$anvil->Database->get_anvils();
	
	my $retry = 0;
	while(1)
	{
		my $default_anvil = $anvil->data->{new_server}{anvil_name};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_anvil => $default_anvil }});
		if (not $default_anvil)
		{
			my $known_anvils = keys %{$anvil->data->{anvils}{anvil_name}};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { known_anvils => $known_anvils }});
			if (not $known_anvils)
			{
				print $anvil->Words->string({key => "job_0149"})."\n";
				$anvil->nice_exit({exit_code => 1});
			}
			elsif ($known_anvils == 1)
			{
				foreach my $anvil_name (keys %{$anvil->data->{anvils}{anvil_name}})
				{
					$default_anvil = $anvil_name;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_anvil => $default_anvil }});
				}
			}
		}
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $default_anvil }})."\n\n\n";
		
		# Show all the current server names.
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0152"})."\n\n";
		}
		print $anvil->Words->string({key => "job_0153"})."\n";
		foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
		{
			print "- ".$anvil_name.": - ".$anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description}."\n";
		}
		
		print $terminal->Tgoto('cm', 0, 2)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ((not $answer) && ($default_anvil))
		{
			$answer = $default_anvil;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		}
		
		# Reload in case a new anvil! was saved while we waited.
		$anvil->Database->get_anvils();
		if (exists $anvil->data->{anvils}{anvil_name}{$answer})
		{
			# Valid.
			$anvil->data->{new_server}{anvil_name} = $answer;
			$anvil->data->{new_server}{anvil_uuid} = $anvil->Cluster->get_anvil_uuid({anvil_name => $answer});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::anvil_name" => $anvil->data->{new_server}{anvil_name},
				"new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid},
			}});
			
			print $anvil->Words->string({key => "job_0154", variables => { 
				anvil_name => $anvil->data->{new_server}{anvil_name}, 
				anvil_uuid => $anvil->data->{new_server}{anvil_uuid}, 
			}})."\n";
			$anvil->Get->available_resources({
				debug      => 2,
				anvil_uuid => $anvil->data->{new_server}{anvil_uuid},
			});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	# Verify that the Anvil! has enough RAM and disk space.
	my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"anvil_resources::${anvil_uuid}::ram::available" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")",
	}});
	if ($anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available} < (1**20))
	{
		print $anvil->Words->string({key => "job_0155"})."\n";
		print $anvil->Words->string({key => "job_0156", variables => { available_ram => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}) }})."\n";
		$anvil->nice_exit({exit_code => 2});
	}
	
	return(0);
}

sub interactive_ask_server_name
{
	my ($anvil, $terminal) = @_;
	
	$anvil->Database->get_servers({debug => 2});
	
	### TODO: Figure out how many rows we have and break the server list into columns if too long.
	my $retry = 0;
	while(1)
	{
		my $default = "";
		if ($anvil->data->{switches}{name})
		{
			$default = $anvil->data->{switches}{name};
		}
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n\n\n";
		
		# Show all the current server names.
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0159"})."\n\n";
		}
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		print $anvil->Words->string({key => "job_0160", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
		{
			print "- ".$server_name."\n";
		}
		
		print $terminal->Tgoto('cm', 0, 3)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ((not $answer) && ($default))
		{
			$answer = $default;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		}
		
		# Reload in case a new anvil! was saved while we waited.
		$anvil->Database->get_servers();
		if (($answer) && (not exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$answer}))
		{
			# Valid.
			$anvil->data->{new_server}{name} = $answer;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::name" => $anvil->data->{new_server}{name},
			}});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_cpu
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		$anvil->Database->get_anvils();
		$anvil->Get->available_resources({anvil_uuid => $anvil_uuid});
		
		my $default_cpu = $anvil->data->{switches}{cpu};
		if (not $default_cpu)
		{
			# Default to 2, unless only one core is available.
			$default_cpu = $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads} == 1 ? 1 : 2;
		}
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $default_cpu }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0161", variables => { max_cores => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads} }})."\n\n";
		}
		print $anvil->Words->string({key => "job_0162", variables => { 
			cores   => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{cores},
			threads => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads},
		}})."\n";
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $dr1_host_uuid   = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
	
		print $anvil->Words->string({key => "job_0163", variables => { core => 1, model => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{cpu}{model} }})."\n";
		print $anvil->Words->string({key => "job_0163", variables => { core => 2, model => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node2_host_uuid}{cpu}{model} }})."\n";
		
		if ($anvil->data->{anvil_resources}{$anvil_uuid}{has_dr})
		{
			print $anvil->Words->string({key => "job_0164", variables => { 
				model   => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{cpu}{model},
				cores   => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$dr1_host_uuid}{cpu}{cores}, 
				threads => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$dr1_host_uuid}{cpu}{threads}, 
			}})."\n";
		}
		
		print $terminal->Tgoto('cm', 0, 4)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if ($answer eq "")
		{
			$answer = $default_cpu; 
		}
		if (($answer) && ($answer =~ /^\d+$/) && ($answer <= $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}))
		{
			# Valid.
			$anvil->data->{new_server}{cpu} = $answer;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::cpu" => $anvil->data->{new_server}{cpu},
			}});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_ram
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		$anvil->Database->get_anvils();
		$anvil->Get->available_resources({anvil_uuid => $anvil_uuid});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"anvil_resources::${anvil_uuid}::ram::available" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")",
		}});
		
		my $default_ram = $anvil->data->{switches}{ram};
		if (not $default_ram)
		{
			# Default to 2, unless only one core is available.
			my $say_2g = (2*(2**30));
			if ($anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available} > $say_2g)
			{
				$default_ram = $anvil->Convert->bytes_to_human_readable({'bytes' => $say_2g});
			}
			else
			{
				$default_ram = $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}});
			}
		}
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $default_ram }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0166", variables => { 
				ram_total     => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{hardware}}),
				ram_available => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}), 
			}})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $dr1_host_uuid   = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
		
		print $anvil->Words->string({key => "job_0167", variables => { 
			ram_available => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}, unit => "M"}).")", 
			ram_reserved  => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{reserved}}), 
			ram_allocated => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{allocated}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{allocated}, unit => "M"}).")",
			ram_node1     => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{ram}{hardware}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{ram}{hardware}, unit => "M"}).")",
			ram_node2     => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node2_host_uuid}{ram}{hardware}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node2_host_uuid}{ram}{hardware}, unit => "M"}).")",
		}})."\n";
		if ($anvil->data->{anvil_resources}{$anvil_uuid}{has_dr})
		{
			print $anvil->Words->string({key => "job_0168", variables => { 
				ram_available => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$dr1_host_uuid}{ram}{hardware}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$dr1_host_uuid}{ram}{hardware}, unit => "M"}).")",
			}})."\n";
		}
		
		print $terminal->Tgoto('cm', 0, 5)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if ($answer eq "")
		{
			$answer = $default_ram; 
		}
		if ($answer)
		{
			# Convert to bytes.
			my $answer_bytes = $anvil->Convert->human_readable_to_bytes({
				base2 => 1,
				size  => $answer, 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer_bytes => $answer_bytes }});
			if (($answer_bytes eq "!!error!!") or 
			    (not $answer_bytes)            or 
			    ($answer_bytes < (640*1024))   or 
			    ($answer_bytes > $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}))
			{
				# Invalid
				$retry = 1;
			}
			else
			{
				# Valid.
				$anvil->data->{new_server}{ram} = $answer_bytes;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"new_server::ram" => $anvil->data->{new_server}{ram},
				}});
				
				last;
			}
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_storage_group
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		$anvil->Database->get_anvils();
		$anvil->Get->available_resources({anvil_uuid => $anvil_uuid});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"anvil_resources::${anvil_uuid}::ram::available" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")",
		}});
		
		# I need a list of Storage groups,
		my $storage_groups        = [""];
		my $show_list             = "";
		my $position              = 0;
		my $default_storage_group = 1;
		foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}})
		{
			   $position++;
			my $storage_group_uuid = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid};
			my $vg_size            = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size};
			my $vg_free            = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
			my $dr_size            = 0;
			my $dr_free            = 0;
			if ($anvil->data->{anvil_resources}{$anvil_uuid}{has_dr})
			{
				$dr_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr};
				$dr_free = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr};
			}
			
			if ($anvil->data->{switches}{'storage-group'})
			{
				if (($anvil->data->{switches}{'storage-group'} eq $storage_group_uuid) or 
				    ($anvil->data->{switches}{'storage-group'} eq $storage_group_name))
				{
					$default_storage_group = $position;
				}
			}
			push @{$storage_groups}, $storage_group_uuid;
			$show_list .= " - ".$position.") ".$storage_group_name." (".$storage_group_uuid.")\n";
			$show_list .= $anvil->Words->string({key => "job_0169", variables => { 
				vg_free => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}),
				vg_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}),
				dr_free => $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->Convert->bytes_to_human_readable({'bytes' => $dr_free}) : "--",
				dr_size => $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->Convert->bytes_to_human_readable({'bytes' => $dr_size}) : "--",
			}})."\n";
		}
		
		if (not $default_storage_group)
		{
			# Default to 2, unless only one core is available.
			$default_storage_group = 1;
		}
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}}) }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $default_storage_group }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0171"})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $dr1_host_uuid   = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
		
		print $anvil->Words->string({key => "job_0171"})."\n";
		print $show_list."\n";
		
		print $terminal->Tgoto('cm', 0, 6)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if ($answer eq "")
		{
			$answer = $default_storage_group; 
		}
		if (($answer) && ($answer =~ /^\d+$/) && (exists $storage_groups->[$answer]) && ($storage_groups->[$answer]))
		{
			# Valid.
			$anvil->data->{new_server}{storage_group} = $storage_groups->[$answer];
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::storage_group" => $anvil->data->{new_server}{storage_group},
			}});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_storage_size
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		$anvil->Database->get_anvils();
		$anvil->Get->available_resources({anvil_uuid => $anvil_uuid});
		
		# I need a list of Storage groups,
		my $default_storage_size = 0;
		my $storage_group_uuid   = $anvil->data->{new_server}{storage_group};
		my $vg_free              = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
		   $default_storage_size = $anvil->data->{switches}{'storage-size'};
		if (not $default_storage_size)
		{
			if ($vg_free < (80 * (2**30)))
			{
				# Too small for default
				$default_storage_size = $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free});
			}
			else
			{
				$default_storage_size = "80 GiB";
			}
		}
		
		my $say_storage_group = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}}) }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
		print $anvil->Words->string({key => "job_0173", variables => { storage_size => $default_storage_size }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0174", variables => { max_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}) }})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $dr1_host_uuid   = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
		
		print $anvil->Words->string({key => "job_0175", variables => { 
			storage_group  => $say_storage_group,
			available_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}),
		}})."\n";
		
		print $terminal->Tgoto('cm', 0, 7)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if ($answer eq "")
		{
			$answer = $default_storage_size; 
		}
		if ($answer) 
		{
			# Is the size sensible?
			my $answer_bytes = $anvil->Convert->human_readable_to_bytes({
				base2 => 1,
				size  => $answer, 
			});
			# Make sure they've asked for at least 10 MiB
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer_bytes => $answer_bytes }});
			if (($answer_bytes eq "!!error!!") or 
			    (not $answer_bytes)            or 
			    ($answer_bytes < (10*(2**20))) or 
			    ($answer_bytes > $vg_free))
			{
				# Invalid
				$retry = 1;
			}
			else
			{
				$anvil->data->{new_server}{storage_size} = $answer_bytes;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"new_server::storage_size" => $anvil->data->{new_server}{storage_size},
				}});
				
				last;
			}
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_install_media
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		$anvil->Database->get_files();
		$anvil->Database->get_file_locations();
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		
		# Create an array of install ISOs. 
		my $install_isos = [""];
		my $iso_list     = "";
		my $position     = 0;
		foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{files}{file_name}})
		{
			my $file_uuid            = $anvil->data->{files}{file_name}{$file_name}{file_uuid};
			my $file_directory       = $anvil->data->{files}{file_name}{$file_name}{file_directory};
			my $file_size            = $anvil->data->{files}{file_name}{$file_name}{file_size};
			my $file_type            = $anvil->data->{files}{file_name}{$file_name}{file_type};
			my $file_path            = $file_directory."/".$file_name;
			my $file_location_uuid   = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
			my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				file_uuid            => $file_uuid, 
				file_name            => $file_name,
				file_directory       => $file_directory, 
				file_size            => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")",
				file_type            => $file_type, 
				file_path            => $file_path, 
				file_location_uuid   => $file_location_uuid, 
				file_location_active => $file_location_active, 
			}});
			
			next if $file_type ne "iso";
			next if not $file_location_active;
			
			# Still here?
			$position++;
			push @{$install_isos}, $file_uuid;
			$iso_list .= " - ".$position.") ".$file_name." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")\n";
		}
		
		my $storage_group_uuid = $anvil->data->{new_server}{storage_group};
		my $say_storage_group  = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
		my $say_storage_size   = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{storage_size}});
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}}) }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
		print $anvil->Words->string({key => "job_0173", variables => { storage_size => $say_storage_size }})."\n";
		print $anvil->Words->string({key => "job_0176", variables => { install_media => "" }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0177"})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $dr1_host_uuid   = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
		
		print $anvil->Words->string({key => "job_0178"})."\n";
		print $iso_list."\n";
		
		print $terminal->Tgoto('cm', 0, 8)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if (($answer) && ($answer =~ /^\d+$/) && (exists $install_isos->[$answer]) && ($install_isos->[$answer]))
		{
			# Valid.
			$anvil->data->{new_server}{install_media} = $install_isos->[$answer];
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::install_media" => $anvil->data->{new_server}{install_media},
			}});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_driver_disc
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		$anvil->Database->get_files();
		$anvil->Database->get_file_locations();
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		
		my $storage_group_uuid      = $anvil->data->{new_server}{storage_group};
		my $say_storage_group       = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
		my $say_storage_size        = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{storage_size}});
		my $install_media_file_uuid = $anvil->data->{new_server}{install_media};
		my $say_install_media       = $anvil->data->{files}{file_uuid}{$install_media_file_uuid}{file_name};
		
		# Create an array of install ISOs. 
		my $driver_isos = ["none"];
		my $iso_list     = $anvil->Words->string({key => "job_0179"})."\n";
		my $position     = 0;
		foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{files}{file_name}})
		{
			my $file_uuid            = $anvil->data->{files}{file_name}{$file_name}{file_uuid};
			my $file_directory       = $anvil->data->{files}{file_name}{$file_name}{file_directory};
			my $file_size            = $anvil->data->{files}{file_name}{$file_name}{file_size};
			my $file_type            = $anvil->data->{files}{file_name}{$file_name}{file_type};
			my $file_path            = $file_directory."/".$file_name;
			my $file_location_uuid   = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
			my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				file_uuid            => $file_uuid, 
				file_name            => $file_name,
				file_directory       => $file_directory, 
				file_size            => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")",
				file_type            => $file_type, 
				file_path            => $file_path, 
				file_location_uuid   => $file_location_uuid, 
				file_location_active => $file_location_active, 
			}});
			
			next if $file_type ne "iso";
			next if not $file_location_active;
			next if $file_uuid eq $install_media_file_uuid;
			
			# Still here?
			$position++;
			push @{$driver_isos}, $file_uuid;
			$iso_list .= " - ".$position.") ".$file_name." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")\n";
		}
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}}) }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
		print $anvil->Words->string({key => "job_0173", variables => { storage_size => $say_storage_size }})."\n";
		print $anvil->Words->string({key => "job_0176", variables => { install_media => $say_install_media }})."\n";
		print $anvil->Words->string({key => "job_0180", variables => { driver_disc => "#!string!unit_0005!#" }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0181"})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $dr1_host_uuid   = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
		
		print $anvil->Words->string({key => "job_0182"})."\n";
		print $iso_list."\n";
		
		print $terminal->Tgoto('cm', 0, 9)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		$answer = 0 if $answer eq "";
		
		if (($answer =~ /^\d+$/) && (exists $driver_isos->[$answer]) && ($driver_isos->[$answer]))
		{
			# Valid.
			$anvil->data->{new_server}{driver_disc} = $driver_isos->[$answer];
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::driver_disc" => $anvil->data->{new_server}{driver_disc},
			}});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_confirm
{
	my ($anvil, $terminal) = @_;
	
	$anvil->Database->get_anvils();
	$anvil->Database->get_files();
	$anvil->Database->get_file_locations();
	my $anvil_uuid      = $anvil->data->{new_server}{anvil_uuid};
	my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		
	my $storage_group_uuid      = $anvil->data->{new_server}{storage_group};
	my $say_storage_group       = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
	my $say_storage_size        = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{storage_size}});
	my $install_media_file_uuid = $anvil->data->{new_server}{install_media};
	my $say_install_media       = $anvil->data->{files}{file_uuid}{$install_media_file_uuid}{file_name};
	my $driver_disc_file_uuid   = $anvil->data->{new_server}{driver_disc};
	my $say_driver_disc         = $driver_disc_file_uuid eq "none" ? "#!string!unit_0005!#" : $anvil->data->{files}{file_uuid}{$driver_disc_file_uuid}{file_name};
	
	print $terminal->Tputs('cl');
	print $anvil->Words->string({key => "job_0150"})."\n";
	print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
	print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
	print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
	print $anvil->Words->string({key => "job_0165", variables => { ram => $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}}) }})."\n";
	print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
	print $anvil->Words->string({key => "job_0173", variables => { storage_size => $say_storage_size }})."\n";
	print $anvil->Words->string({key => "job_0176", variables => { install_media => $say_install_media }})."\n";
	print $anvil->Words->string({key => "job_0180", variables => { driver_disc => $say_driver_disc }})."\n";
	print $anvil->Words->string({key => "message_0206"})." ";
	my $answer = <STDIN>;
	chomp $answer;
	
	$answer = "y" if $answer eq "";
	
	if ($answer =~ /^y/i)
	{
		# Valid.
		my $job_data = "server_name=".$anvil->data->{new_server}{name}."
cpu_cores=".$anvil->data->{new_server}{cpu}."
ram=".$anvil->data->{new_server}{ram}."
storage_group_uuid=".$anvil->data->{new_server}{storage_group}."
storage_size=".$anvil->data->{new_server}{storage_size}."
install_iso=".$anvil->data->{new_server}{install_media}."
driver_iso=".$anvil->data->{new_server}{driver_disc};
		print $anvil->Words->string({key => "job_0183", variables => { job_data => $job_data }})."\n";
		
		# Register the job with the primary node on the Anvil! (or node 1, if neither node is primary).
		my $target_host_uuid = $anvil->Cluster->get_primary_host_uuid({debug => 2, anvil_uuid => $anvil_uuid});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_uuid => $target_host_uuid }});
		if (not $target_host_uuid)
		{
			$target_host_uuid = $node1_host_uuid;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_uuid => $target_host_uuid }});
		}
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-provision-server'}, 
			job_data        => $job_data, 
			job_name        => "server::provision", 
			job_title       => "job_0147", 
			job_description => "job_0148", 
			job_progress    => 0,
			job_host_uuid   => $target_host_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		print $anvil->Words->string({key => "job_0184", variables => { job_uuid => $job_uuid }})."\n";
	}
	else
	{
		# Abort
		print $anvil->Words->string({key => "message_0022"})."\n";
	}
	
	return(0);
}