anvil/tools/anvil-provision-server

#!/usr/bin/perl
# 
# This provisions a new virtual machine server. It handles creating the logical volumes, DRBD resources, 
# verifies the needed files are ready, creates the provision script, begins the installation, and adds the 
# new server to pacemaker.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Any problem that causes an early exit.
# 
# TODO: Support cloning; Example
#       - virt-clone --original-xml /mnt/shared/definitions/<source>.xml --name <new_server> --file <new_server's_drbd_path> --check path_exists=off
#       - Make VNC default
# 

use strict;
use warnings;
use Anvil::Tools;
require POSIX;
use Term::Cap;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

# Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is 
# passed directly, it will be used. Otherwise, the password will be read from the database.
$anvil->data->{switches}{anvil}           = "";
$anvil->data->{switches}{'anvil-name'}    = "";
$anvil->data->{switches}{'anvil-uuid'}    = "";
$anvil->data->{switches}{os}              = "";
$anvil->data->{switches}{cpu}             = "";
$anvil->data->{switches}{'job-uuid'}      = "";
$anvil->data->{switches}{name}            = "";
$anvil->data->{switches}{uuid}            = "";
$anvil->data->{switches}{ram}             = "";
$anvil->data->{switches}{'storage-group'} = "";
$anvil->data->{switches}{'storage-size'}  = "";
$anvil->data->{switches}{'use-image'}     = "";
$anvil->Get->switches;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
	'switches::anvil'         => $anvil->data->{switches}{anvil},
	'switches::anvil-name'    => $anvil->data->{switches}{'anvil-name'},
	'switches::anvil-uuid'    => $anvil->data->{switches}{'anvil-uuid'},
	'switches::os'            => $anvil->data->{switches}{os},
	'switches::cpu'           => $anvil->data->{switches}{cpu},
	'switches::job-uuid'      => $anvil->data->{switches}{'job-uuid'},
	'switches::name'          => $anvil->data->{switches}{name},
	'switches::uuid'          => $anvil->data->{switches}{uuid},
	'switches::ram'           => $anvil->data->{switches}{ram},
	'switches::storage-group' => $anvil->data->{switches}{'storage-group'},
	'switches::storage-size'  => $anvil->data->{switches}{'storage-size'},
}});

$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0306"});
	sleep 10;
	$anvil->nice_exit({exit_code => 1});
}

# If we don't have a job UUID, try to find one.
if (not $anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->data->{switches}{'job-uuid'} = $anvil->Job->get_job_uuid({program => $THIS_FILE});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }});
}

# If we still don't have a job-uuit, go into interactive mode.
if ($anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->Job->clear();
	$anvil->Job->get_job_details();
	$anvil->Job->update_progress({
		progress         => 1,
		job_picked_up_by => $$, 
		job_picked_up_at => time, 
		message          => "message_0190", 
	});
	
	# Job data will be in $anvil->data->{jobs}{job_data}
	run_jobs($anvil);
}
else
{
	# Interactive!
	interactive_question($anvil);
}

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This actually provisions a VM.
sub run_jobs
{
	my ($anvil) = @_;
	
	# If we're here, the job has been assigned to us, so we use our own anvil! uuid.
	$anvil->data->{job}{anvil_uuid} = $anvil->Cluster->get_anvil_uuid();
	$anvil->Get->available_resources({
		debug      => 2,
		anvil_uuid => $anvil->data->{job}{anvil_uuid},
	});
	
	$anvil->Database->get_hosts();
	$anvil->Database->get_files();
	$anvil->Database->get_file_locations();
	
	# This parses the jobs::job_data intp variables.
	parse_job_data($anvil);
	
	# Wait for the cluster to be up, if it's not.
	my $waiting          = 1;
	my $waiting_reported = 0;
	while($waiting)
	{
		my $problem = $anvil->Cluster->parse_cib({debug => 3});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		if ($problem)
		{
			# Cluster isn't up yet, wait.
			if (not $waiting_reported)
			{
				$anvil->Job->update_progress({
					progress   => 5,
					message    => "job_0275", 
				});
				$waiting_reported = 1;
			}
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0275"});
			sleep 10;
		}
		else
		{
			$waiting = 0;
			$anvil->Job->update_progress({
				progress   => 8,
				message    => "job_0276", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0276"});
		}
	}
	
	# Sanity checks passed
	$anvil->Job->update_progress({
		progress => 10,
		message  => "job_0185", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0185"});
	
	# This picks out the next free DRBD minor number and TCP port if it's not set in the job data.
	check_drbd_minor_and_port($anvil);
	
	# Report the DRBD minor and TCP port that we'll use.
	$anvil->Job->update_progress({
		progress => 20,
		message  => "job_0186,!!minor!".$anvil->data->{job}{drbd_minor}."!!,!!port!".$anvil->data->{job}{drbd_tcp_port}."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0186", variables => {
		minor => $anvil->data->{job}{drbd_minor}, 
		port  => $anvil->data->{job}{drbd_tcp_port}, 
	}});
	
	# Prepare some variables
	$anvil->Database->get_anvils();
	my $anvil_uuid      = $anvil->data->{job}{anvil_uuid};
	my $host_uuid       = $anvil->Get->host_uuid();
	my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $peer_host_uuid  = $host_uuid eq $node1_host_uuid ? $node2_host_uuid : $node1_host_uuid;
	my $peer_name       = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{host_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		host_uuid       => $host_uuid,
		peer_host_uuid  => $peer_host_uuid, 
		peer_name       => $peer_name, 
		node1_host_uuid => $node1_host_uuid,
		node2_host_uuid => $node2_host_uuid,
	}});
	
	$anvil->data->{job}{node1_host_uuid}       = $node1_host_uuid;
	$anvil->data->{job}{node2_host_uuid}       = $node2_host_uuid;
	$anvil->data->{job}{peer_short_name}       = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{short_host_name};
	$anvil->data->{job}{short_host_name}       = $anvil->Get->short_host_name;
	$anvil->data->{job}{node1_short_host_name} = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	$anvil->data->{job}{node2_short_host_name} = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	$anvil->data->{job}{drbd_local_node_id}    = $host_uuid eq $node1_host_uuid ? 0 : 1;
	$anvil->data->{job}{drbd_peer_node_id}     = $host_uuid eq $node1_host_uuid ? 1 : 0;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		'job::node1_host_uuid'       => $anvil->data->{job}{node1_host_uuid}, 
		'job::node2_host_uuid'       => $anvil->data->{job}{node2_host_uuid}, 
		'job::node1_short_host_name' => $anvil->data->{job}{node1_short_host_name},
		'job::node2_short_host_name' => $anvil->data->{job}{node2_short_host_name},
		'job::short_host_name'       => $anvil->data->{job}{short_host_name}, 
		'job::drbd_local_node_id'    => $anvil->data->{job}{drbd_local_node_id}, 
		'job::drbd_peer_node_id'     => $anvil->data->{job}{drbd_peer_node_id}, 
	}});
	
	# We convert to extents as it ensure clean boundaries and, being based on bytes in both cases, gets 
	# us as close as we can to what the user asked for. The 'int()' always rounds down, so we don't ever 
	# ask for one more extent than is available by accident.
	my $storage_group_uuid = $anvil->data->{job}{storage_group_uuid};
	my $vg_name            = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_name};
	my $lv_path            = "/dev/".$vg_name."/".$anvil->data->{job}{server_name}."_0";
	my $peer_vg_name       = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$peer_host_uuid}{vg_name};
	my $peer_lv_path       = "/dev/".$peer_vg_name."/".$anvil->data->{job}{server_name}."_0";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		vg_name      => $vg_name, 
		lv_path      => $lv_path, 
		peer_lv_path => $peer_lv_path
	}});
	
	# Node 1 and Node 2 are set independent of which node we have.
	$anvil->data->{job}{node1_lv_path} = "";
	$anvil->data->{job}{node2_lv_path} = "";
	if ($host_uuid eq $anvil->data->{job}{node1_host_uuid})
	{
		# We're node 1
		$anvil->data->{job}{node1_lv_path} = $lv_path;
		$anvil->data->{job}{node2_lv_path} = $peer_lv_path;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			'job::node1_lv_path' => $anvil->data->{job}{node1_lv_path},
			'job::node2_lv_path' => $anvil->data->{job}{node2_lv_path},
		}});
	}
	else
	{
		# We're node 2
		$anvil->data->{job}{node1_lv_path} = $peer_lv_path;
		$anvil->data->{job}{node2_lv_path} = $lv_path;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			'job::node1_lv_path' => $anvil->data->{job}{node1_lv_path},
			'job::node2_lv_path' => $anvil->data->{job}{node2_lv_path},
		}});
	}
	
	# If we're not the peer, create the peer's job.
	if (not $anvil->data->{job}{peer_mode})
	{
		my $job_data   =  $anvil->data->{jobs}{job_data}."\n";
		   $job_data   .= "peer_mode=true\n";
		   $job_data   .= "drbd_minor=".$anvil->data->{job}{drbd_minor}."\n";
		   $job_data   .= "drbd_tcp_port=".$anvil->data->{job}{drbd_tcp_port}."\n";
		my ($job_uuid) =  $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-provision-server'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::provision", 
			job_title       => "job_0147", 
			job_description => "job_0148", 
			job_progress    => 0,
			job_host_uuid   => $peer_host_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		$anvil->Job->update_progress({
			progress => 25,
			message  => "job_0188,!!job_uuid!".$job_uuid."!!,!!peer_name!".$peer_name."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0188", variables => {
			job_uuid  => $job_uuid, 
			peer_name => $peer_name, 
		}});
	}
	
	# If the LV exists, we won't initialize the DRBD resource. If the LV exists but isn't a DRBD backing 
	# device, this will cause the resource to not come up, but it's better than risking data loss.
	create_lv($anvil, $lv_path);
	
	# Load the IPs so we can find which SN network and IPs to use 
	get_sn_details($anvil);
	
	# Create the DRBD resource file
	create_resource_file($anvil);
	
	# Create the DRBD metadata, if needed.
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::initialize_drbd' => $anvil->data->{job}{initialize_drbd} }});
	create_md($anvil) if $anvil->data->{job}{initialize_drbd};
	
	### NOTE: This function also forces the resource to be UpToDate if this is not a peer job and we 
	###       created the LV.
	# Check to see if the resource needs to be started. It almost always will.
	startup_resource($anvil);
	
	# If we're here, we can finally craft the 'virt-install' call!.
	if ($anvil->data->{job}{peer_mode})
	{
		# Wait until we have seen the definition in the database and written to disk
		write_definition($anvil);
	}
	else
	{
		provision_server($anvil);
	}
	
	# Add the server to the cluster.
	add_server_to_cluster($anvil);
	
	# Done!
	$anvil->Job->update_progress({
		progress => 100,
		message  => "job_0202", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0202"});

	return(0);
}

sub add_server_to_cluster
{
	my ($anvil) = @_;
	
	# Parse the cluster
	$anvil->Job->update_progress({
		progress => 90,
		message  => "job_0207", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0207"});
	
	# Is our peer in the cluster? For that matter, are we?
	my $problem = $anvil->Cluster->add_server({server_name => $anvil->data->{job}{server_name}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	if ($problem)
	{
		# Something went wrong
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0215", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0215"});
		$anvil->nice_exit({exit_code => 1});
	}
	
	return(0);
}

# This loops until the definition file is found in the database and then writes it out to disk.
sub write_definition
{
	my ($anvil) = @_;
	
	my $anvil_uuid  = $anvil->data->{job}{anvil_uuid};
	my $server_name = $anvil->data->{job}{server_name};
	my $xml_file    = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { xml_file => $xml_file }});
	if (not -f $xml_file)
	{
		# Tell the user we're going to wait until we find the server in the database.
		$anvil->Job->update_progress({
			progress => 70,
			message  => "job_0205", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0205"});
		
		while (not -f $xml_file)
		{
			$anvil->Database->get_servers();
			$anvil->Database->get_server_definitions();
			
			my $server_uuid       = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
			my $server_definition = exists $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid} ? $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml} : "";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition => $server_definition }});
			if ($server_definition)
			{
				# Write it!
				my $return = $anvil->Storage->write_file({
					body      => $server_definition,
					file      => $xml_file,
					overwrite => 1,
				});
				
				if (-f $xml_file)
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'return' => $return }});
					$anvil->Job->update_progress({
						progress => 80,
						message  => "job_0206,!!file!".$xml_file."!!", 
					});
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0206", variables => { file => $xml_file }});
				}
			}
			else
			{
				sleep 1;
			}
		}
	}
	
	# Lastly, if there's a variable recording the shell call used to provision this server, write it out.
	my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_uuid => $server_uuid }});
	my ($variable_value, $variable_uuid, $mtime, $modified_date) = $anvil->Database->read_variable({
		debug                 => 2, 
		variable_name         => "server::provision_script", 
		variable_source_uuid  => $server_uuid, 
		variable_source_table => "servers", 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		variable_value => $variable_value,
		variable_uuid  => $variable_uuid, 
	}});
	if (($variable_value) && ($variable_value =~ /virt-install/gs))
	{
		my $provision_file = $anvil->data->{path}{directories}{shared}{provision}."/".$server_name.".sh";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { provision_file => $provision_file }});
		my $problem = $anvil->Storage->write_file({
			overwrite => 1, 
			backup    => 1,
			file      => $provision_file, 
			body      => $variable_value, 
			mode      => "0755",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	}
	
	# The peer is done, it'll pick up the XML definition when ScanCore runs 
	$anvil->Job->update_progress({
		progress => 100,
		message  => "job_0204", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0204"});
	
	return(0);
}

# This creates the virt-install call and creates the VM.
sub provision_server
{
	my ($anvil) = @_;
	
	# The logic behind this is explained here: https://libvirt.org/formatdomain.html#elementsTime
	my $server       = $anvil->data->{job}{server_name};
	my $clock_offset = $anvil->data->{job}{os} =~ /win/i ? "localtime" : "utc";
	my $say_memory   =  sprintf("%.0f", ($anvil->data->{job}{ram} /= (2 ** 20)));
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		clock_offset => $clock_offset,
		say_memory   => $say_memory,
	}});
	
	### TODO: Support user-selected IFN. For now, we hard-code it to 'ifn_bridge1' Also allow a MAC to be
	###       set with '--network ifn1_bridge1[:mac],ifn2_bridge1[:mac]...'.
	###       Support disk images (ie: sysprep) via '--import'. The device used for booting is the first 
	###       device specified via "--disk"
	###       Consider support for TPM, RNG and watchdog devices
	my $server_uuid = $anvil->data->{job}{server_uuid} ? $anvil->data->{job}{server_uuid} : $anvil->Get->uuid();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"job::os"   => $anvil->data->{job}{os}, 
		server_uuid => $server_uuid, 
	}});
	my $disk_bus  = ",target.bus=virtio";
	my $nic_model = ",model.type=virtio";
	if ($anvil->data->{job}{os} eq "win7")
	{
		$disk_bus  = "";
		$nic_model = ",model.type=e1000e";
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		disk_bus  => $disk_bus,
		nic_model => $nic_model, 
	}});
	
	my $shell_call =  $anvil->data->{path}{exe}{'virt-install'}." --connect qemu:///system \\\n";
	   $shell_call .= "--name ".$server." \\\n";
	   $shell_call .= " --os-variant ".$anvil->data->{job}{os}." \\\n";
	   $shell_call .= " --memory ".$say_memory." \\\n";
	   $shell_call .= " --events on_poweroff=destroy,on_reboot=restart \\\n";
	   $shell_call .= " --vcpus ".$anvil->data->{job}{cpu_cores}.",sockets=1,cores=".$anvil->data->{job}{cpu_cores}." \\\n";
	   $shell_call .= " --cpu host \\\n";
	   $shell_call .= " --network bridge=ifn1_bridge1".$nic_model." \\\n";
	   $shell_call .= " --graphics vnc \\\n";
	   $shell_call .= " --sound ich9 \\\n";
	   $shell_call .= " --clock offset=".$clock_offset.",rtc_tickpolicy=catchup \\\n";
	   $shell_call .= " --boot menu=on \\\n";
	   $shell_call .= " --disk path=/dev/drbd/by-res/".$server."/0".$disk_bus.",driver.io=threads,cache=writeback,driver.discard=unmap,boot.order=1 \\\n";
	   $shell_call .= " --disk path=".$anvil->data->{job}{install_iso_path}.",device=cdrom,shareable=on,boot.order=2 \\\n";
	   $shell_call .= " --uuid=".$server_uuid." \\\n";
	if ($anvil->data->{job}{driver_iso_path})
	{
		$shell_call .= " --disk path=".$anvil->data->{job}{driver_iso_path}.",device=cdrom,shareable=on,boot.order=3 --force \\\n";
	}
	$shell_call .= " --noautoconsole --wait -1 > /var/log/anvil-server_".$server.".log\n";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	
	$anvil->Job->update_progress({
		progress => 70,
		message  => "job_0199,!!shell_call!".$shell_call."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0199", variables => { shell_call => $shell_call }});
	
	# Write the command out to /mnt/shared/provision/
	my $provision_file = $anvil->data->{path}{directories}{shared}{provision}."/".$server.".sh";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	my $problem = $anvil->Storage->write_file({
		overwrite => 1, 
		backup    => 1,
		file      => $provision_file, 
		body      => $shell_call, 
		mode      => "0755",
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	
	# Also, record the provision command 
	my $variable_uuid = $anvil->Database->insert_or_update_variables({
		variable_name         => "server::provision_script",
		variable_value        => $shell_call,
		variable_default      => "", 
		variable_description  => "striker_0293", 
		variable_section      => "servers", 
		variable_source_uuid  => $server_uuid, 
		variable_source_table => "servers", 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }});
	
	# Call as a background process.
	my ($handle, $return_code) = $anvil->System->call({
		background  => 1, 
		shell_call  => $shell_call,
		stdout_file => "/var/log/anvil-server_".$server.".stdout",
		stderr_file => "/var/log/anvil-server_".$server.".stderr", 
	});
	my $pid = $handle->pid();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		pid         => $pid,
		return_code => $return_code,
	}});
	
	$anvil->Job->update_progress({
		progress => 80,
		message  => "job_0200", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0200"});
	
	# Loop for up to 10 seconds waiting to see the server start running.
	my $wait_until = time + 60;
	my $waiting    = 1;
	my $status     = "";
	while($waiting)
	{
		my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." list --all"});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code,
		}});
		
		foreach my $line (split/\n/, $output)
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
			if ($line =~ /.*?$server\s+(.*)$/)
			{
				$status = $1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { status => $status }});
				if ($status eq "running")
				{
					$waiting = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				}
				elsif ($status eq "shut off")
				{
					# Try to boot it.
					my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." start ".$server});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code,
					}});
				}
			}
			if (($waiting) && (time > $wait_until))
			{
				# Didn't boot, give up
				$waiting = 0;
				$anvil->Job->update_progress({
					progress   => 100,
					message    => "error_0210,!!status!".$status."!!", 
					job_status => "failed", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0210", variables => { status => $status }});
				$anvil->nice_exit({exit_code => 1});
			}
			sleep 1;
		}
	}
	
	$anvil->Job->update_progress({
		progress => 85,
		message  => "job_0201", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0201"});
	
	# Dump the server's XML.
	(my $server_definition_xml, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." dumpxml ".$server});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server_definition_xml => $server_definition_xml,
		return_code           => $return_code,
	}});

	$problem = $anvil->Server->parse_definition({
		server     => $server, 
		source     => "from_virsh",
		definition => $server_definition_xml, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	if ($problem)
	{
		# Something went wrong with the provision
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0209,!!shell_call!".$shell_call."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0209", variables => { shell_call => $shell_call }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Write out the definition file.
	my $definition_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server.".xml";
	$problem = $anvil->Storage->write_file({
		overwrite => 1, 
		backup    => 1,
		file      => $definition_file, 
		body      => $server_definition_xml, 
		mode      => "0644",
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
	# Add the server to the 'servers' table
	my $anvil_uuid      = $anvil->data->{job}{anvil_uuid};
	my $short_host_name = $anvil->Get->short_host_name;
	my $got_server_uuid = $anvil->Database->insert_or_update_servers({
		server_uuid           => $server_uuid, 
		server_name           => $server, 
		server_state          => "running", 
		server_anvil_uuid     => $anvil_uuid, 
		server_host_uuid      => $anvil->Get->host_uuid, 
		server_ram_in_use     => $anvil->data->{job}{ram}, 
		server_configured_ram => $anvil->data->{job}{ram}, 
		server_boot_time      => time,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { got_server_uuid => $got_server_uuid }});
	
	# Save the definition.
	my $server_definition_uuid = $anvil->Database->insert_or_update_server_definitions({
		server_definition_server_uuid => $got_server_uuid, 
		server_definition_xml         => $server_definition_xml, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition_uuid => $server_definition_uuid }});
	
	# Undefine the server
	(my $output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." undefine ".$server});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code,
	}});
	
	return(0);
}

# This starts up the DRBD resource, if needed.
sub startup_resource
{
	my ($anvil) = @_;
	
	$anvil->DRBD->get_status();
	my $startup_needed = 1;
	
	# Is the current resource up locally already? If it is, we're done. 
	my $server          = $anvil->data->{job}{server_name};
	my $short_host_name = $anvil->data->{job}{short_host_name};
	my $role = defined $anvil->data->{drbd}{status}{$short_host_name}{resource}{$server}{role} ? $anvil->data->{drbd}{status}{$short_host_name}{resource}{$server}{role} : "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		'job::server' => $anvil->data->{job}{server_name}, 
		role   => $role,
	}});
	
	if ((lc($role) ne "secondary") && (lc($role) ne "primary"))
	{
		$startup_needed = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { startup_needed => $startup_needed }});
	}
	else
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0434", variables => { 
			resource => $anvil->data->{job}{server_name},
			role     => $role, 
		}});
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { startup_needed => $startup_needed }});
	if (not $startup_needed)
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0431"});
	}
	else
	{
		# Bring the drbd resource up.
		$anvil->Job->update_progress({
			progress => 52,
			message  => "job_0192", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0192"});
		$anvil->DRBD->manage_resource({
			debug    => 2,
			resource => $anvil->data->{job}{server_name}, 
			task     => "up",
		});
		
		my $waiting = 1;
		while($waiting)
		{
			# Reload the DRBD data 
			$anvil->DRBD->get_status();
			
			my $server    = $anvil->data->{job}{server_name};
			my $all_ready = 1;
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{drbd}{status}{$short_host_name}{resource}{$server}{devices}{volume}})
			{
				my $disk_state = $anvil->data->{drbd}{status}{$short_host_name}{resource}{$server}{devices}{volume}{$volume}{'disk-state'};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:short_host_name'  => $short_host_name, 
					's2:job::server_name' => $anvil->data->{job}{server_name}, 
					's3:volume'           => $volume, 
					's4:disk_state'       => $disk_state,
				}});
				
				# Is the peer isn't connected (directly or by being in Sync), or this volume 
				# isn't UpToDate, we need to keep waiting.
				if ((lc($disk_state) ne "uptodate")   && 
				    (lc($disk_state) ne "consistent") && 
				    (lc($disk_state) ne "outdated")   && 
				    (lc($disk_state) ne "inconsistent"))
				{
					$all_ready = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_ready => $all_ready }});
					
					$anvil->Job->update_progress({
						progress => 54,
						message  => "job_0193,!!volume!".$volume."!!,!!disk_state!".$disk_state."!!", 
					});
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0193", variables => { 
						volume     => $volume,
						disk_state => $disk_state,
					}});
				}
				
				# If we're not initializing DRBD, we'll wait until we're uptodate.
				if (not $anvil->data->{job}{initialize_drbd})
				{
					if (lc($disk_state) eq "diskless")
					{
						# This happens when the LV existed but there's no meta-data.
						$anvil->Job->update_progress({
							progress   => 100,
							message    => "error_0206,!!resource!".$anvil->data->{job}{server_name}."!!", 
							job_status => "failed", 
						});
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0206", variables => { resource => $anvil->data->{job}{server_name} }});
						$anvil->nice_exit({exit_code => 1});
					}
					if (lc($disk_state) ne "uptodate")
					{
						# Log why we're waiting.
						$all_ready = 0;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_ready => $all_ready }});
						
						$anvil->Job->update_progress({
							progress => 56,
							message  => "job_0194,!!resource!".$anvil->data->{job}{server_name}."!!", 
						});
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0194", variables => { resource => $anvil->data->{job}{server_name} }});
					}
				}
				
				# If we're initializing and the resource is 'Inconsistent', force the resource to primary
				if (($anvil->data->{job}{initialize_drbd}) && (not $anvil->data->{job}{peer_mode}) && (lc($disk_state) eq "inconsistent"))
				{
					### NOTE: '<server> 1' is the resource followed by the peer ID number (1).
					# To force to primary, we first need to temporarily disable fencing.
					$anvil->Job->update_progress({
						progress => 58,
						message  => "job_0195", 
					});
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0195"});
					
 					my $shell_call = $anvil->data->{path}{exe}{drbdsetup}." net-options ".$anvil->data->{job}{server_name}." ".$anvil->data->{job}{drbd_peer_node_id}." --set-defaults --_name=".$anvil->data->{job}{peer_short_name}." --protocol=C --fencing=dont-care";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code,
					}});
					
					# Now force it to primary.
					$shell_call = $anvil->data->{path}{exe}{drbdadm}." primary ".$anvil->data->{job}{server_name}." --force";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code,
					}});
					# Return code of '0' is success.
					if ($return_code)
					{
						# Failed to promote
						$anvil->Job->update_progress({
							progress   => 100,
							message    => "error_0205,!!return_code!".$return_code."!!,!!output!".$output."!!", 
							job_status => "failed", 
						});
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0205", variables => { 
							return_code => $return_code,
							output      => $output, 
						}});
						$anvil->nice_exit({exit_code => 1});
					}
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0580", variables => { resource => $anvil->data->{job}{server_name} }});
					
					# set the fencing back
					$shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$anvil->data->{job}{server_name};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code,
					}});
					
					# Update and see if the resource's disk state is now UpToDate.
					$anvil->DRBD->get_status();
					my $server = $anvil->data->{job}{server_name};
					my $disk_state = $anvil->data->{drbd}{status}{$short_host_name}{resource}{$server}{devices}{volume}{$volume}{'disk-state'};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:short_host_name'  => $short_host_name, 
						's2:job::server_name' => $anvil->data->{job}{server_name}, 
						's3:volume'           => $volume, 
						's4:disk_state'       => $disk_state,
					}});
					
					if (lc($disk_state) ne "uptodate")
					{
						# Something went wrong.
						$anvil->Job->update_progress({
							progress   => 100,
							message    => "error_0207,!!resource!".$anvil->data->{job}{server_name}."!!,!!disk_state!".$disk_state."!!", 
							job_status => "failed", 
						});
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0207", variables => { 
							resource   => $anvil->data->{job}{server_name},
							disk_state => $disk_state, 
						}});
						$anvil->nice_exit({exit_code => 1});
					}
				}
			}
			
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_ready => $all_ready }});
			if ($all_ready)
			{
				$waiting = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
			}
			
			if ($waiting)
			{
				sleep 10;
			}
		}
	}
	
	$anvil->Job->update_progress({
		progress => 60,
		message  => "job_0203,!!resource!".$anvil->data->{job}{server_name}."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0203", variables => { resource => $anvil->data->{job}{server_name} }});
	
	return(0);
}

# Thos does the job of creating the metadata.
sub create_md
{
	my ($anvil) = @_;
	
	### NOTE: The '--max-peers=3' is needed to make space for future DR additions
	# Create the DRBD metadata
	my $shell_call = $anvil->data->{path}{exe}{drbdadm}." --force create-md --max-peers=3 ".$anvil->data->{job}{server_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	
	my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code,
	}});
	
	### Return codes
	# 0 == Success
	# 1 == ?
	# 3 == Configuration not found.
	if ($return_code)
	{
		# Metadata creation failed.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0204,!!return_code!".$return_code."!!,!!output!".$output."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0204", variables => { 
			return_code => $return_code,
			output      => $output, 
		}});
		$anvil->nice_exit({exit_code => 1});
	}
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0579", variables => { resource => $anvil->data->{job}{server_name} }});
	
	$anvil->Job->update_progress({
		progress => 50,
		message  => "job_0191,!!resource!".$anvil->data->{job}{server_name}."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0191", variables => { resource => $anvil->data->{job}{server_name} }});
	
	return(0);
}

# This finds which SN network and IPs we're using.
sub get_sn_details
{
	my ($anvil) = @_;
	
	$anvil->data->{job}{node1_sn_ip} = "";
	$anvil->data->{job}{node2_sn_ip} = "";
	$anvil->data->{job}{sn_network}  = "";
	$anvil->Network->load_ips({host => $anvil->data->{job}{node1_short_host_name}, host_uuid => $anvil->data->{job}{node1_host_uuid}});
	$anvil->Network->load_ips({host => $anvil->data->{job}{node2_short_host_name}, host_uuid => $anvil->data->{job}{node2_host_uuid}});
	my $match = $anvil->Network->find_matches({
		first  => $anvil->data->{job}{node1_short_host_name},
		second => $anvil->data->{job}{node2_short_host_name}, 
	});
	
	my $node1_short_host_name = $anvil->data->{job}{node1_short_host_name};
	my $node2_short_host_name = $anvil->data->{job}{node2_short_host_name};
	my $matched_ips = keys %{$match};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { matched_ips => $matched_ips }});
	foreach my $interface (sort {$a cmp $b} keys %{$match->{$node1_short_host_name}})
	{
		if ($interface =~ /sn/)
		{
			### TODO: This always chooses SN1 at this time, we need to support (later) VM 
			###       build-time SN selection when 2+ SNs exist.
			# Found an SN.
			$anvil->data->{job}{sn_network}  = uc(($interface =~ /^(sn\d+)_/)[0]);
			$anvil->data->{job}{node1_sn_ip} = $match->{$node1_short_host_name}{$interface}{ip};
			$anvil->data->{job}{node2_sn_ip} = $match->{$node2_short_host_name}{$interface}{ip};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				sn_network  => $anvil->data->{job}{sn_network}, 
				node1_sn_ip => $anvil->data->{job}{node1_sn_ip}, 
				node2_sn_ip => $anvil->data->{job}{node2_sn_ip}, 
			}});
		}
	}
	
	return(0);
}

# The generates and saves the DRBD resource config.
sub create_resource_file
{
	my ($anvil) = @_;
	
	# Create the resource configuration body
	my $drbd_res_body = $anvil->Words->string({key => "file_0001", variables => {
		server           => $anvil->data->{job}{server_name}, 
		minor            => $anvil->data->{job}{drbd_minor}, 
		tcp_port         => $anvil->data->{job}{drbd_tcp_port}, 
		sn_network       => $anvil->data->{job}{sn_network},
		node1_sn_ip      => $anvil->data->{job}{node1_sn_ip},
		node2_sn_ip      => $anvil->data->{job}{node2_sn_ip}, 
		node1_short_name => $anvil->data->{job}{node1_short_host_name}, 
		node2_short_name => $anvil->data->{job}{node2_short_host_name}, 
		node1_lv_path    => $anvil->data->{job}{node1_lv_path}, 
		node2_lv_path    => $anvil->data->{job}{node2_lv_path}, 
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_res_body => $drbd_res_body }});
	
	# Write the file, then call drbdadm to validate it.
	my $drbd_res_file =  $anvil->data->{path}{directories}{drbd_resources}."/".$anvil->data->{job}{server_name}.".res";
	   $drbd_res_file =~ s/\/\//\//g;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_res_file => $drbd_res_file }});
	
	my $problem = $anvil->Storage->write_file({
		overwrite => 1, 
		backup    => 1,
		file      => $drbd_res_file, 
		body      => $drbd_res_body, 
		mode      => "0644",
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	
	if (not -e $drbd_res_file)
	{
		# Failed to write the config file.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0202,!!drbd_res_file!".$drbd_res_file."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0202", variables => { drbd_res_file => $drbd_res_file }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Call drbdadm and verify it's good.
	$problem = $anvil->DRBD->gather_data({debug => 2});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});

	# Verify that the resource is now shown in drbdadm
	my $server = $anvil->data->{job}{server_name};
	if ((not exists $anvil->data->{new}{resource}{$server}) or ($anvil->data->{new}{resource}{$server}{config_file} ne $drbd_res_file))
	{
		# Failed to load the resource.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0203,!!drbd_res_file!".$drbd_res_file."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0203", variables => { drbd_res_file => $drbd_res_file }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Manually call 'scan-drbd' to make sure that the new data is in scan_drbd_resources so that any 
	my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd";
	if (-f $shell_call)
	{
		$anvil->Job->update_progress({
			progress => 35,
			message  => "job_0218", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0218"});
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code,
		}});
	}
	
	$anvil->Job->update_progress({
		progress => 40,
		message  => "job_0190,!!resource!".$anvil->data->{job}{server_name}."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0190", variables => { resource => $anvil->data->{job}{server_name} }});
	
	return(0);
}

# This creates a logical volume
sub create_lv
{
	my ($anvil, $lv_path) = @_;
	
	$anvil->data->{job}{initialize_drbd} = 1;
	if (-e $lv_path)
	{
		# Don't create and don't initialize.
		$anvil->data->{job}{initialize_drbd} = 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::initialize_drbd' => $anvil->data->{job}{initialize_drbd} }});
		
		# Tell the user this might be an issue.
		$anvil->Job->update_progress({
			progress => 30,
			message  => "job_0187,!!lv_path!".$lv_path."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0187", variables => { lv_path => $lv_path }});
	}
	else
	{
		# Make sure we have enough space.
		my $anvil_uuid         = $anvil->data->{job}{anvil_uuid};
		my $storage_group_uuid = $anvil->data->{job}{storage_group_uuid};
		if ($anvil->data->{job}{storage_size} > $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size})
		{
			my $say_available_size = $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}})." (".$anvil->Convert->add_commas({number => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}}).")";
			my $say_requested_size = $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{job}{storage_size}})." (".$anvil->Convert->add_commas({number => $anvil->data->{job}{storage_size}}).")";
			my $say_storage_group  = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{group_name};
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0194,!!server_name!".$anvil->data->{job}{server_name}."!!,!!available_size!".$say_available_size."!!,!!requested_size!".$say_requested_size."!!,!!storage_group!".$say_storage_group."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0194", variables => { 
				server_name    => $anvil->data->{job}{server_name}, 
				available_size => $say_available_size, 
				requested_size => $say_requested_size, 
				storage_group  => $say_storage_group, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Create the LV.
		my $host_uuid    = $anvil->Get->host_uuid();
		my $vg_name      = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_name};
		my $extent_size  = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_extent_size};
		my $extent_count = int($anvil->data->{job}{storage_size} / $extent_size);
		my $shell_call   = $anvil->data->{path}{exe}{lvcreate}." -l ".$extent_count." -n ".$anvil->data->{job}{server_name}."_0 ".$vg_name." -y";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:vg_name'      => $vg_name, 
			's2:extent_size'  => $extent_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $extent_size}).")", 
			's3:extent_count' => $extent_count, 
			's4:shell_call'   => $shell_call,
		}});
		
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code,
		}});
		
		if (not -e $lv_path)
		{
			# Something went wrong.
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0201,!!lv_path!".$lv_path."!!,!!lv_create!".$shell_call."!!,!!return_code!".$return_code."!!,!!output!".$output."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0201", variables => { 
				lv_path     => $lv_path, 
				lv_create   => $shell_call, 
				return_code => $return_code, 
				output      => $output, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Report 
		$anvil->Job->update_progress({
			progress => 30,
			message  => "job_0189,!!lv_path!".$lv_path."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0189", variables => { lv_path => $lv_path }});
	}
	
	return(0);
}

# This picks out the next free DRBD minor number and TCP port if it's not set in the job data.
sub check_drbd_minor_and_port
{
	my ($anvil) = @_;

	# Read in VG information;
	my $problem = $anvil->Storage->get_storage_group_details({storage_group_uuid => $anvil->data->{job}{storage_group_uuid}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	if ($problem)
	{
		# Failed to collect storage group data.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0199,!!storage_group_uuid!".$anvil->data->{job}{storage_group_uuid}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0199", variables => { storage_group_uuid => $anvil->data->{job}{storage_group_uuid} }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	if (not $anvil->data->{job}{peer_mode})
	{
		# We're primary, so query the minor number and TCP port
		# The peer must use the TCP and minor as set in the job
		($anvil->data->{job}{drbd_minor}, $anvil->data->{job}{drbd_tcp_port}) = $anvil->DRBD->get_next_resource({
			debug         => 2, 
			anvil_uuid    => $anvil->data->{job}{anvil_uuid},
			resource_name => $anvil->data->{job}{server_name}, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			'job::drbd_minor'    => $anvil->data->{job}{drbd_minor},
			'job::drbd_tcp_port' => $anvil->data->{job}{drbd_tcp_port}, 
		}});
	}
	
	# If we don't have a DRBD minor or TCP port, we're stuck.
	if (($anvil->data->{job}{drbd_minor} eq "") or ($anvil->data->{job}{drbd_tcp_port} eq ""))
	{
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0200", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0200"});
		$anvil->nice_exit({exit_code => 1});
	}
	
	return(0);
}

# This parses and verifies the job data
sub parse_job_data
{
	my ($anvil) = @_;
	
	$anvil->data->{job}{server_name}        = "";
	$anvil->data->{job}{cpu_cores}          = "";
	$anvil->data->{job}{ram}                = "";
	$anvil->data->{job}{storage_group_uuid} = "";
	$anvil->data->{job}{storage_size}       = "";
	$anvil->data->{job}{install_iso_uuid}   = "";
	$anvil->data->{job}{driver_iso_uuid}    = "";
	$anvil->data->{job}{drbd_minor}         = "";
	$anvil->data->{job}{drbd_tcp_port}      = "";
	$anvil->data->{job}{peer_mode}          = 0;
	foreach my $line (split/\n/, $anvil->data->{jobs}{job_data})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
		if ($line =~ /server_name=(.*)$/)
		{
			$anvil->data->{job}{server_name} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::server_name' => $anvil->data->{job}{server_name} }});
		}
		if ($line =~ /server_uuid=(.*)$/)
		{
			$anvil->data->{job}{server_uuid} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::server_uuid' => $anvil->data->{job}{server_uuid} }});
		}
		if ($line =~ /cpu_cores=(.*)$/)
		{
			$anvil->data->{job}{cpu_cores} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::cpu_cores' => $anvil->data->{job}{cpu_cores} }});
		}
		if ($line =~ /ram=(.*)$/)
		{
			$anvil->data->{job}{ram} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::ram' => $anvil->data->{job}{ram} }});
		}
		if ($line =~ /storage_group_uuid=(.*)$/)
		{
			$anvil->data->{job}{storage_group_uuid} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::storage_group_uuid' => $anvil->data->{job}{storage_group_uuid} }});
		}
		if ($line =~ /storage_size=(.*)$/)
		{
			$anvil->data->{job}{storage_size} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::storage_size' => $anvil->data->{job}{storage_size} }});
		}
		if ($line =~ /install_iso=(.*)$/)
		{
			$anvil->data->{job}{install_iso_uuid} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::install_iso_uuid' => $anvil->data->{job}{install_iso_uuid} }});
		}
		if ($line =~ /driver_iso=(.*)$/)
		{
			$anvil->data->{job}{driver_iso_uuid} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::driver_iso_uuid' => $anvil->data->{job}{driver_iso_uuid} }});
		}
		if ($line =~ /peer_mode=true$/)
		{
			# We'll ONLY setup our DRBD resource, nothing else.
			$anvil->data->{job}{peer_mode} = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::peer_mode' => $anvil->data->{job}{peer_mode} }});
		}
		if ($line =~ /drbd_minor=(.*)$/)
		{
			$anvil->data->{job}{drbd_minor} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::drbd_minor' => $anvil->data->{job}{drbd_minor} }});
		}
		if ($line =~ /drbd_tcp_port=(.*)$/)
		{
			$anvil->data->{job}{drbd_tcp_port} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::drbd_tcp_port' => $anvil->data->{job}{drbd_tcp_port} }});
		}
		if ($line =~ /os=(.*)$/)
		{
			$anvil->data->{job}{os} = $1;
			$anvil->data->{job}{os} =~ s/^\s+//;
			$anvil->data->{job}{os} =~ s/\s+?//;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::os' => $anvil->data->{job}{os} }});
		}
	}
	
	# We need a server name and storage group UUID regardless of which mode we're in.
	if (not $anvil->data->{job}{server_name})
	{
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0187,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0187", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }});
		$anvil->nice_exit({exit_code => 1});
	}
	if (not $anvil->data->{job}{storage_group_uuid})
	{
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0192,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0192", variables => { 
			server_name => $anvil->data->{job}{server_name}, 
			job_uuid    => $anvil->data->{switches}{'job-uuid'},
		}});
		$anvil->nice_exit({exit_code => 1});
	}
	if (not $anvil->data->{job}{storage_size})
	{
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0193,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0193", variables => { 
			server_name => $anvil->data->{job}{server_name}, 
			job_uuid    => $anvil->data->{switches}{'job-uuid'},
		}});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# If we're the peer, there's only a few things we need to check.
	if (not $anvil->data->{job}{peer_mode})
	{
		# The server should already exist on the peer. All we need to do is create our LV, create the
		# DRBD resource, bring the resource up, and save the XML definition file.
		
		# Is the server name unique?
		$anvil->Database->get_servers();
		my $anvil_uuid = $anvil->data->{job}{anvil_uuid};
		my $server     = $anvil->data->{job}{server_name};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			anvil_uuid => $anvil_uuid,
			server     => $server, 
		}});
		if (exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server})
		{
			# Is this name used by a server marked as DELETED?
			my $server_uuid  = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server}{server_uuid};
			my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				server_uuid  => $server_uuid,
				server_state => $server_state, 
			}});
			if ($server_state ne "DELETED")
			{
				# Duplicate name
				$anvil->Job->update_progress({
					progress   => 100,
					message    => "error_0198,!!server_name!".$anvil->data->{job}{server_name}."!!", 
					job_status => "failed", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0198", variables => { server_name => $anvil->data->{job}{server_name} }});
				$anvil->nice_exit({exit_code => 1});
			}
		}

		if (not $anvil->data->{job}{server_name})
		{
			# No server name given
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0188,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0188", variables => { 
				server_name => $anvil->data->{job}{server_name}, 
				job_uuid    => $anvil->data->{switches}{'job-uuid'},
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		elsif ($anvil->data->{job}{cpu_cores} > $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads})
		{
			# Too many cores requested
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0189,!!server_name!".$anvil->data->{job}{server_name}."!!,!!available_cores!".$anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}."!!,!!requested_cores!".$anvil->data->{job}{cpu_cores}."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0189", variables => { 
				server_name     => $anvil->data->{job}{server_name}, 
				available_cores => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}, 
				requested_cores => $anvil->data->{job}{server_name}, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		if (not $anvil->data->{job}{ram})
		{
			# No requested RAM
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0190,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0190", variables => { 
				server_name => $anvil->data->{job}{server_name}, 
				job_uuid    => $anvil->data->{switches}{'job-uuid'},
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		elsif ($anvil->data->{job}{ram} > $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available})
		{
			# Too many cores requested
			my $say_available_ram = $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}})." (".$anvil->Convert->add_commas({number => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")";
			my $say_requested_ram = $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{job}{ram}})." (".$anvil->Convert->add_commas({number => $anvil->data->{job}{ram}}).")"; 
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0191,!!server_name!".$anvil->data->{job}{server_name}."!!,!!available_ram!".$say_available_ram."!!,!!requested_ram!".$say_requested_ram."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0191", variables => { 
				server_name   => $anvil->data->{job}{server_name}, 
				available_ram => $say_available_ram, 
				requested_ram => $say_requested_ram, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		
		if (not $anvil->data->{job}{install_iso_uuid})
		{
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0195,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0195", variables => { 
				server_name => $anvil->data->{job}{server_name}, 
				job_uuid    => $anvil->data->{switches}{'job-uuid'},
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		my $install_iso_uuid = $anvil->data->{job}{install_iso_uuid};
		my $install_iso      = $anvil->data->{files}{file_uuid}{$install_iso_uuid}{file_directory}."/".$anvil->data->{files}{file_uuid}{$install_iso_uuid}{file_name};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			install_iso_uuid => $install_iso_uuid,
			install_iso      => $install_iso, 
		}});
		if (-e $install_iso)
		{
			$anvil->data->{job}{install_iso_path} = $install_iso;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				'job::install_iso_path' => $anvil->data->{job}{install_iso_path}, 
			}});
		}
		else
		{
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0196,!!server_name!".$anvil->data->{job}{server_name}."!!,!!install_iso!".$install_iso."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0196", variables => { 
				server_name => $anvil->data->{job}{server_name}, 
				install_iso => $install_iso,
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		# Driver disc is optional.
		$anvil->data->{new_server}{driver_iso_path} = "";
		if (($anvil->data->{job}{driver_iso_uuid}) && ($anvil->data->{job}{driver_iso_uuid} ne "none"))
		{
			my $driver_iso_uuid = $anvil->data->{job}{driver_iso_uuid};
			my $driver_iso      = $anvil->data->{files}{file_uuid}{$driver_iso_uuid}{file_directory}."/".$anvil->data->{files}{file_uuid}{$driver_iso_uuid}{file_name};
			if (-e $driver_iso)
			{
				$anvil->data->{job}{driver_iso_path} = $driver_iso;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					'job::driver_iso_path' => $anvil->data->{job}{driver_iso_path}, 
				}});
			}
			else
			{
				$anvil->Job->update_progress({
					progress   => 100,
					message    => "error_0197,!!server_name!".$anvil->data->{job}{server_name}."!!,!!driver_iso!".$driver_iso."!!", 
					job_status => "failed", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0197", variables => { 
					server_name => $anvil->data->{job}{server_name}, 
					driver_iso => $driver_iso,
				}});
				$anvil->nice_exit({exit_code => 1});
			}
		}
		if (not $anvil->data->{job}{os})
		{
			# No requested RAM
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0208,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0208", variables => { 
				server_name => $anvil->data->{job}{server_name}, 
				job_uuid    => $anvil->data->{switches}{'job-uuid'},
			}});
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	return(0);
}

# Go through a series of questions to ask the user how they want to build their server.
sub interactive_question
{
	my ($anvil) = @_;
	
	# If 'switches::anvil' is set, see if it's a UUID and then set either 'anvil-uuid' or 'anvil-name'.
	if (($anvil->data->{switches}{anvil}) && (not $anvil->data->{switches}{'anvil-uuid'}) && (not $anvil->data->{switches}{'anvil-name'}))
	{
		if ($anvil->Validate->uuid({uuid => $anvil->data->{switches}{anvil}}))
		{
			$anvil->data->{switches}{'anvil-uuid'} = $anvil->data->{switches}{anvil};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid},
			}});
		}
		else
		{
			$anvil->data->{switches}{'anvil-name'} = $anvil->data->{switches}{anvil};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::anvil_name" => $anvil->data->{new_server}{anvil_name},
			}});
		}
	}
	
	# Do we know or can we find the Anvil! UUID?
	$anvil->data->{new_server}{anvil_uuid} = $anvil->data->{switches}{'anvil-uuid'} ? $anvil->data->{switches}{'anvil-uuid'} : "";
	$anvil->data->{new_server}{anvil_name} = $anvil->data->{switches}{'anvil-name'} ? $anvil->data->{switches}{'anvil-name'} : "";
	
	if ((not $anvil->data->{new_server}{anvil_uuid}) && (not $anvil->data->{new_server}{anvil_name}))
	{
		# Nothing given. Is this host a node, perhaps?
		my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
		
		if ($anvil_uuid)
		{
			$anvil->data->{new_server}{anvil_uuid} = $anvil_uuid;
			$anvil->data->{new_server}{anvil_name} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::anvil_name" => $anvil->data->{new_server}{anvil_name},
				"new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid},
			}});
		}
	}
	elsif (not $anvil->data->{new_server}{anvil_uuid})
	{
		$anvil->data->{new_server}{anvil_uuid} = $anvil->Cluster->get_anvil_uuid({anvil_name => $anvil->data->{new_server}{anvil_name}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid} }});
	}
	elsif (not $anvil->data->{new_server}{anvil_name})
	{
		$anvil->data->{new_server}{anvil_name} = $anvil->Cluster->get_anvil_name({anvil_uuid => $anvil->data->{new_server}{anvil_uuid}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "new_server::anvil_name" => $anvil->data->{new_server}{anvil_name} }});
	}
	
	$anvil->data->{new_server}{name} = $anvil->data->{switches}{name} ? $anvil->data->{switches}{name} : "";
	$anvil->data->{new_server}{uuid} = $anvil->data->{switches}{uuid} ? $anvil->data->{switches}{uuid} : "";
	
	# If this is a node, load the anvil_uuid automatically.
	
	my $termios = new POSIX::Termios;
	$termios->getattr;
	my $ospeed = $termios->getospeed;
	
	my $terminal = Tgetent Term::Cap { TERM => undef, OSPEED => $ospeed };
	$terminal->Trequire(qw/ce ku kd/);
	
	interactive_ask_anvil_name($anvil, $terminal);
	interactive_ask_server_name($anvil, $terminal);
	interactive_ask_server_cpu($anvil, $terminal);
	interactive_ask_server_ram($anvil, $terminal);
	interactive_ask_server_storage_group($anvil, $terminal);
	interactive_ask_server_storage_size($anvil, $terminal);
	interactive_ask_server_install_media($anvil, $terminal);
	interactive_ask_server_driver_disc($anvil, $terminal);
	interactive_ask_server_os($anvil, $terminal);
	interactive_ask_server_confirm($anvil, $terminal);
	
	return(0);
}

sub interactive_ask_anvil_name
{
	my ($anvil, $terminal) = @_;
	
	$anvil->Database->get_anvils();
	
	my $retry = 0;
	while(1)
	{
		my $default_anvil = $anvil->data->{new_server}{anvil_name};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_anvil => $default_anvil }});
		if (not $default_anvil)
		{
			my $known_anvils = keys %{$anvil->data->{anvils}{anvil_name}};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { known_anvils => $known_anvils }});
			if (not $known_anvils)
			{
				print $anvil->Words->string({key => "job_0149"})."\n";
				$anvil->nice_exit({exit_code => 1});
			}
			elsif ($known_anvils == 1)
			{
				foreach my $anvil_name (keys %{$anvil->data->{anvils}{anvil_name}})
				{
					$default_anvil = $anvil_name;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_anvil => $default_anvil }});
				}
			}
		}
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $default_anvil }})."\n\n\n";
		
		# Show all the current server names.
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0152"})."\n\n";
		}
		print $anvil->Words->string({key => "job_0153"})."\n";
		foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
		{
			print "- ".$anvil_name.": - ".$anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description}."\n";
		}
		
		print $terminal->Tgoto('cm', 0, 2)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ((not $answer) && ($default_anvil))
		{
			$answer = $default_anvil;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		}
		
		# Reload in case a new anvil! was saved while we waited.
		$anvil->Database->get_anvils();
		if (exists $anvil->data->{anvils}{anvil_name}{$answer})
		{
			# Valid.
			$anvil->data->{new_server}{anvil_name} = $answer;
			$anvil->data->{new_server}{anvil_uuid} = $anvil->Cluster->get_anvil_uuid({anvil_name => $answer});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::anvil_name" => $anvil->data->{new_server}{anvil_name},
				"new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid},
			}});
			
			print $anvil->Words->string({key => "job_0154", variables => { 
				anvil_name => $anvil->data->{new_server}{anvil_name}, 
				anvil_uuid => $anvil->data->{new_server}{anvil_uuid}, 
			}})."\n";
			my $problem = $anvil->Get->available_resources({
				debug      => 2,
				anvil_uuid => $anvil->data->{new_server}{anvil_uuid},
			});
			if ($problem eq "!!no_data!!")
			{
				# We can't proceed.
				print $anvil->Words->string({key => "job_0217"})."\n";
				$anvil->nice_exit({exit_code => 1});
			}
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	# Verify that the Anvil! has enough RAM and disk space.
	my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"anvil_resources::${anvil_uuid}::ram::available" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")",
	}});
	if ($anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available} < (1**20))
	{
		print $anvil->Words->string({key => "job_0155"})."\n";
		print $anvil->Words->string({key => "job_0156", variables => { available_ram => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}) }})."\n";
		$anvil->nice_exit({exit_code => 2});
	}
	
	return(0);
}

sub interactive_ask_server_name
{
	my ($anvil, $terminal) = @_;
	
	$anvil->Database->get_servers({debug => 2});
	
	### TODO: Figure out how many rows we have and break the server list into columns if too long.
	my $retry     = 0;
	my $duplicate = "";
	while(1)
	{
		my $default = "";
		if ($anvil->data->{switches}{name})
		{
			$default = $anvil->data->{switches}{name};
		}
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n\n\n";
		
		# Show all the current server names.
		if ($retry)
		{
			if ($duplicate)
			{
				print $anvil->Words->string({key => "job_0219", variables => { server_name => $duplicate }})."\n\n";
				$duplicate = "";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { duplicate => $duplicate }});
			}
			else
			{
				print $anvil->Words->string({key => "job_0159"})."\n\n";
			}
		}
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		print $anvil->Words->string({key => "job_0160", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
		{
			my $server_uuid  = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
			my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				server_uuid  => $server_uuid,
				server_state => $server_state, 
			}});
			if ($server_state eq "DELETED")
			{
				### NOTE: This could get cluttered, so for now we'll not show them.
				#print $anvil->Words->string({key => "message_0220", variables => { server_name => $server_name }})."\n";
			}
			else
			{
				print $anvil->Words->string({key => "message_0219", variables => { 
					server_name  => $server_name,
					server_state => $server_state, 
				}})."\n";
			}
		}
		
		print $terminal->Tgoto('cm', 0, 3)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ((not $answer) && ($default))
		{
			$answer = $default;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		}
		
		# Reload in case a new anvil! was saved while we waited.
		$anvil->Database->get_servers();
		if ($answer)
		{
			# Duplicate?
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
			if (exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$answer})
			{
				my $server_uuid  = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$answer}{server_uuid};
				my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					server_uuid  => $server_uuid,
					server_state => $server_state, 
				}});
				if ($server_state eq "DELETED")
				{
					# Valid, we can re-use deleted server names. We'll also re-use the 
					# UUID, if the user didn't specifically specify a UUID.
					$anvil->data->{new_server}{name} = $answer;
					$anvil->data->{new_server}{uuid} = $server_uuid if not $anvil->data->{new_server}{uuid};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"new_server::name" => $anvil->data->{new_server}{name},
						"new_server::uuid" => $anvil->data->{new_server}{uuid},
					}});
				}
				else
				{
					# Invalid, duplicate.
					$duplicate = $answer;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { duplicate => $duplicate }});
				}
			}
			else
			{
				# Valid.
				$anvil->data->{new_server}{name} = $answer;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"new_server::name" => $anvil->data->{new_server}{name},
				}});
			}
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_cpu
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		$anvil->Database->get_anvils();
		$anvil->Get->available_resources({anvil_uuid => $anvil_uuid});
		
		my $default_cpu = $anvil->data->{switches}{cpu};
		if (not $default_cpu)
		{
			# Default to 2, unless only one core is available.
			$default_cpu = $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads} == 1 ? 1 : 2;
		}
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $default_cpu }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0161", variables => { max_cores => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads} }})."\n\n";
		}
		print $anvil->Words->string({key => "job_0162", variables => { 
			cores   => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{cores},
			threads => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads},
		}})."\n";
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $dr1_host_uuid   = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
	
		print $anvil->Words->string({key => "job_0163", variables => { core => 1, model => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{cpu}{model} }})."\n";
		print $anvil->Words->string({key => "job_0163", variables => { core => 2, model => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node2_host_uuid}{cpu}{model} }})."\n";
		
		if ($anvil->data->{anvil_resources}{$anvil_uuid}{has_dr})
		{
			print $anvil->Words->string({key => "job_0164", variables => { 
				model   => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{cpu}{model},
				cores   => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$dr1_host_uuid}{cpu}{cores}, 
				threads => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$dr1_host_uuid}{cpu}{threads}, 
			}})."\n";
		}
		
		print $terminal->Tgoto('cm', 0, 4)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if ($answer eq "")
		{
			$answer = $default_cpu; 
		}
		if (($answer) && ($answer =~ /^\d+$/) && ($answer <= $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}))
		{
			# Valid.
			$anvil->data->{new_server}{cpu} = $answer;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::cpu" => $anvil->data->{new_server}{cpu},
			}});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_ram
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		$anvil->Database->get_anvils();
		$anvil->Get->available_resources({anvil_uuid => $anvil_uuid});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"anvil_resources::${anvil_uuid}::ram::available" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")",
		}});
		
		my $default_ram = $anvil->data->{switches}{ram};
		if (not $default_ram)
		{
			# Default to 2, unless only one core is available.
			my $say_2g = (2*(2**30));
			if ($anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available} > $say_2g)
			{
				$default_ram = $anvil->Convert->bytes_to_human_readable({'bytes' => $say_2g});
			}
			else
			{
				$default_ram = $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}});
			}
		}
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $default_ram }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0166", variables => { 
				ram_total     => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{hardware}}),
				ram_available => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}), 
			}})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $dr1_host_uuid   = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
		
		print $anvil->Words->string({key => "job_0167", variables => { 
			ram_available => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}, unit => "M"}).")", 
			ram_reserved  => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{reserved}}), 
			ram_allocated => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{allocated}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{allocated}, unit => "M"}).")",
			ram_node1     => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{ram}{hardware}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{ram}{hardware}, unit => "M"}).")",
			ram_node2     => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node2_host_uuid}{ram}{hardware}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node2_host_uuid}{ram}{hardware}, unit => "M"}).")",
		}})."\n";
		if ($anvil->data->{anvil_resources}{$anvil_uuid}{has_dr})
		{
			print $anvil->Words->string({key => "job_0168", variables => { 
				ram_available => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$dr1_host_uuid}{ram}{hardware}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$dr1_host_uuid}{ram}{hardware}, unit => "M"}).")",
			}})."\n";
		}
		
		print $terminal->Tgoto('cm', 0, 5)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if ($answer eq "")
		{
			$answer = $default_ram; 
		}
		if ($answer)
		{
			# Convert to bytes.
			my $answer_bytes = $anvil->Convert->human_readable_to_bytes({
				base2 => 1,
				size  => $answer, 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer_bytes => $answer_bytes }});
			if (($answer_bytes eq "!!error!!") or 
			    (not $answer_bytes)            or 
			    ($answer_bytes < (640*1024))   or 
			    ($answer_bytes > $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}))
			{
				# Invalid
				$retry = 1;
			}
			else
			{
				# Valid.
				$anvil->data->{new_server}{ram} = $answer_bytes;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"new_server::ram" => $anvil->data->{new_server}{ram},
				}});
				
				last;
			}
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_storage_group
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		$anvil->Database->get_anvils();
		$anvil->Get->available_resources({debug => 2, anvil_uuid => $anvil_uuid});
		
		# I need a list of Storage groups,
		my $say_ram               = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}});
		my $storage_groups        = [""];
		my $show_list             = "";
		my $position              = 0;
		my $default_storage_group = 1;
		foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}})
		{
			   $position++;
			my $storage_group_uuid = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid};
			my $vg_size            = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size};
			my $vg_free            = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
			my $dr_size            = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr}   : 0;
			my $dr_free            = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr} : 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:storage_group_name' => $storage_group_name,
				's2:storage_group_uuid' => $storage_group_uuid, 
				's3:vg_size'            => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")",
				's4:vg_free'            => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")",
				's5:dr_size'            => $dr_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_size}).")",
				's6:dr_free'            => $dr_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_free}).")",
			}});
			
			if ($anvil->data->{switches}{'storage-group'})
			{
				if (($anvil->data->{switches}{'storage-group'} eq $storage_group_uuid) or 
				    ($anvil->data->{switches}{'storage-group'} eq $storage_group_name))
				{
					$default_storage_group = $position;
				}
			}
			push @{$storage_groups}, $storage_group_uuid;
			$show_list .= " - ".$position.") ".$storage_group_name." (".$storage_group_uuid.")\n";
			$show_list .= $anvil->Words->string({key => "job_0169", variables => { 
				vg_free => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}),
				vg_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}),
				dr_free => $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->Convert->bytes_to_human_readable({'bytes' => $dr_free}) : "--",
				dr_size => $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->Convert->bytes_to_human_readable({'bytes' => $dr_size}) : "--",
			}})."\n";
		}
		
		if (not $default_storage_group)
		{
			# Default to 2, unless only one core is available.
			$default_storage_group = 1;
		}
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $say_ram }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $default_storage_group }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0171"})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $dr1_host_uuid   = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
		
		print $anvil->Words->string({key => "job_0171"})."\n";
		print $show_list."\n";
		
		print $terminal->Tgoto('cm', 0, 6)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if ($answer eq "")
		{
			$answer = $default_storage_group; 
		}
		if (($answer) && ($answer =~ /^\d+$/) && (exists $storage_groups->[$answer]) && ($storage_groups->[$answer]))
		{
			# Valid.
			$anvil->data->{new_server}{storage_group} = $storage_groups->[$answer];
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::storage_group" => $anvil->data->{new_server}{storage_group},
			}});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_storage_size
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid         = $anvil->data->{new_server}{anvil_uuid};
		my $storage_group_uuid = $anvil->data->{new_server}{storage_group};
		my $storage_group_name = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
		my $vg_size            = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size};
		my $vg_free            = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
		my $dr_size            = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr}   : 0;
		my $dr_free            = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr} : 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:anvil_uuid'         => $anvil_uuid,
			's2:storage_group_name' => $storage_group_name,
			's3:storage_group_uuid' => $storage_group_uuid, 
			's4:vg_size'            => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")",
			's5:vg_free'            => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")",
			's6:dr_size'            => $dr_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_size}).")",
			's7:dr_free'            => $dr_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_free}).")",
		}});
		
		$anvil->Database->get_anvils();
		$anvil->Database->get_storage_group_data();
		$anvil->Get->available_resources({debug => 2, anvil_uuid => $anvil_uuid});
		
		$vg_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size};
		$vg_free = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
		$dr_size = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr}   : 0;
		$dr_free = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr} : 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:vg_size' => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")",
			's2:vg_free' => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")",
			's3:dr_size' => $dr_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_size}).")",
			's4:dr_free' => $dr_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_free}).")",
		}});
		
		# I need a list of Storage groups,
		my $default_storage_size = 0;
		   $vg_free              = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
		   $default_storage_size = $anvil->data->{switches}{'storage-size'};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			vg_free              => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free})." (".$anvil->Convert->add_commas({number => $vg_free}).")",
			default_storage_size => $default_storage_size, 
		}});
		   
		if (not $default_storage_size)
		{
			if ($vg_free < (80 * (2**30)))
			{
				# Too small for default
				$default_storage_size = $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_storage_size => $default_storage_size }});
			}
			else
			{
				$default_storage_size = "80 GiB";
			}
		}
		
		my $say_ram           = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}});
		my $say_storage_group = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $say_ram }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
		print $anvil->Words->string({key => "job_0173", variables => { storage_size => $default_storage_size }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0174", variables => { max_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}) }})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $dr1_host_uuid   = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
		
		print $anvil->Words->string({key => "job_0175", variables => { 
			storage_group  => $say_storage_group,
			available_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}),
		}})."\n";
		
		print $terminal->Tgoto('cm', 0, 7)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if ($answer eq "")
		{
			$answer = $default_storage_size; 
		}
		if ($answer) 
		{
			# Is the size sensible?
			my $answer_bytes = $anvil->Convert->human_readable_to_bytes({
				base2 => 1,
				size  => $answer, 
			});
			# Make sure they've asked for at least 10 MiB
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer_bytes => $answer_bytes }});
			if (($answer_bytes eq "!!error!!") or 
			    (not $answer_bytes)            or 
			    ($answer_bytes < (10*(2**20))) or 
			    ($answer_bytes > $vg_free))
			{
				# Invalid
				$retry = 1;
			}
			else
			{
				$anvil->data->{new_server}{storage_size} = $answer_bytes;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"new_server::storage_size" => $anvil->data->{new_server}{storage_size},
				}});
				
				last;
			}
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_install_media
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		$anvil->Database->get_files();
		$anvil->Database->get_file_locations();
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		
		# Create an array of install ISOs. 
		my $install_isos = [""];
		my $iso_list     = "";
		my $position     = 0;
		foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{files}{file_name}})
		{
			my $file_uuid            = $anvil->data->{files}{file_name}{$file_name}{file_uuid};
			my $file_directory       = $anvil->data->{files}{file_name}{$file_name}{file_directory};
			my $file_size            = $anvil->data->{files}{file_name}{$file_name}{file_size};
			my $file_type            = $anvil->data->{files}{file_name}{$file_name}{file_type};
			my $file_path            = $file_directory."/".$file_name;
			my $file_location_uuid   = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
			my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				file_uuid            => $file_uuid, 
				file_name            => $file_name,
				file_directory       => $file_directory, 
				file_size            => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")",
				file_type            => $file_type, 
				file_path            => $file_path, 
				file_location_uuid   => $file_location_uuid, 
				file_location_active => $file_location_active, 
			}});
			
			next if $file_type ne "iso";
			next if not $file_location_active;
			
			# Still here?
			$position++;
			push @{$install_isos}, $file_uuid;
			$iso_list .= " - ".$position.") ".$file_name." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")\n";
		}
		
		my $say_ram            = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}});
		my $storage_group_uuid = $anvil->data->{new_server}{storage_group};
		my $say_storage_group  = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
		my $say_storage_size   = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{storage_size}});
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $say_ram }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
		print $anvil->Words->string({key => "job_0173", variables => { storage_size => $say_storage_size }})."\n";
		print $anvil->Words->string({key => "job_0176", variables => { install_media => "" }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0177"})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $dr1_host_uuid   = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
		
		print $anvil->Words->string({key => "job_0178"})."\n";
		print $iso_list."\n";
		
		print $terminal->Tgoto('cm', 0, 8)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if (($answer) && ($answer =~ /^\d+$/) && (exists $install_isos->[$answer]) && ($install_isos->[$answer]))
		{
			# Valid.
			$anvil->data->{new_server}{install_media} = $install_isos->[$answer];
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::install_media" => $anvil->data->{new_server}{install_media},
			}});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_driver_disc
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		$anvil->Database->get_files();
		$anvil->Database->get_file_locations();
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		
		my $say_ram                 = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}});
		my $storage_group_uuid      = $anvil->data->{new_server}{storage_group};
		my $say_storage_group       = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
		my $say_storage_size        = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{storage_size}});
		my $install_media_file_uuid = $anvil->data->{new_server}{install_media};
		my $say_install_media       = $anvil->data->{files}{file_uuid}{$install_media_file_uuid}{file_name};
		
		# Create an array of install ISOs. 
		my $driver_isos = ["none"];
		my $iso_list     = $anvil->Words->string({key => "job_0179"})."\n";
		my $position     = 0;
		foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{files}{file_name}})
		{
			my $file_uuid            = $anvil->data->{files}{file_name}{$file_name}{file_uuid};
			my $file_directory       = $anvil->data->{files}{file_name}{$file_name}{file_directory};
			my $file_size            = $anvil->data->{files}{file_name}{$file_name}{file_size};
			my $file_type            = $anvil->data->{files}{file_name}{$file_name}{file_type};
			my $file_path            = $file_directory."/".$file_name;
			my $file_location_uuid   = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
			my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				file_uuid            => $file_uuid, 
				file_name            => $file_name,
				file_directory       => $file_directory, 
				file_size            => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")",
				file_type            => $file_type, 
				file_path            => $file_path, 
				file_location_uuid   => $file_location_uuid, 
				file_location_active => $file_location_active, 
			}});
			
			next if $file_type ne "iso";
			next if not $file_location_active;
			next if $file_uuid eq $install_media_file_uuid;
			
			# Still here?
			$position++;
			push @{$driver_isos}, $file_uuid;
			$iso_list .= " - ".$position.") ".$file_name." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")\n";
		}
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $say_ram }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
		print $anvil->Words->string({key => "job_0173", variables => { storage_size => $say_storage_size }})."\n";
		print $anvil->Words->string({key => "job_0176", variables => { install_media => $say_install_media }})."\n";
		print $anvil->Words->string({key => "job_0180", variables => { driver_disc => "#!string!unit_0005!#" }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0181"})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $dr1_host_uuid   = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
		
		print $anvil->Words->string({key => "job_0182"})."\n";
		print $iso_list."\n";
		
		print $terminal->Tgoto('cm', 0, 9)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		$answer = 0 if $answer eq "";
		
		if (($answer =~ /^\d+$/) && (exists $driver_isos->[$answer]) && ($driver_isos->[$answer]))
		{
			# Valid.
			$anvil->data->{new_server}{driver_disc} = $driver_isos->[$answer];
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::driver_disc" => $anvil->data->{new_server}{driver_disc},
			}});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_os
{
	my ($anvil, $terminal) = @_;
	
	my $words_file = $anvil->data->{path}{words}{'words.xml'};
	my $language   = $anvil->Words->language;
	my $os_list    = "";
	my $default_os = $anvil->data->{switches}{os} ? $anvil->data->{switches}{os} : "";
	foreach my $os_code (split/,/, $anvil->data->{sys}{servers}{os_short_list})
	{
		   $os_code =~ s/ //g;
		my $os_key  =  "os_list_".$os_code;
		my $os_name =  $anvil->Words->string({key => $os_key});
		if ($os_name =~ /#!not_found/)
		{
			# Skip it.
		}
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:os_code' => $os_code,
			's2:os_key'  => $os_key, 
			's2:os_name' => $os_name, 
		}});
		
		# Still here?
		$os_list .= " - ".sprintf("%-10s", $os_code)." - ".$os_name."\n";
	}
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid              = $anvil->data->{new_server}{anvil_uuid};
		my $storage_group_uuid      = $anvil->data->{new_server}{storage_group};
		my $say_ram                 = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}});
		my $say_storage_group       = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
		my $say_storage_size        = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{storage_size}});
		my $install_media_file_uuid = $anvil->data->{new_server}{install_media};
		my $say_install_media       = $anvil->data->{files}{file_uuid}{$install_media_file_uuid}{file_name};
		my $driver_disc_file_uuid   = $anvil->data->{new_server}{driver_disc};
		my $say_driver_disc         = $driver_disc_file_uuid eq "none" ? "#!string!unit_0005!#" : $anvil->data->{files}{file_uuid}{$driver_disc_file_uuid}{file_name};
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $say_ram }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
		print $anvil->Words->string({key => "job_0173", variables => { storage_size => $say_storage_size }})."\n";
		print $anvil->Words->string({key => "job_0176", variables => { install_media => $say_install_media }})."\n";
		print $anvil->Words->string({key => "job_0180", variables => { driver_disc => $say_driver_disc }})."\n";
		print $anvil->Words->string({key => "job_0198", variables => { os => $default_os }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0197"})."\n\n";
		}
		
		print $anvil->Words->string({key => "job_0196"})."\n";
		print $os_list."\n";
		
		print $terminal->Tgoto('cm', 0, 10)."? ";
		my $os_name = "";
		my $answer  = <STDIN>;
		chomp $answer;
		
		if ((not $answer) && ($default_os))
		{
			$answer = $default_os;
		}
		if ($answer)
		{
			# Is this valid?
			my $os_key  = "os_list_".$answer;
			   $os_name = $anvil->Words->string({key => $os_key});
		}
		
		if ((not $answer) or ($os_name =~ /#!not_found/))
		{
			# invalid.
			$retry = 1;
		}
		else
		{
			$anvil->data->{new_server}{os}     = $answer;
			$anvil->data->{new_server}{say_os} = $os_name;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::os"     => $anvil->data->{new_server}{os},
				"new_server::say_os" => $anvil->data->{new_server}{say_os},
			}});
			
			last;
		}
	}
	
	return(0);
}

sub interactive_ask_server_confirm
{
	my ($anvil, $terminal) = @_;
	
	$anvil->Database->get_anvils();
	$anvil->Database->get_files();
	$anvil->Database->get_file_locations();
	my $anvil_uuid      = $anvil->data->{new_server}{anvil_uuid};
	my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		
	my $storage_group_uuid      = $anvil->data->{new_server}{storage_group};
	my $say_ram                 = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}});
	my $say_storage_group       = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
	my $say_storage_size        = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{storage_size}});
	my $install_media_file_uuid = $anvil->data->{new_server}{install_media};
	my $say_install_media       = $anvil->data->{files}{file_uuid}{$install_media_file_uuid}{file_name};
	my $driver_disc_file_uuid   = $anvil->data->{new_server}{driver_disc};
	my $say_driver_disc         = $driver_disc_file_uuid eq "none" ? "#!string!unit_0005!#" : $anvil->data->{files}{file_uuid}{$driver_disc_file_uuid}{file_name};
	
	print $terminal->Tputs('cl');
	print $anvil->Words->string({key => "job_0150"})."\n";
	print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
	print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
	print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
	print $anvil->Words->string({key => "job_0165", variables => { ram => $say_ram }})."\n";
	print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
	print $anvil->Words->string({key => "job_0173", variables => { storage_size => $say_storage_size }})."\n";
	print $anvil->Words->string({key => "job_0176", variables => { install_media => $say_install_media }})."\n";
	print $anvil->Words->string({key => "job_0180", variables => { driver_disc => $say_driver_disc }})."\n";
	print $anvil->Words->string({key => "job_0198", variables => { os => $anvil->data->{new_server}{say_os} }})."\n\n";
	print $anvil->Words->string({key => "message_0206"})." ";
	my $answer = <STDIN>;
	chomp $answer;
	
	$answer = "y" if $answer eq "";
	
	if ($answer =~ /^y/i)
	{
		# Valid.
		my $job_data = "server_name=".$anvil->data->{new_server}{name}."
os=".$anvil->data->{new_server}{os}." 
cpu_cores=".$anvil->data->{new_server}{cpu}."
ram=".$anvil->data->{new_server}{ram}."
storage_group_uuid=".$anvil->data->{new_server}{storage_group}."
storage_size=".$anvil->data->{new_server}{storage_size}."
install_iso=".$anvil->data->{new_server}{install_media}."
driver_iso=".$anvil->data->{new_server}{driver_disc};
		if ($anvil->data->{new_server}{uuid})
		{
			$job_data .= "
server_uuid=".$anvil->data->{new_server}{name};
		}
		print "\n".$anvil->Words->string({key => "job_0183", variables => { job_data => $job_data }})."\n";
		
		# Register the job with the primary node on the Anvil! (or node 1, if neither node is primary).
		my $target_host_uuid = $anvil->Cluster->get_primary_host_uuid({debug => 2, anvil_uuid => $anvil_uuid});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_uuid => $target_host_uuid }});
		if (not $target_host_uuid)
		{
			$target_host_uuid = $node1_host_uuid;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_uuid => $target_host_uuid }});
		}
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-provision-server'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::provision", 
			job_title       => "job_0147", 
			job_description => "job_0148", 
			job_progress    => 0,
			job_host_uuid   => $target_host_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		print $anvil->Words->string({key => "job_0184", variables => { job_uuid => $job_uuid }})."\n";
	}
	else
	{
		# Abort
		print $anvil->Words->string({key => "message_0022"})."\n";
	}
	
	return(0);
}