anvil/tools/anvil-provision-server

#!/usr/bin/perl
# 
# This provisions a new virtual machine server. It handles creating the logical volumes, DRBD resources, 
# verifies the needed files are ready, creates the provision script, begins the installation, and adds the 
# new server to pacemaker.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Any problem that causes an early exit.
# 
# TODO: Support cloning; Example
#       - virt-clone --original-xml /mnt/shared/definitions/<source>.xml --name <new_server> --file <new_server's_drbd_path> --check path_exists=off
#       - Make VNC default
#       

use strict;
use warnings;
use Anvil::Tools;
require POSIX;
use Term::Cap;
use Data::Dumper;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

# Read switches
$anvil->Get->switches({list => [
	"anvil", 
	"anvil-name", 
	"anvil-uuid", 
	"ci-test", 
	"driver-disc", 
	"cpu", 
	"install-media", 
	"machine", 
	"name", 
	"network", 
	"options", 
	"os", 
	"pre-test", 
	"uuid", 
	"ram", 
	"storage-group", 
	"storage-size", 
	"use-image"], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0306"});
	sleep 10;
	$anvil->nice_exit({exit_code => 1});
}

# If we still don't have a job-uuit, go into interactive mode.
if ($anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->Job->clear();
	$anvil->Job->get_job_details({debug => 2});
	$anvil->Job->update_progress({
		progress         => 1,
		job_picked_up_by => $$, 
		job_picked_up_at => time, 
		message          => "message_0190", 
	});
	
	# Job data will be in $anvil->data->{jobs}{job_data}
	run_jobs($anvil);
}
elsif ($anvil->data->{switches}{'ci-test'})
{
	interactive_ask_server_confirm($anvil, "");
}
else
{
	# Interactive!
	interactive_question($anvil);
}

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This checks to see if any storage groups exist yet. If they don't, an attempt to assemble the storage 
# groups will be made
sub check_Storage_groups
{
	my ($anvil) = @_;
	
	# Make sure we have an anvil_uuid
	if ($anvil->data->{new_server}{anvil_uuid})
	{
		# Nope, nothing to do.
		return(1);
	}
	
	my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
	
	# Load Storage Group data
	$anvil->Database->get_storage_group_data({debug => 2});
	
	if (exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid})
	{
		my $count = 0;
		if ((exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_name}) && 
		    (ref($anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_name}) eq "HASH"))
		{
			$count = keys %{$anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_name}};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
		}
		
		if ($count)
		{
			# There's a storage group already, nothing to do.
			return(0);
		}
		
		# Still alive? Try to assemble a storage group.
		$anvil->Cluster->assemble_storage_groups({
			debug      => 2,
			anvil_uuid => $anvil_uuid, 
		});
		
		# Reload in case we created one.
		$anvil->Database->get_storage_group_data({debug => 2});
	}
	
	return(0);
}

# This actually provisions a VM.
sub run_jobs
{
	my ($anvil) = @_;
	
	# If we're here, the job has been assigned to us, so we use our own anvil! uuid.
	$anvil->data->{job}{anvil_uuid} = $anvil->Cluster->get_anvil_uuid();
	$anvil->Get->available_resources({
		debug      => 2,
		anvil_uuid => $anvil->data->{job}{anvil_uuid},
	});
	
	$anvil->Database->get_hosts();
	$anvil->Database->get_files();
	$anvil->Database->get_file_locations();
	
	# This parses the jobs::job_data intp variables.
	parse_job_data($anvil);
	
	# Wait for the cluster to be up, if it's not.
	my $waiting          = 1;
	my $waiting_reported = 0;
	while($waiting)
	{
		my $problem = $anvil->Cluster->parse_cib({debug => 3});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		if ($problem)
		{
			# Cluster isn't up yet, wait.
			if (not $waiting_reported)
			{
				$anvil->Job->update_progress({
					progress => 5,
					message  => "job_0275", 
				});
				$waiting_reported = 1;
			}
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0275"});
			sleep 10;
		}
		else
		{
			# Make sure we're actually in the cluster now.
			if ($anvil->data->{cib}{parsed}{'local'}{ready})
			{
				# We're ready!
				$waiting = 0;
				$anvil->Job->update_progress({
					progress => 8,
					message  => "job_0276", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0276"});
			}
			else
			{
				# Cluster is coming up, but it's not up yet.
				$anvil->Job->update_progress({
					progress => 6,
					message  => "job_0278", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0278"});
			}
		}
	}
	
	# Sanity checks passed
	$anvil->Job->update_progress({
		progress => 10,
		message  => "job_0185", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0185"});
	
	# This picks out the next free DRBD minor number and TCP port if it's not set in the job data.
	check_drbd_minor_and_port($anvil);
	
	# Report the DRBD minor and TCP port that we'll use.
	$anvil->Job->update_progress({
		progress => 20,
		message  => "job_0186,!!minor!".$anvil->data->{job}{drbd_minor}."!!,!!port!".$anvil->data->{job}{drbd_tcp_port}."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0186", variables => {
		minor => $anvil->data->{job}{drbd_minor}, 
		port  => $anvil->data->{job}{drbd_tcp_port}, 
	}});
	
	# Prepare some variables
	$anvil->Database->get_anvils();
	my $anvil_uuid      = $anvil->data->{job}{anvil_uuid};
	my $host_uuid       = $anvil->Get->host_uuid();
	my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $peer_host_uuid  = $host_uuid eq $node1_host_uuid ? $node2_host_uuid : $node1_host_uuid;
	my $peer_name       = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{host_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		host_uuid       => $host_uuid,
		peer_host_uuid  => $peer_host_uuid, 
		peer_name       => $peer_name, 
		node1_host_uuid => $node1_host_uuid,
		node2_host_uuid => $node2_host_uuid,
	}});
	
	$anvil->data->{job}{node1_host_uuid}       = $node1_host_uuid;
	$anvil->data->{job}{node2_host_uuid}       = $node2_host_uuid;
	$anvil->data->{job}{peer_short_name}       = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{short_host_name};
	$anvil->data->{job}{short_host_name}       = $anvil->Get->short_host_name;
	$anvil->data->{job}{node1_short_host_name} = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	$anvil->data->{job}{node2_short_host_name} = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	$anvil->data->{job}{drbd_local_node_id}    = $host_uuid eq $node1_host_uuid ? 0 : 1;
	$anvil->data->{job}{drbd_peer_node_id}     = $host_uuid eq $node1_host_uuid ? 1 : 0;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		'job::node1_host_uuid'       => $anvil->data->{job}{node1_host_uuid}, 
		'job::node2_host_uuid'       => $anvil->data->{job}{node2_host_uuid}, 
		'job::node1_short_host_name' => $anvil->data->{job}{node1_short_host_name},
		'job::node2_short_host_name' => $anvil->data->{job}{node2_short_host_name},
		'job::short_host_name'       => $anvil->data->{job}{short_host_name}, 
		'job::drbd_local_node_id'    => $anvil->data->{job}{drbd_local_node_id}, 
		'job::drbd_peer_node_id'     => $anvil->data->{job}{drbd_peer_node_id}, 
	}});
	
	# We convert to extents as it ensure clean boundaries and, being based on bytes in both cases, gets 
	# us as close as we can to what the user asked for. The 'int()' always rounds down, so we don't ever 
	# ask for one more extent than is available by accident.
	my $storage_group_uuid = $anvil->data->{job}{storage_group_uuid};
	my $vg_name            = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_name};
	my $lv_path            = "/dev/".$vg_name."/".$anvil->data->{job}{server_name}."_0";
	my $peer_vg_name       = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$peer_host_uuid}{vg_name};
	my $peer_lv_path       = "/dev/".$peer_vg_name."/".$anvil->data->{job}{server_name}."_0";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		vg_name      => $vg_name, 
		lv_path      => $lv_path, 
		peer_lv_path => $peer_lv_path
	}});
	
	# Node 1 and Node 2 are set independent of which node we have.
	$anvil->data->{job}{node1_lv_path} = "";
	$anvil->data->{job}{node2_lv_path} = "";
	if ($host_uuid eq $anvil->data->{job}{node1_host_uuid})
	{
		# We're node 1
		$anvil->data->{job}{node1_lv_path} = $lv_path;
		$anvil->data->{job}{node2_lv_path} = $peer_lv_path;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			'job::node1_lv_path' => $anvil->data->{job}{node1_lv_path},
			'job::node2_lv_path' => $anvil->data->{job}{node2_lv_path},
		}});
	}
	else
	{
		# We're node 2
		$anvil->data->{job}{node1_lv_path} = $peer_lv_path;
		$anvil->data->{job}{node2_lv_path} = $lv_path;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			'job::node1_lv_path' => $anvil->data->{job}{node1_lv_path},
			'job::node2_lv_path' => $anvil->data->{job}{node2_lv_path},
		}});
	}
	
	# If we're not the peer, create the peer's job.
	if (not $anvil->data->{job}{peer_mode})
	{
		my $job_data   =  $anvil->data->{jobs}{job_data}."\n";
		   $job_data   .= "peer_mode=true\n";
		   $job_data   .= "drbd_minor=".$anvil->data->{job}{drbd_minor}."\n";
		   $job_data   .= "drbd_tcp_port=".$anvil->data->{job}{drbd_tcp_port}."\n";
		my ($job_uuid) =  $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-provision-server'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::provision", 
			job_title       => "job_0147", 
			job_description => "job_0148", 
			job_progress    => 0,
			job_host_uuid   => $peer_host_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		$anvil->Job->update_progress({
			progress => 25,
			message  => "job_0188,!!job_uuid!".$job_uuid."!!,!!peer_name!".$peer_name."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0188", variables => {
			job_uuid  => $job_uuid, 
			peer_name => $peer_name, 
		}});
	}
	
	# If the LV exists, we won't initialize the DRBD resource. If the LV exists but isn't a DRBD backing 
	# device, this will cause the resource to not come up, but it's better than risking data loss.
	create_lv($anvil, $lv_path);
	
	# Load the IPs so we can find which SN network and IPs to use 
	get_sn_details($anvil);
	
	# Create the DRBD resource file
	create_resource_file($anvil);
	
	# Make sure the DRBD ports are open.
	$anvil->Network->manage_firewall();
	
	# Create the DRBD metadata, if needed.
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::initialize_drbd' => $anvil->data->{job}{initialize_drbd} }});
	create_md($anvil) if $anvil->data->{job}{initialize_drbd};
	
	### NOTE: This function also forces the resource to be UpToDate if this is not a peer job and we 
	###       created the LV.
	# Check to see if the resource needs to be started. It almost always will.
	startup_resource($anvil);
	
	# If we're here, we can finally craft the 'virt-install' call!.
	if ($anvil->data->{job}{peer_mode})
	{
		# Wait until we have seen the definition in the database and written to disk
		write_definition($anvil);
	}
	else
	{
		provision_server($anvil);
	}
	
	# Make sure the VNC port is open.
	$anvil->Network->manage_firewall();
	
	# If we're not the peer, add the server to the cluster.
	if (not $anvil->data->{job}{peer_mode})
	{
		add_server_to_cluster($anvil);
	}
	
	# Update the database by calling select scan agents
	$anvil->Job->update_progress({
		progress => 95,
		message  => "job_0464", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0464"});
	foreach my $agent ("scan-drbd", "scan-cluster", "scan-network", "scan-server")
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0740", variables => { agent => $agent }});
		
		my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/".$agent."/".$agent.$anvil->Log->switches();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Done!
	$anvil->Job->update_progress({
		debug    => 2,
		progress => 100,
		message  => "job_0202", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0202"});

	return(0);
}

sub add_server_to_cluster
{
	my ($anvil) = @_;
	
	# Parse the cluster
	$anvil->Job->update_progress({
		progress => 90,
		message  => "job_0207", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0207"});
	
	# Is our peer in the cluster? For that matter, are we?
	my $problem = $anvil->Cluster->add_server({
		debug        => 2, 
		server_name  => $anvil->data->{job}{server_name},
		ok_if_exists => 1,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	if ($problem)
	{
		# Something went wrong
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0215", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0215"});
		$anvil->nice_exit({exit_code => 1});
	}
	
	return(0);
}

# This loops until the definition file is found in the database and then writes it out to disk.
sub write_definition
{
	my ($anvil) = @_;
	
	my $anvil_uuid  = $anvil->data->{job}{anvil_uuid};
	my $server_name = $anvil->data->{job}{server_name};
	my $xml_file    = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { xml_file => $xml_file }});
	if (not -f $xml_file)
	{
		# Tell the user we're going to wait until we find the server in the database.
		$anvil->Job->update_progress({
			progress => 70,
			message  => "job_0205", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0205"});
		
		while (not -f $xml_file)
		{
			$anvil->Database->get_servers({debug => 2});
			$anvil->Database->get_server_definitions({debug => 2});
			
			if (not exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid})
			{
				sleep 1;
				next;
			}
			
			my $server_uuid       = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
			my $server_definition = exists $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid} ? $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml} : "";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition => $server_definition }});
			if ($server_definition)
			{
				# Write it!
				my $return = $anvil->Storage->write_file({
					debug     => 2,
					body      => $server_definition,
					file      => $xml_file,
					overwrite => 1,
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'return' => $return }});
				
				if (-f $xml_file)
				{
					$anvil->Job->update_progress({
						progress => 80,
						message  => "job_0206,!!file!".$xml_file."!!", 
					});
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0206", variables => { file => $xml_file }});
				}
			}
			else
			{
				sleep 1;
			}
		}
	}
	
	# Lastly, if there's a variable recording the shell call used to provision this server, write it out.
	my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_uuid => $server_uuid }});
	my ($variable_value, $variable_uuid, $mtime, $modified_date) = $anvil->Database->read_variable({
		debug                 => 2, 
		variable_name         => "server::provision_script", 
		variable_source_uuid  => $server_uuid, 
		variable_source_table => "servers", 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		variable_value => $variable_value,
		variable_uuid  => $variable_uuid, 
	}});
	if (($variable_value) && ($variable_value =~ /virt-install/gs))
	{
		my $provision_file = $anvil->data->{path}{directories}{shared}{provision}."/".$server_name.".sh";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { provision_file => $provision_file }});
		my $problem = $anvil->Storage->write_file({
			overwrite => 1, 
			backup    => 1,
			file      => $provision_file, 
			body      => $variable_value, 
			mode      => "0755",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	}
	
	# The peer is done, it'll pick up the XML definition when ScanCore runs 
	$anvil->Job->update_progress({
		progress => 85,
		message  => "job_0204", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0204"});
	
	return(0);
}

# This creates the virt-install call and creates the VM.
sub provision_server
{
	my ($anvil) = @_;
	
	# The logic behind this is explained here: https://libvirt.org/formatdomain.html#elementsTime
	my $server       = $anvil->data->{job}{server_name};
	my $clock_offset = $anvil->data->{job}{os} =~ /win/i ? "localtime" : "utc";
	my $say_memory   =  sprintf("%.0f", ($anvil->data->{job}{ram} /= (2 ** 20)));
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		clock_offset => $clock_offset,
		say_memory   => $say_memory,
	}});
	
	### TODO: Support disk images (ie: sysprep) via '--import'. The device used for booting is the first 
	###       device specified via "--disk"
	###       Consider support for TPM, RNG and watchdog devices
	my $server_uuid = $anvil->data->{job}{server_uuid} ? $anvil->data->{job}{server_uuid} : $anvil->Get->uuid();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"job::os"   => $anvil->data->{job}{os}, 
		server_uuid => $server_uuid, 
	}});
	my $disk_bus = ",target.bus=virtio";
	if ($anvil->data->{job}{os} eq "win7")
	{
		$disk_bus = "";
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { disk_bus => $disk_bus }});
	
	# Setup the network line
	my $nic = "bridge=";
	if ($anvil->data->{job}{network}{bridge})
	{
		$nic .= $anvil->data->{job}{network}{bridge};
	}
	else
	{
		$nic .= "ifn1_bridge1";
	}
	if ($anvil->data->{job}{network}{model})
	{
		$nic .= ",model.type=".$anvil->data->{job}{network}{model};
	}
	elsif ($anvil->data->{job}{os} eq "win7")
	{
		$nic .= ",model.type=e1000e";
	}
	if ($anvil->data->{job}{network}{mac})
	{
		$nic .= ",mac.address=".$anvil->data->{job}{network}{mac};
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nic => $nic }});
	
	my $shell_call =  $anvil->data->{path}{exe}{'virt-install'}." --connect qemu:///system \\\n";
	   $shell_call .= "--name ".$server." \\\n";
	   $shell_call .= " --os-variant ".$anvil->data->{job}{os}." \\\n";
	   $shell_call .= " --memory ".$say_memory." \\\n";
	   $shell_call .= " --events on_poweroff=destroy,on_reboot=restart \\\n";
	   $shell_call .= " --vcpus ".$anvil->data->{job}{cpu_cores}.",sockets=1,cores=".$anvil->data->{job}{cpu_cores}." \\\n";
	   $shell_call .= " --cpu host \\\n";
	   $shell_call .= " --network ".$nic." \\\n";
	   $shell_call .= " --graphics vnc \\\n";
	   $shell_call .= " --sound ich9 \\\n";
	   $shell_call .= " --clock offset=".$clock_offset.",rtc_tickpolicy=catchup \\\n";
	   $shell_call .= " --boot menu=on \\\n";
	   $shell_call .= " --disk path=/dev/drbd/by-res/".$server."/0".$disk_bus.",driver.io=threads,cache=writeback,driver.discard=unmap,boot.order=1 \\\n";
	   $shell_call .= " --disk path=".$anvil->data->{job}{install_iso_path}.",device=cdrom,shareable=on,boot.order=2 \\\n";
	   $shell_call .= " --uuid=".$server_uuid." \\\n";
	if ($anvil->data->{job}{driver_iso_path})
	{
		$shell_call .= " --disk path=".$anvil->data->{job}{driver_iso_path}.",device=cdrom,shareable=on,boot.order=3 --force \\\n";
	}
	$shell_call .= " --noautoconsole > /var/log/anvil-server_".$server.".log\n";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	
	$anvil->Job->update_progress({
		progress => 70,
		message  => "job_0199,!!shell_call!".$shell_call."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0199", variables => { shell_call => $shell_call }});
	
	# Write the command out to /mnt/shared/provision/
	my $provision_file = $anvil->data->{path}{directories}{shared}{provision}."/".$server.".sh";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	my $problem = $anvil->Storage->write_file({
		overwrite => 1, 
		backup    => 1,
		file      => $provision_file, 
		body      => $shell_call, 
		mode      => "0755",
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	
	# Also, record the provision command 
	my $variable_uuid = $anvil->Database->insert_or_update_variables({
		variable_name         => "server::provision_script",
		variable_value        => $shell_call,
		variable_default      => "", 
		variable_description  => "striker_0293", 
		variable_section      => "servers", 
		variable_source_uuid  => $server_uuid, 
		variable_source_table => "servers", 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }});
	
	# Wait until file(s) are ready.
	my $waiting   = 1;
	my $host_uuid = $anvil->Get->host_uuid();
	my @files     = ($anvil->data->{job}{install_iso_path});
	if ($anvil->data->{job}{driver_iso_path})
	{
		push @files, $anvil->data->{job}{driver_iso_path};
	}
	while ($waiting)
	{
		# This loads files and file locations.
		$anvil->Storage->check_files({debug => 2});
		$waiting = 0;
		foreach my $file_path (sort {$a cmp $b} @files)
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_path => $file_path }});
			
			$anvil->data->{file_uuid}{$file_path} = "" if not exists $anvil->data->{uuid}{$file_path}{file_uuid};
			
			# What's this file's file_uuid?
			my $file_uuid = $anvil->data->{file_uuid}{$file_path};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
			
			my $file_location_uuid = "";
			if (not $file_uuid)
			{
				foreach my $this_file_uuid (sort {$a cmp $b} keys %{$anvil->data->{files}{file_uuid}})
				{
					my $this_file_path =  $anvil->data->{files}{file_uuid}{$this_file_uuid}{file_directory}."/".$anvil->data->{files}{file_uuid}{$this_file_uuid}{file_name};
					   $this_file_path =~ s/\/\//\//g;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						this_file_uuid => $this_file_uuid, 
						this_file_path => $this_file_path,
					}});
					if ($this_file_path eq $file_path)
					{
						# Found it.
						$file_uuid = $this_file_uuid;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
						last;
					}
				}
			}
			
			# Did we find the file?
			if (not $file_uuid)
			{
				# Nope.
				$waiting = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "alert", key => "warning_0154", variables => {
					file_path   => $file_path, 
					server_name => $server, 
				}});
				next;
			}
			
			# Yes, now do we have the file_location_uuid?
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"file_locations::host_uuid::${host_uuid}::file_uuid::${file_uuid}" => exists $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid} ? 1 : 0,
			}});
			if (not exists $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid})
			{
				# Nope
				$waiting = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "alert", key => "warning_0155", variables => {
					file_path   => $file_path, 
					server_name => $server, 
				}});
				
				# This is being hit improperly somehow. Added logging to help diagnose.
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:Get->host_uuid' => $anvil->Get->host_uuid,
					's2:host_uuid'      => $host_uuid, 
					's3:file_uuid'      => $file_uuid,
				}});
				foreach my $this_host_uuid (sort {$a cmp $b} keys %{$anvil->data->{file_locations}{host_uuid}})
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:this_host_uuid' => $this_host_uuid,
						's2:this_host_name' => $anvil->Get->host_name_from_uuid({host_uuid => $this_host_uuid}),
					}});
					foreach my $this_file_uuid (sort {$a cmp $b} keys %{$anvil->data->{file_locations}{host_uuid}{$this_host_uuid}{file_uuid}})
					{
						my $this_file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$this_host_uuid}{file_uuid}{$this_file_uuid}{file_location_uuid};
						my $this_anvil_uuid         = $anvil->data->{file_locations}{file_location_uuid}{$this_file_location_uuid}{file_location_anvil_uuid};
						my $this_file_name          = $anvil->data->{anvils}{anvil_uuid}{$this_anvil_uuid}{file_uuid}{$this_file_uuid}{file_name};
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							's1:this_file_uuid'          => $this_file_uuid,
							's2:this_file_location_uuid' => $this_file_location_uuid,
							's3:this_anvil_uuid'         => $this_anvil_uuid, 
							's4:this_file_name'          => $this_file_name, 
						}});
					}
				}
				next;
			}
			
			   $file_location_uuid  = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
			my $file_location_ready = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_ready};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				file_location_uuid  => $file_location_uuid,
				file_location_ready => $file_location_ready, 
			}});
			
			if (not $file_location_ready)
			{
				$waiting = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "alert", key => "warning_0156", variables => {
					file_path   => $file_path, 
					server_name => $server, 
				}});
				next;
			}
		}
		
		if ($waiting)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "alert", key => "warning_0157"});
			sleep 10;
		}
	}
	
	# Call as a background process.
	my ($handle, $return_code) = $anvil->System->call({
		background  => 1, 
		shell_call  => $shell_call,
		stdout_file => "/var/log/anvil-server_".$server.".stdout",
		stderr_file => "/var/log/anvil-server_".$server.".stderr", 
	});
	my $pid = $handle->pid();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		pid         => $pid,
		return_code => $return_code,
	}});
	
	$anvil->Job->update_progress({
		progress => 80,
		message  => "job_0200", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0200"});
	
	# Loop for up to 10 seconds waiting to see the server start running.
	my $wait_until = time + 60;
	   $waiting    = 1;
	my $status     = "";
	while($waiting)
	{
		my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list --all"});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code,
		}});
		
		foreach my $line (split/\n/, $output)
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
			if ($line =~ /.*?$server\s+(.*)$/)
			{
				$status = $1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { status => $status }});
				if ($status eq "running")
				{
					$waiting = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				}
				elsif ($status eq "shut off")
				{
					# Try to boot it.
					my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." start ".$server});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code,
					}});
				}
			}
			if (($waiting) && (time > $wait_until))
			{
				# Didn't boot, give up
				$waiting = 0;
				$anvil->Job->update_progress({
					progress   => 100,
					message    => "error_0210,!!status!".$status."!!", 
					job_status => "failed", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0210", variables => { status => $status }});
				$anvil->nice_exit({exit_code => 1});
			}
			sleep 1;
		}
	}
	
	$anvil->Job->update_progress({
		progress => 85,
		message  => "job_0201", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0201"});
	
	# Dump the server's XML.
	(my $server_definition_xml, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." dumpxml ".$server});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server_definition_xml => $server_definition_xml,
		return_code           => $return_code,
	}});

	$problem = $anvil->Server->parse_definition({
		server     => $server, 
		source     => "from_virsh",
		definition => $server_definition_xml, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	if ($problem)
	{
		# Something went wrong with the provision
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0209,!!shell_call!".$shell_call."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0209", variables => { shell_call => $shell_call }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Write out the definition file.
	my $definition_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server.".xml";
	$problem = $anvil->Storage->write_file({
		overwrite => 1, 
		backup    => 1,
		file      => $definition_file, 
		body      => $server_definition_xml, 
		mode      => "0644",
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
	# Add the server to the 'servers' table
	my $anvil_uuid      = $anvil->data->{job}{anvil_uuid};
	my $short_host_name = $anvil->Get->short_host_name;
	my $got_server_uuid = $anvil->Database->insert_or_update_servers({
		server_uuid           => $server_uuid, 
		server_name           => $server, 
		server_state          => "running", 
		server_anvil_uuid     => $anvil_uuid, 
		server_host_uuid      => $anvil->Get->host_uuid, 
		server_ram_in_use     => $anvil->data->{job}{ram}, 
		server_configured_ram => $anvil->data->{job}{ram}, 
		server_boot_time      => time,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { got_server_uuid => $got_server_uuid }});
	
	# Save the definition.
	my $server_definition_uuid = $anvil->Database->insert_or_update_server_definitions({
		server_definition_server_uuid => $got_server_uuid, 
		server_definition_xml         => $server_definition_xml, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition_uuid => $server_definition_uuid }});
	
	# Undefine the server
	(my $output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." undefine ".$server});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code,
	}});

	# Call an adjust in case we forced the resource to disable fencing.
	$shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$anvil->data->{job}{server_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});

	($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
		output      => $output,
		return_code => $return_code,
	}});

	return(0);
}

# This starts up the DRBD resource, if needed.
sub startup_resource
{
	my ($anvil) = @_;
	
	$anvil->DRBD->get_status();
	my $startup_needed = 1;
	
	# Is the current resource up locally already? If it is, we're done. 
	my $server          = $anvil->data->{job}{server_name};
	my $short_host_name = $anvil->data->{job}{short_host_name};
	my $role            = defined $anvil->data->{drbd}{status}{$short_host_name}{resource}{$server}{role} ? $anvil->data->{drbd}{status}{$short_host_name}{resource}{$server}{role} : "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server          => $server, 
		short_host_name => $short_host_name, 
		role            => $role,
	}});
	
	if ((lc($role) ne "secondary") && (lc($role) ne "primary"))
	{
		$startup_needed = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { startup_needed => $startup_needed }});
	}
	else
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0434", variables => { 
			resource => $anvil->data->{job}{server_name},
			role     => $role, 
		}});
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { startup_needed => $startup_needed }});
	if (not $startup_needed)
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0431"});
	}
	else
	{
		# Bring the drbd resource up.
		$anvil->Job->update_progress({
			progress => 52,
			message  => "job_0192", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0192"});
		$anvil->DRBD->manage_resource({
			debug    => 2,
			resource => $anvil->data->{job}{server_name}, 
			task     => "up",
		});
		
		# If both sides are Inconsistent, for node 1 to primary. If we're primary and the peer is
		# Unknown, force to primary.
		my $waiting = 1;
		while($waiting)
		{
			# Reload the DRBD data 
			$anvil->DRBD->get_status();
			
			my $server    = $anvil->data->{job}{server_name};
			my $all_ready = 1;
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{drbd}{status}{$short_host_name}{resource}{$server}{devices}{volume}})
			{
				my $disk_state = $anvil->data->{drbd}{status}{$short_host_name}{resource}{$server}{devices}{volume}{$volume}{'disk-state'};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:short_host_name'  => $short_host_name, 
					's2:job::server_name' => $anvil->data->{job}{server_name}, 
					's3:volume'           => $volume, 
					's4:disk_state'       => $disk_state,
				}});
				
				# Is the peer isn't connected (directly or by being in Sync), or this volume 
				# isn't UpToDate, we need to keep waiting.
				if ((lc($disk_state) ne "uptodate")   && 
				    (lc($disk_state) ne "consistent") && 
				    (lc($disk_state) ne "outdated")   && 
				    (lc($disk_state) ne "inconsistent"))
				{
					$all_ready = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_ready => $all_ready }});
					
					$anvil->Job->update_progress({
						progress => 54,
						message  => "job_0193,!!volume!".$volume."!!,!!disk_state!".$disk_state."!!", 
					});
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0193", variables => { 
						volume     => $volume,
						disk_state => $disk_state,
					}});
				}
				
				# If we're not initializing DRBD, we'll wait until we're uptodate.
				if (not $anvil->data->{job}{initialize_drbd})
				{
					if (lc($disk_state) eq "diskless")
					{
						# This happens when the LV existed but there's no meta-data.
						$anvil->Job->update_progress({
							progress   => 100,
							message    => "error_0206,!!resource!".$anvil->data->{job}{server_name}."!!", 
							job_status => "failed", 
						});
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0206", variables => { resource => $anvil->data->{job}{server_name} }});
						$anvil->nice_exit({exit_code => 1});
					}
					if (lc($disk_state) ne "uptodate")
					{
						# Log why we're waiting.
						$all_ready = 0;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_ready => $all_ready }});
						
						$anvil->Job->update_progress({
							progress => 56,
							message  => "job_0194,!!resource!".$anvil->data->{job}{server_name}."!!", 
						});
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0194", variables => { resource => $anvil->data->{job}{server_name} }});
					}
				}
				
				# If we're initializing and the resource is 'Inconsistent', force the resource to primary
				if (($anvil->data->{job}{initialize_drbd}) && (not $anvil->data->{job}{peer_mode}) && (lc($disk_state) eq "inconsistent"))
				{
					### NOTE: '<server> 1' is the resource followed by the peer ID number (1).
					# To force to primary, we first need to temporarily disable fencing.
					$anvil->Job->update_progress({
						progress => 58,
						message  => "job_0195", 
					});
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0195"});
					
 					my $shell_call = $anvil->data->{path}{exe}{drbdsetup}." net-options ".$anvil->data->{job}{server_name}." ".$anvil->data->{job}{drbd_peer_node_id}." --set-defaults --_name=".$anvil->data->{job}{peer_short_name}." --protocol=C --fencing=dont-care";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code,
					}});
					
					# Now force it to primary.
					$shell_call = $anvil->data->{path}{exe}{drbdadm}." primary ".$anvil->data->{job}{server_name}." --force";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code,
					}});
					# Return code of '0' is success.
					if ($return_code)
					{
						# Failed to promote
						$anvil->Job->update_progress({
							progress   => 100,
							message    => "error_0205,!!return_code!".$return_code."!!,!!output!".$output."!!", 
							job_status => "failed", 
						});
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0205", variables => { 
							return_code => $return_code,
							output      => $output, 
						}});
						$anvil->nice_exit({exit_code => 1});
					}
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0580", variables => { resource => $anvil->data->{job}{server_name} }});
					
					# Make it secondary again. This is needed as it won't auto-demote 
					# otherwise.
					$shell_call = $anvil->data->{path}{exe}{drbdadm}." secondary ".$anvil->data->{job}{server_name};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code,
					}});
					
					# set the fencing back
					$shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$anvil->data->{job}{server_name};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code,
					}});
					
					# Update and see if the resource's disk state is now UpToDate.
					$anvil->DRBD->get_status();
					my $server = $anvil->data->{job}{server_name};
					my $disk_state = $anvil->data->{drbd}{status}{$short_host_name}{resource}{$server}{devices}{volume}{$volume}{'disk-state'};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:short_host_name'  => $short_host_name, 
						's2:job::server_name' => $anvil->data->{job}{server_name}, 
						's3:volume'           => $volume, 
						's4:disk_state'       => $disk_state,
					}});
					
					if (lc($disk_state) ne "uptodate")
					{
						# Something went wrong.
						$anvil->Job->update_progress({
							progress   => 100,
							message    => "error_0207,!!resource!".$anvil->data->{job}{server_name}."!!,!!disk_state!".$disk_state."!!", 
							job_status => "failed", 
						});
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0207", variables => { 
							resource   => $anvil->data->{job}{server_name},
							disk_state => $disk_state, 
						}});
						$anvil->nice_exit({exit_code => 1});
					}
				}
			}
			
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_ready => $all_ready }});
			if ($all_ready)
			{
				# If we're here, we're mostly ready, however we're in a bit of a spot if our
				# peer hasn't connected. We're going to wait up to one minute for the peer to
				# connect. If it doesn't, we're going to force back to primary.
				$anvil->Job->update_progress({
					progress => 59,
					message  => "job_0434",
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0434"});

				my $wait_until = time + 60;
				my $resource   = $anvil->data->{job}{server_name};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
					's1:time'       => time,
					's2:wait_until' => $wait_until,
					's3:resource'   => $resource,
				}});
				while ($waiting)
				{
					$anvil->DRBD->gather_data({debug => 3});
					my $any_unknown = 0;
					foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{volume}})
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
						foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}})
						{
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { peer => $peer }});
							if (not defined $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{local_disk_state})
							{
								$any_unknown = 1;
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
								next;
							}

							my $local_disk_state = $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{local_disk_state};
							my $peer_disk_state  = $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{peer_disk_state};
							my $local_role       = $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{local_role};
							my $peer_role        = $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{peer_role};
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
								's1:local_disk_state' => $local_disk_state,
								's2:peer_disk_state'  => $peer_disk_state,
								's3:local_role'       => $local_role,
								's4:peer_role'        => $peer_role,
							}});

							if (($peer_disk_state =~ /unknown/i) or ($peer_role =~ /unknown/i))
							{
								$any_unknown = 1;
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
							}
							last if $any_unknown;
						}
						last if $any_unknown;
					}

					if ($any_unknown)
					{
						if (time > $wait_until)
						{
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0436"});

							my $shell_call = $anvil->data->{path}{exe}{drbdsetup}." net-options ".$anvil->data->{job}{server_name}." ".$anvil->data->{job}{drbd_peer_node_id}." --set-defaults --_name=".$anvil->data->{job}{peer_short_name}." --protocol=C --fencing=dont-care";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});

							my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
								output      => $output,
								return_code => $return_code,
							}});

							$waiting = 0;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
						}
						else
						{
							my $waiting = $wait_until - time;
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0435", variables => { waiting => $waiting }});
							sleep 5;
						}
					}
					else
					{
						$waiting = 0;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
					}
				}

				$waiting = 0;
			}
			
			if ($waiting)
			{
				sleep 10;
			}
		}
	}
	
	$anvil->Job->update_progress({
		progress => 60,
		message  => "job_0203,!!resource!".$anvil->data->{job}{server_name}."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0203", variables => { resource => $anvil->data->{job}{server_name} }});
	
	return(0);
}

# Thos does the job of creating the metadata.
sub create_md
{
	my ($anvil) = @_;
	
	### NOTE: The '--max-peers=3' is needed to make space for future DR additions
	# Create the DRBD metadata
	my $shell_call = $anvil->data->{path}{exe}{drbdadm}." --force create-md --max-peers=3 ".$anvil->data->{job}{server_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});

	my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
		output      => $output,
		return_code => $return_code,
	}});

	### Return codes
	# 0 == Success
	# 1 == ?
	# 3 == Configuration not found.
	if ($return_code)
	{
		# Metadata creation failed.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0204,!!return_code!".$return_code."!!,!!output!".$output."!!",
			job_status => "failed",
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0204", variables => {
			return_code => $return_code,
			output      => $output,
		}});
		$anvil->nice_exit({exit_code => 1});
	}
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0579", variables => { resource => $anvil->data->{job}{server_name} }});

	$anvil->Job->update_progress({
		progress => 50,
		message  => "job_0191,!!resource!".$anvil->data->{job}{server_name}."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0191", variables => { resource => $anvil->data->{job}{server_name} }});
	
	return(0);
}

# This finds which SN (or BCN, IFN) network and IPs we're using.
sub get_sn_details
{
	my ($anvil) = @_;
	
	$anvil->data->{job}{node1_sn_ip} = "";
	$anvil->data->{job}{node2_sn_ip} = "";
	$anvil->data->{job}{sn_network}  = "";
	
	my $node1_short_host_name = $anvil->data->{job}{node1_short_host_name};
	my $node1_host_uuid       = $anvil->data->{job}{node1_host_uuid};
	my $node2_short_host_name = $anvil->data->{job}{node2_short_host_name};
	my $node2_host_uuid       = $anvil->data->{job}{node1_host_uuid};
	my $peer_short_name       = $anvil->data->{job}{peer_short_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"s1:node1_short_host_name" => $node1_short_host_name, 
		"s2:node1_host_uuid"       => $node1_host_uuid, 
		"s3:node2_short_host_name" => $node2_short_host_name, 
		"s4:node2_host_uuid"       => $node2_host_uuid, 
		"s5:peer_short_name"       => $peer_short_name, 
	}});
	
	# Look for a match on the SN, and failing that the BCN, and finally the IFN. We don't check the MN.
	my $matches = $anvil->Network->find_access({
		debug  => 2,
		target => $peer_short_name, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { matches => $matches }});
	foreach my $preferred_network ("sn")
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { preferred_network => $preferred_network }});
		foreach my $network_name (sort {$a cmp $b} keys %{$anvil->data->{network_access}})
		{
			next if $network_name !~ /^$preferred_network/;
			
			# Network and link speed
			$anvil->data->{job}{sn_network}    = uc($network_name);
			$anvil->data->{job}{network_speed} = $anvil->data->{network_access}{$network_name}{local_speed} < $anvil->data->{network_access}{$network_name}{target_speed} ? $anvil->data->{network_access}{$network_name}{local_speed} : $anvil->data->{network_access}{$network_name}{target_speed};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"s1:job::sn_network"    => $anvil->data->{job}{sn_network}, 
				"s2:job::network_speed" => $anvil->data->{job}{network_speed}, 
			}});
			if ($node1_host_uuid eq $anvil->Get->host_uuid)
			{
				# We're node 1
				$anvil->data->{job}{node1_sn_ip} = $anvil->data->{network_access}{$network_name}{local_ip_address}  ? $anvil->data->{network_access}{$network_name}{local_ip_address}  : "";
				$anvil->data->{job}{node2_sn_ip} = $anvil->data->{network_access}{$network_name}{target_ip_address} ? $anvil->data->{network_access}{$network_name}{target_ip_address} : "";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:job::node1_sn_ip" => $anvil->data->{job}{node1_sn_ip}, 
					"s2:job::node2_sn_ip" => $anvil->data->{job}{node2_sn_ip}, 
				}});
			}
			else
			{
				# We're node 2
				$anvil->data->{job}{node1_sn_ip} = $anvil->data->{network_access}{$network_name}{target_ip_address} ? $anvil->data->{network_access}{$network_name}{target_ip_address} : "";
				$anvil->data->{job}{node2_sn_ip} = $anvil->data->{network_access}{$network_name}{local_ip_address}  ? $anvil->data->{network_access}{$network_name}{local_ip_address}  : "";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:job::node1_sn_ip" => $anvil->data->{job}{node1_sn_ip}, 
					"s2:job::node2_sn_ip" => $anvil->data->{job}{node2_sn_ip}, 
				}});
			}
			
			if ((not $anvil->data->{job}{node1_sn_ip}) or (not $anvil->data->{job}{node2_sn_ip}))
			{
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0417", variables => {
					node1_name => $node1_short_host_name, 
					node2_name => $node2_short_host_name, 
					node1_ip   => $anvil->data->{job}{node1_sn_ip}, 
					node2_ip   => $anvil->data->{job}{node2_sn_ip}, 
				}});
				next;
			}
			else
			{
				last;
			}
		}
	}
	
	if ((not $anvil->data->{job}{node1_sn_ip}) or (not $anvil->data->{job}{node2_sn_ip}))
	{
		# Fail out.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0418", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0418"});
		$anvil->nice_exit({exit_code => 1});
	}
	
	return(0);
}

# The generates and saves the DRBD resource config.
sub create_resource_file
{
	my ($anvil) = @_;
	
	# Create the resource configuration body
	my $drbd_res_body = $anvil->Words->string({key => "file_0001", variables => {
		server           => $anvil->data->{job}{server_name}, 
		minor            => $anvil->data->{job}{drbd_minor}, 
		tcp_port         => $anvil->data->{job}{drbd_tcp_port}, 
		sn_network       => $anvil->data->{job}{sn_network},
		node1_sn_ip      => $anvil->data->{job}{node1_sn_ip},
		node2_sn_ip      => $anvil->data->{job}{node2_sn_ip}, 
		node1_short_name => $anvil->data->{job}{node1_short_host_name}, 
		node2_short_name => $anvil->data->{job}{node2_short_host_name}, 
		node1_lv_path    => $anvil->data->{job}{node1_lv_path}, 
		node2_lv_path    => $anvil->data->{job}{node2_lv_path}, 
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_res_body => $drbd_res_body }});
	
	# Write the file, then call drbdadm to validate it.
	my $drbd_res_file =  $anvil->data->{path}{directories}{drbd_resources}."/".$anvil->data->{job}{server_name}.".res";
	   $drbd_res_file =~ s/\/\//\//g;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_res_file => $drbd_res_file }});
	
	my $problem = $anvil->Storage->write_file({
		overwrite => 1, 
		backup    => 1,
		file      => $drbd_res_file, 
		body      => $drbd_res_body, 
		mode      => "0644",
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	
	if (not -e $drbd_res_file)
	{
		# Failed to write the config file.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0202,!!drbd_res_file!".$drbd_res_file."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0202", variables => { drbd_res_file => $drbd_res_file }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Call drbdadm and verify it's good.
	$problem = $anvil->DRBD->gather_data({debug => 2});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});

	# Verify that the resource is now shown in drbdadm
	my $server = $anvil->data->{job}{server_name};
	if ((not exists $anvil->data->{new}{resource}{$server}) or ($anvil->data->{new}{resource}{$server}{config_file} ne $drbd_res_file))
	{
		# Failed to load the resource.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0203,!!drbd_res_file!".$drbd_res_file."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0203", variables => { drbd_res_file => $drbd_res_file }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Manually call 'scan-drbd' to make sure that the new data is in scan_drbd_resources so that any 
	my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd";
	if (-f $shell_call)
	{
		$anvil->Job->update_progress({
			progress => 35,
			message  => "job_0218", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0218"});
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code,
		}});
	}
	
	$anvil->Job->update_progress({
		progress => 40,
		message  => "job_0190,!!resource!".$anvil->data->{job}{server_name}."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0190", variables => { resource => $anvil->data->{job}{server_name} }});
	
	return(0);
}

# This creates a logical volume
sub create_lv
{
	my ($anvil, $lv_path) = @_;
	
	$anvil->data->{job}{initialize_drbd} = 1;
	if (-e $lv_path)
	{
		# Don't create and don't initialize.
		$anvil->data->{job}{initialize_drbd} = 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::initialize_drbd' => $anvil->data->{job}{initialize_drbd} }});
		
		# Tell the user this might be an issue.
		$anvil->Job->update_progress({
			progress => 30,
			message  => "job_0187,!!lv_path!".$lv_path."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0187", variables => { lv_path => $lv_path }});
	}
	else
	{
		# Make sure we have enough space. This checks on the local system as the other node could 
		# record less free space before we create our LV.
		my $anvil_uuid         = $anvil->data->{job}{anvil_uuid};
		my $storage_group_uuid = $anvil->data->{job}{storage_group_uuid};
		if ($anvil->data->{job}{storage_size} > $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size})
		{
			# How's the local storage?
			$anvil->System->check_storage({debug => 2});
			my $host_name  = $anvil->Get->short_host_name();
			my $host_uuid  = $anvil->Get->host_uuid();
			my $vg_name    = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_name};
			my $free_space = $anvil->data->{lvm}{$host_name}{vg}{$vg_name}{free_space};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:host_name'         => $host_name,
				's2:host_uuid'         => $host_uuid, 
				's3:vg_name'           => $vg_name, 
				's4:free_space'        => $free_space." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $free_space}).")", 
				's5:job::storage_size' => $anvil->data->{job}{storage_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{job}{storage_size}}).")"
			}});
			
			if ((not $free_space) or ($free_space =~ /\D/) or ($anvil->data->{job}{storage_size} > $free_space))
			{
				# We're out.
				my $say_available_size = $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}})." (".$anvil->Convert->add_commas({number => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}}).")";
				my $say_requested_size = $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{job}{storage_size}})." (".$anvil->Convert->add_commas({number => $anvil->data->{job}{storage_size}}).")";
				my $say_storage_group  = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{group_name};
				$anvil->Job->update_progress({
					progress   => 100,
					message    => "error_0194,!!server_name!".$anvil->data->{job}{server_name}."!!,!!available_size!".$say_available_size."!!,!!requested_size!".$say_requested_size."!!,!!storage_group!".$say_storage_group."!!", 
					job_status => "failed", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0194", variables => { 
					server_name    => $anvil->data->{job}{server_name}, 
					available_size => $say_available_size, 
					requested_size => $say_requested_size, 
					storage_group  => $say_storage_group, 
				}});
				$anvil->nice_exit({exit_code => 1});
			}
		}
		
		# Create the LV.
		my $host_uuid    = $anvil->Get->host_uuid();
		my $vg_name      = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_name};
		my $extent_size  = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_extent_size};
		my $extent_count = int($anvil->data->{job}{storage_size} / $extent_size);
		my $shell_call   = $anvil->data->{path}{exe}{lvcreate}." -l ".$extent_count." -n ".$anvil->data->{job}{server_name}."_0 ".$vg_name." -y";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:vg_name'      => $vg_name, 
			's2:extent_size'  => $extent_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $extent_size}).")", 
			's3:extent_count' => $extent_count, 
			's4:shell_call'   => $shell_call,
		}});
		
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code,
		}});
		
		if (not -e $lv_path)
		{
			# Something went wrong.
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0201,!!lv_path!".$lv_path."!!,!!lv_create!".$shell_call."!!,!!return_code!".$return_code."!!,!!output!".$output."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0201", variables => { 
				lv_path     => $lv_path, 
				lv_create   => $shell_call, 
				return_code => $return_code, 
				output      => $output, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Call scan-lvm now to make sure we've recorded this LV.
		$anvil->Job->update_progress({
			progress => 28,
			message  => "job_0464", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0464"});
		my $agent = "scan-lvm";
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0740", variables => { agent => $agent }});
		
		$shell_call = $anvil->data->{path}{directories}{scan_agents}."/".$agent."/".$agent.$anvil->Log->switches();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
		
		# Report 
		$anvil->Job->update_progress({
			progress => 30,
			message  => "job_0189,!!lv_path!".$lv_path."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0189", variables => { lv_path => $lv_path }});
	}
	
	return(0);
}

# This picks out the next free DRBD minor number and TCP port if it's not set in the job data.
sub check_drbd_minor_and_port
{
	my ($anvil) = @_;

	# Read in VG information;
	my $problem = $anvil->Storage->get_storage_group_details({storage_group_uuid => $anvil->data->{job}{storage_group_uuid}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	if ($problem)
	{
		# Failed to collect storage group data.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0199,!!storage_group_uuid!".$anvil->data->{job}{storage_group_uuid}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0199", variables => { storage_group_uuid => $anvil->data->{job}{storage_group_uuid} }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	if (not $anvil->data->{job}{peer_mode})
	{
		# We're primary, so query the minor number and TCP port
		# The peer must use the TCP and minor as set in the job
		($anvil->data->{job}{drbd_minor}, $anvil->data->{job}{drbd_tcp_port}) = $anvil->DRBD->get_next_resource({
			debug         => 2, 
			anvil_uuid    => $anvil->data->{job}{anvil_uuid},
			resource_name => $anvil->data->{job}{server_name}, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			'job::drbd_minor'    => $anvil->data->{job}{drbd_minor},
			'job::drbd_tcp_port' => $anvil->data->{job}{drbd_tcp_port}, 
		}});
	}
	
	# If we don't have a DRBD minor or TCP port, we're stuck.
	if (($anvil->data->{job}{drbd_minor} eq "") or ($anvil->data->{job}{drbd_tcp_port} eq ""))
	{
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0200", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0200"});
		$anvil->nice_exit({exit_code => 1});
	}
	
	return(0);
}

# This parses and verifies the job data
sub parse_job_data
{
	my ($anvil) = @_;
	
	$anvil->data->{job}{server_name}        = "";
	$anvil->data->{job}{cpu_cores}          = "";
	$anvil->data->{job}{ram}                = "";
	$anvil->data->{job}{storage_group_uuid} = "";
	$anvil->data->{job}{storage_size}       = "";
	$anvil->data->{job}{install_iso_uuid}   = "";
	$anvil->data->{job}{driver_iso_uuid}    = "";
	$anvil->data->{job}{drbd_minor}         = "";
	$anvil->data->{job}{drbd_tcp_port}      = "";
	$anvil->data->{job}{peer_mode}          = 0;
	foreach my $line (split/\n/, $anvil->data->{jobs}{job_data})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
		if ($line =~ /server_name=(.*)$/)
		{
			$anvil->data->{job}{server_name} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::server_name' => $anvil->data->{job}{server_name} }});
		}
		if ($line =~ /server_uuid=(.*)$/)
		{
			$anvil->data->{job}{server_uuid} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::server_uuid' => $anvil->data->{job}{server_uuid} }});
		}
		if ($line =~ /cpu_cores=(.*)$/)
		{
			$anvil->data->{job}{cpu_cores} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::cpu_cores' => $anvil->data->{job}{cpu_cores} }});
		}
		if ($line =~ /ram=(.*)$/)
		{
			$anvil->data->{job}{ram} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::ram' => $anvil->data->{job}{ram} }});
			
			# If needed, convert the RAM to bytes.
			if ($anvil->data->{job}{ram} =~ /\D/)
			{
				$anvil->data->{job}{ram} = $anvil->Convert->human_readable_to_bytes({debug => 2, size => $anvil->data->{job}{ram}});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::ram' => $anvil->data->{job}{ram} }});
			}
		}
		if ($line =~ /storage_group_uuid=(.*)$/)
		{
			$anvil->data->{job}{storage_group_uuid} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::storage_group_uuid' => $anvil->data->{job}{storage_group_uuid} }});
		}
		if ($line =~ /storage_size=(.*)$/)
		{
			$anvil->data->{job}{storage_size} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::storage_size' => $anvil->data->{job}{storage_size} }});
			
			# If needed, convert the disk size to bytes.
			if ($anvil->data->{job}{storage_size} =~ /\D/)
			{
				$anvil->data->{job}{storage_size} = $anvil->Convert->human_readable_to_bytes({debug => 2, size => $anvil->data->{job}{storage_size}});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::storage_size' => $anvil->data->{job}{storage_size} }});
			}
		}
		if ($line =~ /install_iso=(.*)$/)
		{
			$anvil->data->{job}{install_iso_uuid} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::install_iso_uuid' => $anvil->data->{job}{install_iso_uuid} }});
		}
		if ($line =~ /driver_iso=(.*)$/)
		{
			$anvil->data->{job}{driver_iso_uuid} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::driver_iso_uuid' => $anvil->data->{job}{driver_iso_uuid} }});
		}
		if ($line =~ /peer_mode=true$/)
		{
			# We'll ONLY setup our DRBD resource, nothing else.
			$anvil->data->{job}{peer_mode} = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::peer_mode' => $anvil->data->{job}{peer_mode} }});
		}
		if ($line =~ /drbd_minor=(.*)$/)
		{
			$anvil->data->{job}{drbd_minor} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::drbd_minor' => $anvil->data->{job}{drbd_minor} }});
		}
		if ($line =~ /drbd_tcp_port=(.*)$/)
		{
			$anvil->data->{job}{drbd_tcp_port} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::drbd_tcp_port' => $anvil->data->{job}{drbd_tcp_port} }});
		}
		if ($line =~ /os=(.*)$/)
		{
			$anvil->data->{job}{os} = $1;
			$anvil->data->{job}{os} =~ s/^\s+//;
			$anvil->data->{job}{os} =~ s/\s+?//;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::os' => $anvil->data->{job}{os} }});
		}
		if ($line =~ /network=(.*)$/)
		{
			$anvil->data->{job}{network}{raw} =  $1;
			$anvil->data->{job}{network}{raw} =~ s/^\s+//;
			$anvil->data->{job}{network}{raw} =~ s/\s+?//;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::network::raw' => $anvil->data->{job}{network}{raw} }});
			
			# Now break up the data, if needed.
			my $problem = process_network($anvil, $anvil->data->{job}{network}{raw});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
			if ($problem)
			{
				$anvil->Job->update_progress({
					progress   => 100,
					message    => "error_0398,!!network!".$anvil->data->{job}{network}{raw}."!!", 
					job_status => "failed", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0398", variables => { job_uuid => $anvil->data->{job}{network}{raw} }});
				$anvil->nice_exit({exit_code => 1});
			}
		}
	}
	
	# We need a server name and storage group UUID regardless of which mode we're in.
	if (not $anvil->data->{job}{server_name})
	{
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0187,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0187", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }});
		$anvil->nice_exit({exit_code => 1});
	}
	if (not $anvil->data->{job}{storage_group_uuid})
	{
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0192,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0192", variables => { 
			server_name => $anvil->data->{job}{server_name}, 
			job_uuid    => $anvil->data->{switches}{'job-uuid'},
		}});
		$anvil->nice_exit({exit_code => 1});
	}
	if (not $anvil->data->{job}{storage_size})
	{
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0193,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0193", variables => { 
			server_name => $anvil->data->{job}{server_name}, 
			job_uuid    => $anvil->data->{switches}{'job-uuid'},
		}});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# If we're the peer, there's only a few things we need to check.
	if (not $anvil->data->{job}{peer_mode})
	{
		# The server should already exist on the peer. All we need to do is create our LV, create the
		# DRBD resource, bring the resource up, and save the XML definition file.
		
		# Is the server name unique?
		$anvil->Database->get_servers();
		my $anvil_uuid = $anvil->data->{job}{anvil_uuid};
		my $server     = $anvil->data->{job}{server_name};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			anvil_uuid => $anvil_uuid,
			server     => $server, 
		}});
		if (exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server})
		{
			# Is this name used by a server marked as DELETED?
			my $server_uuid  = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server}{server_uuid};
			my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				server_uuid  => $server_uuid,
				server_state => $server_state, 
			}});
			if ($server_state ne "DELETED")
			{
				# Duplicate name
				$anvil->Job->update_progress({
					progress   => 100,
					message    => "error_0198,!!server_name!".$anvil->data->{job}{server_name}."!!", 
					job_status => "failed", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0198", variables => { server_name => $anvil->data->{job}{server_name} }});
				$anvil->nice_exit({exit_code => 1});
			}
		}
		
		if (not $anvil->data->{job}{server_name})
		{
			# No server name given
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0188,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0188", variables => { 
				server_name => $anvil->data->{job}{server_name}, 
				job_uuid    => $anvil->data->{switches}{'job-uuid'},
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		elsif ($anvil->data->{job}{cpu_cores} > $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads})
		{
			# Too many cores requested
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0189,!!server_name!".$anvil->data->{job}{server_name}."!!,!!available_cores!".$anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}."!!,!!requested_cores!".$anvil->data->{job}{cpu_cores}."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0189", variables => { 
				server_name     => $anvil->data->{job}{server_name}, 
				available_cores => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}, 
				requested_cores => $anvil->data->{job}{server_name}, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		if (not $anvil->data->{job}{ram})
		{
			# No requested RAM
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0190,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0190", variables => { 
				server_name => $anvil->data->{job}{server_name}, 
				job_uuid    => $anvil->data->{switches}{'job-uuid'},
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		elsif ($anvil->data->{job}{ram} > $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available})
		{
			# Too many cores requested
			my $say_available_ram = $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}})." (".$anvil->Convert->add_commas({number => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")";
			my $say_requested_ram = $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{job}{ram}})." (".$anvil->Convert->add_commas({number => $anvil->data->{job}{ram}}).")"; 
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0191,!!server_name!".$anvil->data->{job}{server_name}."!!,!!available_ram!".$say_available_ram."!!,!!requested_ram!".$say_requested_ram."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0191", variables => { 
				server_name   => $anvil->data->{job}{server_name}, 
				available_ram => $say_available_ram, 
				requested_ram => $say_requested_ram, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		
		if (not $anvil->data->{job}{install_iso_uuid})
		{
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0195,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0195", variables => { 
				server_name => $anvil->data->{job}{server_name}, 
				job_uuid    => $anvil->data->{switches}{'job-uuid'},
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		my $install_iso_uuid = $anvil->data->{job}{install_iso_uuid};
		my $install_iso      = $anvil->data->{files}{file_uuid}{$install_iso_uuid}{file_directory}."/".$anvil->data->{files}{file_uuid}{$install_iso_uuid}{file_name};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			install_iso_uuid => $install_iso_uuid,
			install_iso      => $install_iso, 
		}});
		if (-e $install_iso)
		{
			$anvil->data->{job}{install_iso_path} = $install_iso;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				'job::install_iso_path' => $anvil->data->{job}{install_iso_path}, 
			}});
		}
		else
		{
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0196,!!server_name!".$anvil->data->{job}{server_name}."!!,!!install_iso!".$install_iso."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0196", variables => { 
				server_name => $anvil->data->{job}{server_name}, 
				install_iso => $install_iso,
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Driver disc is optional.
		$anvil->data->{new_server}{driver_iso_path} = "";
		if (($anvil->data->{job}{driver_iso_uuid}) && ($anvil->data->{job}{driver_iso_uuid} ne "none"))
		{
			my $driver_iso_uuid = $anvil->data->{job}{driver_iso_uuid};
			my $driver_iso      = $anvil->data->{files}{file_uuid}{$driver_iso_uuid}{file_directory}."/".$anvil->data->{files}{file_uuid}{$driver_iso_uuid}{file_name};
			if (-e $driver_iso)
			{
				$anvil->data->{job}{driver_iso_path} = $driver_iso;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					'job::driver_iso_path' => $anvil->data->{job}{driver_iso_path}, 
				}});
			}
			else
			{
				$anvil->Job->update_progress({
					progress   => 100,
					message    => "error_0197,!!server_name!".$anvil->data->{job}{server_name}."!!,!!driver_iso!".$driver_iso."!!", 
					job_status => "failed", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0197", variables => { 
					server_name => $anvil->data->{job}{server_name}, 
					driver_iso => $driver_iso,
				}});
				$anvil->nice_exit({exit_code => 1});
			}
		}
		if (not $anvil->data->{job}{os})
		{
			# No requested RAM
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0208,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0208", variables => { 
				server_name => $anvil->data->{job}{server_name}, 
				job_uuid    => $anvil->data->{switches}{'job-uuid'},
			}});
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	return(0);
}

sub process_network
{
	my ($anvil, $string) = @_;
	
	my $problem    = 0;
	my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
	my $anvil_name = $anvil->data->{new_server}{anvil_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid => $anvil_uuid,
		anvil_name => $anvil_name, 
	}});
	
	$anvil->data->{job}{network}{bridge} = "";
	$anvil->data->{job}{network}{mac}    = "";
	$anvil->data->{job}{network}{model}  = "";
	if (($string =~ /bridge=(.*?),/) or ($string =~ /bridge=(.*?)$/))
	{
		my $bridge_name = $1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bridge_name => $bridge_name }});
		
		# Make sure the bridge is valid.
		if ((exists $anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}) && ($anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}{on_nodes}))
		{
			$anvil->data->{job}{network}{bridge} = $bridge_name;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::network::bridge' => $anvil->data->{job}{network}{bridge} }});
		}
		else
		{
			print "The requested bridge: [".$bridge_name."] is not valid.\n";
			print "Valid bridges on the Anvil! node: [".$anvil_name."] are:'\n";
			foreach my $bridge_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{bridges}})
			{
				next if $anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}{on_nodes};
				print "- ".$bridge_name."\n";
			}
			$problem = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		}
	}
	if (($string =~ /mac=(.*?),/) or ($string =~ /mac=(.*?)$/))
	{
		my $mac_address = $1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mac_address => $mac_address }});
		
		# Validate it
		if (not $anvil->Validate->mac({mac => $mac_address}))
		{
			print "The requested MAC address: [".$mac_address."] does not appear to be valid.\n";
			$problem = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		}
		else
		{
			$anvil->data->{job}{network}{mac} = $mac_address;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::network::mac' => $anvil->data->{job}{network}{mac} }});
		}
	}
	if (($string =~ /model=(.*?),/) or ($string =~ /model=(.*?)$/))
	{
		my $model = $1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { model => $model }});
		
		$anvil->Get->virsh_list_net({debug => 2});
		if (exists $anvil->data->{osinfo}{net}{$model})
		{
			$anvil->data->{job}{network}{model} = $model;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::network::model' => $anvil->data->{job}{network}{model} }});
		}
		else
		{
			print "The requested network model: [".$model."] does not appear to be valid.\n";
			print "Valid network models are:\n";
			foreach my $model (sort {$a cmp $b} keys %{$anvil->data->{osinfo}{net}})
			{
				print "- ".$model." (".$anvil->data->{osinfo}{net}{$model}{vendor}." - ".$anvil->data->{osinfo}{net}{$model}{product}.")\n";
			}
			$problem = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		}
	}
	
	return($problem);
}

sub check_anvil
{
	my ($anvil) = @_;
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid},
	}});
	if (($anvil->data->{switches}{anvil}) && (not $anvil->data->{switches}{'anvil-uuid'}) && (not $anvil->data->{switches}{'anvil-name'}))
	{
		if ($anvil->Validate->uuid({uuid => $anvil->data->{switches}{anvil}}))
		{
			$anvil->data->{switches}{'anvil-uuid'} = $anvil->data->{switches}{anvil};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid},
			}});
		}
		else
		{
			$anvil->data->{switches}{'anvil-name'} = $anvil->data->{switches}{anvil};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::anvil_name" => $anvil->data->{new_server}{anvil_name},
			}});
		}
	}
	
	# Do we know or can we find the Anvil! UUID?
	if (not $anvil->data->{new_server}{anvil_uuid})
	{
		$anvil->data->{new_server}{anvil_uuid} = $anvil->data->{switches}{'anvil-uuid'} ? $anvil->data->{switches}{'anvil-uuid'} : "";
		$anvil->data->{new_server}{anvil_name} = $anvil->data->{switches}{'anvil-name'} ? $anvil->data->{switches}{'anvil-name'} : "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid},
			"new_server::anvil_name" => $anvil->data->{new_server}{anvil_name},
		}});
	}
	
	if ((not $anvil->data->{new_server}{anvil_uuid}) && (not $anvil->data->{new_server}{anvil_name}))
	{
		# Nothing given. Is this host a node, perhaps?
		my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
		
		if ($anvil_uuid)
		{
			$anvil->data->{new_server}{anvil_uuid} = $anvil_uuid;
			$anvil->data->{new_server}{anvil_name} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::anvil_name" => $anvil->data->{new_server}{anvil_name},
				"new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid},
			}});
		}
	}
	elsif (not $anvil->data->{new_server}{anvil_uuid})
	{
		$anvil->data->{new_server}{anvil_uuid} = $anvil->Cluster->get_anvil_uuid({anvil_name => $anvil->data->{new_server}{anvil_name}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid} }});
	}
	elsif (not $anvil->data->{new_server}{anvil_name})
	{
		$anvil->data->{new_server}{anvil_name} = $anvil->Cluster->get_anvil_name({anvil_uuid => $anvil->data->{new_server}{anvil_uuid}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "new_server::anvil_name" => $anvil->data->{new_server}{anvil_name} }});
	}
	
	return(0);
}

# Go through a series of questions to ask the user how they want to build their server.
sub interactive_question
{
	my ($anvil) = @_;
	
	# Process the anvil switch
	check_anvil($anvil);
	
	$anvil->data->{new_server}{name} = $anvil->data->{switches}{name} ? $anvil->data->{switches}{name} : "";
	$anvil->data->{new_server}{uuid} = $anvil->data->{switches}{uuid} ? $anvil->data->{switches}{uuid} : "";
	
	# If this is a node, load the anvil_uuid automatically.
	
	my $termios = new POSIX::Termios;
	$termios->getattr;
	my $ospeed = $termios->getospeed;
	
	my $terminal = Tgetent Term::Cap { TERM => undef, OSPEED => $ospeed };
	$terminal->Trequire(qw/ce ku kd/);
	
	interactive_ask_anvil_name($anvil, $terminal);
	interactive_ask_server_name($anvil, $terminal);
	interactive_ask_server_cpu($anvil, $terminal);
	interactive_ask_server_ram($anvil, $terminal);
	interactive_ask_server_storage_group($anvil, $terminal);
	interactive_ask_server_storage_size($anvil, $terminal);
	interactive_ask_server_install_media($anvil, $terminal);
	interactive_ask_server_driver_disc($anvil, $terminal);
	interactive_ask_server_os($anvil, $terminal);
	interactive_ask_server_confirm($anvil, $terminal);
	
	return(0);
}

sub interactive_ask_anvil_name
{
	my ($anvil, $terminal) = @_;
	
	$anvil->Database->get_anvils();
	
	my $retry = 0;
	while(1)
	{
		my $default_anvil = $anvil->data->{new_server}{anvil_name};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_anvil => $default_anvil }});
		if (not $default_anvil)
		{
			my $known_anvils = keys %{$anvil->data->{anvils}{anvil_name}};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { known_anvils => $known_anvils }});
			if (not $known_anvils)
			{
				print $anvil->Words->string({key => "job_0149"})."\n";
				$anvil->nice_exit({exit_code => 1});
			}
			elsif ($known_anvils == 1)
			{
				foreach my $anvil_name (keys %{$anvil->data->{anvils}{anvil_name}})
				{
					$default_anvil = $anvil_name;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_anvil => $default_anvil }});
				}
			}
		}
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $default_anvil }})."\n\n\n";
		
		# Show all the current server names.
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0152"})."\n\n";
		}
		print $anvil->Words->string({key => "job_0153"})."\n";
		foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
		{
			print "- ".$anvil_name.": - ".$anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description}."\n";
		}
		
		print $terminal->Tgoto('cm', 0, 2)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ((not $answer) && ($default_anvil))
		{
			$answer = $default_anvil;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		}
		
		# Reload in case a new anvil! was saved while we waited.
		$anvil->Database->get_anvils();
		if (exists $anvil->data->{anvils}{anvil_name}{$answer})
		{
			# Valid.
			$anvil->data->{new_server}{anvil_name} = $answer;
			$anvil->data->{new_server}{anvil_uuid} = $anvil->Cluster->get_anvil_uuid({anvil_name => $answer});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::anvil_name" => $anvil->data->{new_server}{anvil_name},
				"new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid},
			}});
			
			print $anvil->Words->string({key => "job_0154", variables => { 
				anvil_name => $anvil->data->{new_server}{anvil_name}, 
				anvil_uuid => $anvil->data->{new_server}{anvil_uuid}, 
			}})."\n";
			my $problem = $anvil->Get->available_resources({
				debug      => 2,
				anvil_uuid => $anvil->data->{new_server}{anvil_uuid},
			});
			if ($problem eq "!!no_data!!")
			{
				# We can't proceed.
				print $anvil->Words->string({key => "job_0217"})."\n";
				$anvil->nice_exit({exit_code => 1});
			}
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	# Verify that the Anvil! has enough RAM and disk space.
	my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"anvil_resources::${anvil_uuid}::ram::available" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")",
	}});
	if ($anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available} < (1**20))
	{
		print $anvil->Words->string({key => "job_0155"})."\n";
		print $anvil->Words->string({key => "job_0156", variables => { available_ram => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}) }})."\n";
		$anvil->nice_exit({exit_code => 2});
	}
	
	return(0);
}

sub interactive_ask_server_name
{
	my ($anvil, $terminal) = @_;
	
	$anvil->Database->get_servers({debug => 2});
	
	### TODO: Figure out how many rows we have and break the server list into columns if too long.
	my $retry     = 0;
	my $duplicate = "";
	while(1)
	{
		my $default = "";
		if ($anvil->data->{switches}{name})
		{
			$default = $anvil->data->{switches}{name};
		}
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n\n\n";
		
		# Show all the current server names.
		if ($retry)
		{
			if ($duplicate)
			{
				print $anvil->Words->string({key => "job_0219", variables => { server_name => $duplicate }})."\n\n";
				$duplicate = "";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { duplicate => $duplicate }});
			}
			else
			{
				print $anvil->Words->string({key => "job_0159"})."\n\n";
			}
		}
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		print $anvil->Words->string({key => "job_0160", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
		{
			my $server_uuid  = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
			my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				server_uuid  => $server_uuid,
				server_state => $server_state, 
			}});
			if ($server_state eq "DELETED")
			{
				### NOTE: This could get cluttered, so for now we'll not show them.
				#print $anvil->Words->string({key => "message_0220", variables => { server_name => $server_name }})."\n";
			}
			else
			{
				print $anvil->Words->string({key => "message_0219", variables => { 
					server_name  => $server_name,
					server_state => $server_state, 
				}})."\n";
			}
		}
		
		print $terminal->Tgoto('cm', 0, 3)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ((not $answer) && ($default))
		{
			$answer = $default;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		}
		
		# Reload in case a new anvil! was saved while we waited.
		$anvil->Database->get_servers();
		if ($answer)
		{
			# Duplicate?
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
			if (exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$answer})
			{
				my $server_uuid  = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$answer}{server_uuid};
				my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					server_uuid  => $server_uuid,
					server_state => $server_state, 
				}});
				if ($server_state eq "DELETED")
				{
					# Valid, we can re-use deleted server names. We'll also re-use the 
					# UUID, if the user didn't specifically specify a UUID.
					$anvil->data->{new_server}{name} = $answer;
					$anvil->data->{new_server}{uuid} = $server_uuid if not $anvil->data->{new_server}{uuid};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"new_server::name" => $anvil->data->{new_server}{name},
						"new_server::uuid" => $anvil->data->{new_server}{uuid},
					}});
				}
				else
				{
					# Invalid, duplicate.
					$duplicate = $answer;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { duplicate => $duplicate }});
				}
			}
			else
			{
				# Valid.
				$anvil->data->{new_server}{name} = $answer;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"new_server::name" => $anvil->data->{new_server}{name},
				}});
			}
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_cpu
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		$anvil->Database->get_anvils();
		$anvil->Get->available_resources({
			debug      => 2,
			anvil_uuid => $anvil_uuid,
		});
		
		my $default_cpu = $anvil->data->{switches}{cpu};
		if (not $default_cpu)
		{
			# Default to 2, unless only one core is available.
			$default_cpu = $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads} == 1 ? 1 : 2;
		}
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $default_cpu }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0161", variables => { max_cores => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads} }})."\n\n";
		}
		print $anvil->Words->string({key => "job_0162", variables => { 
			cores   => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{cores},
			threads => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads},
		}})."\n";
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	
		print $anvil->Words->string({key => "job_0163", variables => { core => 1, model => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{cpu}{model} }})."\n";
		print $anvil->Words->string({key => "job_0163", variables => { core => 2, model => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node2_host_uuid}{cpu}{model} }})."\n";
		
		print $terminal->Tgoto('cm', 0, 4)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if ($answer eq "")
		{
			$answer = $default_cpu; 
		}
		if (($answer) && ($answer =~ /^\d+$/) && ($answer <= $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}))
		{
			# Valid.
			$anvil->data->{new_server}{cpu} = $answer;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::cpu" => $anvil->data->{new_server}{cpu},
			}});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_ram
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		$anvil->Database->get_anvils();
		$anvil->Get->available_resources({
			debug      => 2,
			anvil_uuid => $anvil_uuid,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"anvil_resources::${anvil_uuid}::ram::available" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")",
		}});
		
		my $default_ram = $anvil->data->{switches}{ram};
		if (not $default_ram)
		{
			# Default to 2, unless only one core is available.
			my $say_2g = (2*(2**30));
			if ($anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available} > $say_2g)
			{
				$default_ram = $anvil->Convert->bytes_to_human_readable({'bytes' => $say_2g});
			}
			else
			{
				$default_ram = $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}});
			}
		}
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $default_ram }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0166", variables => { 
				ram_total     => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{hardware}}),
				ram_available => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}), 
			}})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		
		print $anvil->Words->string({key => "job_0167", variables => { 
			ram_available => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}, unit => "M"}).")", 
			ram_reserved  => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{reserved}}), 
			ram_allocated => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{allocated}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{allocated}, unit => "M"}).")",
			ram_node1     => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{ram}{hardware}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{ram}{hardware}, unit => "M"}).")",
			ram_node2     => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node2_host_uuid}{ram}{hardware}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node2_host_uuid}{ram}{hardware}, unit => "M"}).")",
		}})."\n";
		
		print $terminal->Tgoto('cm', 0, 5)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if ($answer eq "")
		{
			$answer = $default_ram; 
		}
		if ($answer)
		{
			# Convert to bytes.
			my $answer_bytes = $anvil->Convert->human_readable_to_bytes({
				base2 => 1,
				size  => $answer, 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer_bytes => $answer_bytes }});
			if (($answer_bytes eq "!!error!!") or 
			    (not $answer_bytes)            or 
			    ($answer_bytes < (640*1024))   or 
			    ($answer_bytes > $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}))
			{
				# Invalid
				$retry = 1;
			}
			else
			{
				# Valid.
				$anvil->data->{new_server}{ram} = $answer_bytes;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"new_server::ram" => $anvil->data->{new_server}{ram},
				}});
				
				last;
			}
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_storage_group
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		$anvil->Database->get_anvils();
		$anvil->Get->available_resources({
			debug      => 2, 
			anvil_uuid => $anvil_uuid,
		});
		
		# I need a list of Storage groups,
		my $say_ram               = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}});
		my $storage_groups        = [""];
		my $show_list             = "";
		my $position              = 0;
		my $default_storage_group = 1;
		foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}})
		{
			   $position++;
			my $storage_group_uuid = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid};
			my $vg_size            = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size};
			my $vg_free            = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:storage_group_name' => $storage_group_name,
				's2:storage_group_uuid' => $storage_group_uuid, 
				's3:vg_size'            => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")",
				's4:vg_free'            => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")",
			}});
			
			if ($anvil->data->{switches}{'storage-group'})
			{
				if (($anvil->data->{switches}{'storage-group'} eq $storage_group_uuid) or 
				    ($anvil->data->{switches}{'storage-group'} eq $storage_group_name))
				{
					$default_storage_group = $position;
				}
			}
			push @{$storage_groups}, $storage_group_uuid;
			$show_list .= " - ".$position.") ".$storage_group_name." (".$storage_group_uuid.")\n";
			$show_list .= $anvil->Words->string({key => "job_0169", variables => { 
				vg_free => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}),
				vg_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}),
			}})."\n";
		}
		
		if (not $default_storage_group)
		{
			# Default to 2, unless only one core is available.
			$default_storage_group = 1;
		}
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $say_ram }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $default_storage_group }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0171"})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		
		print $anvil->Words->string({key => "job_0171"})."\n";
		print $show_list."\n";
		
		print $terminal->Tgoto('cm', 0, 6)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if ($answer eq "")
		{
			$answer = $default_storage_group; 
		}
		if (($answer) && ($answer =~ /^\d+$/) && (exists $storage_groups->[$answer]) && ($storage_groups->[$answer]))
		{
			# Valid.
			$anvil->data->{new_server}{storage_group} = $storage_groups->[$answer];
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::storage_group" => $anvil->data->{new_server}{storage_group},
			}});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_storage_size
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid         = $anvil->data->{new_server}{anvil_uuid};
		my $storage_group_uuid = $anvil->data->{new_server}{storage_group};
		my $storage_group_name = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
		my $vg_size            = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size};
		my $vg_free            = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:anvil_uuid'         => $anvil_uuid,
			's2:storage_group_name' => $storage_group_name,
			's3:storage_group_uuid' => $storage_group_uuid, 
			's4:vg_size'            => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")",
			's5:vg_free'            => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")",
		}});
		
		$anvil->Database->get_anvils();
		$anvil->Database->get_storage_group_data({debug => 2});
		$anvil->Get->available_resources({
			debug      => 2, 
			anvil_uuid => $anvil_uuid,
		});
		
		$vg_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size};
		$vg_free = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:vg_size' => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")",
			's2:vg_free' => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")",
		}});
		
		# I need a list of Storage groups,
		my $default_storage_size = 0;
		   $vg_free              = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
		   $default_storage_size = $anvil->data->{switches}{'storage-size'};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			vg_free              => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free})." (".$anvil->Convert->add_commas({number => $vg_free}).")",
			default_storage_size => $default_storage_size, 
		}});
		   
		if (not $default_storage_size)
		{
			if ($vg_free < (80 * (2**30)))
			{
				# Too small for default
				$default_storage_size = $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_storage_size => $default_storage_size }});
			}
			else
			{
				$default_storage_size = "80 GiB";
			}
		}
		
		my $say_ram           = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}});
		my $say_storage_group = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $say_ram }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
		print $anvil->Words->string({key => "job_0173", variables => { storage_size => $default_storage_size }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0174", variables => { max_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}) }})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		
		print $anvil->Words->string({key => "job_0175", variables => { 
			storage_group  => $say_storage_group,
			available_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}),
		}})."\n";
		
		print $terminal->Tgoto('cm', 0, 7)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if (($answer eq "") or ($answer eq "100%"))
		{
			$answer = $default_storage_size; 
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		}
		if ($answer) 
		{
			# Is the size sensible?
			my $answer_bytes = $anvil->Convert->human_readable_to_bytes({
				base2 => 1,
				size  => $answer, 
			});
			# Make sure they've asked for at least 10 MiB
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer_bytes => $answer_bytes }});
			
			# If the answer is within 1GiB set it to the available space.
			if ($answer_bytes =~ /^\d+$/)
			{
				my $difference = $answer_bytes - $vg_free;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
				
				$difference =~ s/^-//;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					difference => $difference." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $difference}).")",
				}});
				
				if ($difference < (2**30))
				{
					# Close enough.
					$answer_bytes = $vg_free;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						answer_bytes => $answer_bytes." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $answer_bytes}).")",
					}});
				}
			}
			
			if (($answer_bytes eq "!!error!!") or 
			    (not $answer_bytes)            or 
			    ($answer_bytes < (10*(2**20))) or 
			    ($answer_bytes > $vg_free))
			{
				# Invalid
				$retry = 1;
			}
			else
			{
				$anvil->data->{new_server}{storage_size} = $answer_bytes;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"new_server::storage_size" => $anvil->data->{new_server}{storage_size},
				}});
				
				last;
			}
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_install_media
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		$anvil->Database->get_files({debug => 2});
		$anvil->Database->get_file_locations({debug => 2});
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
		
		# Create an array of install ISOs. 
		my $install_isos = [""];
		my $iso_list     = "";
		my $position     = 0;
		foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}})
		{
			my $file_uuid            = $anvil->data->{files}{file_name}{$file_name}{file_uuid};
			my $file_directory       = $anvil->data->{files}{file_name}{$file_name}{file_directory};
			my $file_size            = $anvil->data->{files}{file_name}{$file_name}{file_size};
			my $file_type            = $anvil->data->{files}{file_name}{$file_name}{file_type};
			my $file_path            = $file_directory."/".$file_name;
			my $file_location_uuid   = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
			my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				file_uuid            => $file_uuid, 
				file_name            => $file_name,
				file_directory       => $file_directory, 
				file_size            => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")",
				file_type            => $file_type, 
				file_path            => $file_path, 
				file_location_uuid   => $file_location_uuid, 
				file_location_active => $file_location_active, 
			}});
			
			next if $file_type ne "iso";
			next if not $file_location_active;
			
			# Still here?
			$position++;
			push @{$install_isos}, $file_uuid;
			$iso_list .= " - ".$position.") ".$file_name." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")\n";
		}
		
		my $say_ram            = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}});
		my $storage_group_uuid = $anvil->data->{new_server}{storage_group};
		my $say_storage_group  = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
		my $say_storage_size   = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{storage_size}});
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $say_ram }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
		print $anvil->Words->string({key => "job_0173", variables => { storage_size => $say_storage_size }})."\n";
		print $anvil->Words->string({key => "job_0176", variables => { install_media => "" }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0177"})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		
		print $anvil->Words->string({key => "job_0178"})."\n";
		print $iso_list."\n";
		
		print $terminal->Tgoto('cm', 0, 8)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		if (($answer) && ($answer =~ /^\d+$/) && (exists $install_isos->[$answer]) && ($install_isos->[$answer]))
		{
			# Valid.
			$anvil->data->{new_server}{install_media} = $install_isos->[$answer];
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::install_media" => $anvil->data->{new_server}{install_media},
			}});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_driver_disc
{
	my ($anvil, $terminal) = @_;
	
	my $retry = 0;
	while(1)
	{
		$anvil->Database->get_files();
		$anvil->Database->get_file_locations();
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
		
		my $say_ram                 = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}});
		my $storage_group_uuid      = $anvil->data->{new_server}{storage_group};
		my $say_storage_group       = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
		my $say_storage_size        = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{storage_size}});
		my $install_media_file_uuid = $anvil->data->{new_server}{install_media};
		my $say_install_media       = $anvil->data->{files}{file_uuid}{$install_media_file_uuid}{file_name};
		
		# Create an array of install ISOs. 
		my $driver_isos = ["none"];
		my $iso_list     = $anvil->Words->string({key => "job_0179"})."\n";
		my $position     = 0;
		foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{files}{file_name}})
		{
			my $file_uuid            = $anvil->data->{files}{file_name}{$file_name}{file_uuid};
			my $file_directory       = $anvil->data->{files}{file_name}{$file_name}{file_directory};
			my $file_size            = $anvil->data->{files}{file_name}{$file_name}{file_size};
			my $file_type            = $anvil->data->{files}{file_name}{$file_name}{file_type};
			my $file_path            = $file_directory."/".$file_name;
			my $file_location_uuid   = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
			my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				file_uuid            => $file_uuid, 
				file_name            => $file_name,
				file_directory       => $file_directory, 
				file_size            => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")",
				file_type            => $file_type, 
				file_path            => $file_path, 
				file_location_uuid   => $file_location_uuid, 
				file_location_active => $file_location_active, 
			}});
			
			next if $file_type ne "iso";
			next if not $file_location_active;
			next if $file_uuid eq $install_media_file_uuid;
			
			# Still here?
			$position++;
			push @{$driver_isos}, $file_uuid;
			$iso_list .= " - ".$position.") ".$file_name." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")\n";
		}
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $say_ram }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
		print $anvil->Words->string({key => "job_0173", variables => { storage_size => $say_storage_size }})."\n";
		print $anvil->Words->string({key => "job_0176", variables => { install_media => $say_install_media }})."\n";
		print $anvil->Words->string({key => "job_0180", variables => { driver_disc => "#!string!unit_0005!#" }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0181"})."\n\n";
		}
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		
		print $anvil->Words->string({key => "job_0182"})."\n";
		print $iso_list."\n";
		
		print $terminal->Tgoto('cm', 0, 9)."? ";
		my $answer = <STDIN>;
		chomp $answer;
		
		$answer = 0 if $answer eq "";
		
		if (($answer =~ /^\d+$/) && (exists $driver_isos->[$answer]) && ($driver_isos->[$answer]))
		{
			# Valid.
			$anvil->data->{new_server}{driver_disc} = $driver_isos->[$answer];
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::driver_disc" => $anvil->data->{new_server}{driver_disc},
			}});
			
			last;
		}
		else
		{
			$retry = 1;
		}
	}
	
	return(0);
}

sub interactive_ask_server_os
{
	my ($anvil, $terminal) = @_;
	
	my $words_file = $anvil->data->{path}{words}{'words.xml'};
	my $language   = $anvil->Words->language;
	my $os_list    = "";
	my $default_os = $anvil->data->{switches}{os} ? $anvil->data->{switches}{os} : "";
	my $longest_os = 0;
	foreach my $os_code (split/,/, $anvil->data->{sys}{servers}{os_short_list})
	{
		$os_code =~ s/ //g;
		
		next if not exists $anvil->data->{osinfo}{'os-list'}{$os_code};
		if (length($os_code) > $longest_os)
		{
			$longest_os = length($os_code);
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_os => $longest_os }});
		}
	}
	
	$anvil->Get->virsh_list_os();
	foreach my $os_code (split/,/, $anvil->data->{sys}{servers}{os_short_list})
	{
		$os_code =~ s/ //g;
		
		next if not exists $anvil->data->{osinfo}{'os-list'}{$os_code};
		my $os_name = $anvil->data->{osinfo}{'os-list'}{$os_code}{name};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:os_code' => $os_code,
			's2:os_name' => $os_name, 
		}});
		
		# Still here?
		$os_list .= " - ".sprintf("%-${longest_os}s", $os_code)." - ".$os_name."\n";
	}
	
	my $retry = 0;
	while(1)
	{
		my $anvil_uuid              = $anvil->data->{new_server}{anvil_uuid};
		my $storage_group_uuid      = $anvil->data->{new_server}{storage_group};
		my $say_ram                 = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}});
		my $say_storage_group       = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
		my $say_storage_size        = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{storage_size}});
		my $install_media_file_uuid = $anvil->data->{new_server}{install_media};
		my $say_install_media       = $anvil->data->{files}{file_uuid}{$install_media_file_uuid}{file_name};
		my $driver_disc_file_uuid   = $anvil->data->{new_server}{driver_disc};
		my $say_driver_disc         = $driver_disc_file_uuid eq "none" ? "#!string!unit_0005!#" : $anvil->data->{files}{file_uuid}{$driver_disc_file_uuid}{file_name};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
		
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $say_ram }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
		print $anvil->Words->string({key => "job_0173", variables => { storage_size => $say_storage_size }})."\n";
		print $anvil->Words->string({key => "job_0176", variables => { install_media => $say_install_media }})."\n";
		print $anvil->Words->string({key => "job_0180", variables => { driver_disc => $say_driver_disc }})."\n";
		print $anvil->Words->string({key => "job_0198", variables => { os => $default_os }})."\n\n\n";
		
		if ($retry)
		{
			print $anvil->Words->string({key => "job_0197"})."\n\n";
		}
		
		print $anvil->Words->string({key => "job_0196"})."\n";
		print $os_list."\n";
		
		print $terminal->Tgoto('cm', 0, 10)."? ";
		my $os_name = "";
		my $answer  = <STDIN>;
		chomp $answer;
		
		if ((not $answer) && ($default_os))
		{
			$answer = $default_os;
		}
		if ($answer)
		{
			# Is this valid?
			if (exists $anvil->data->{osinfo}{'os-list'}{$answer})
			{
				$os_name = $anvil->data->{osinfo}{'os-list'}{$answer}{name};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { os_name => $os_name }});
			}
		}
		
		if ((not $answer) or ($os_name eq ""))
		{
			# invalid.
			$retry = 1;
		}
		else
		{
			$anvil->data->{new_server}{os}     = $answer;
			$anvil->data->{new_server}{say_os} = $os_name;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"new_server::os"     => $anvil->data->{new_server}{os},
				"new_server::say_os" => $anvil->data->{new_server}{say_os},
			}});
			
			last;
		}
	}
	
	return(0);
}

sub interactive_ask_server_confirm
{
	my ($anvil, $terminal) = @_;
	
	# terminal isn't set with --ci-test set.
	check_anvil($anvil);
	
	$anvil->Database->get_hosts({debug => 3});
	$anvil->Database->get_anvils({debug => 3});
	$anvil->Database->get_files({debug => 2});
	$anvil->Database->get_file_locations({debug => 2});
	$anvil->Get->virsh_list_os({debug => 3});
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid},
		"switches::ci-test"      => $anvil->data->{switches}{'ci-test'},
	}});
	if ($anvil->data->{switches}{'ci-test'})
	{
		### NOTE: Show available options;
		# anvil-provision-server --ci-test --options
		check_anvil($anvil);
		if (not $anvil->data->{new_server}{anvil_uuid})
		{
			# Instantly fatal
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0437"});
			$anvil->nice_exit({exit_code => 1});
		}
		
		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
		$anvil->Get->available_resources({
			debug      => 2,
			anvil_uuid => $anvil_uuid, 
		});
		
		$anvil->data->{new_server}{name}          = "";
		$anvil->data->{new_server}{uuid}          = "";
		$anvil->data->{new_server}{os}            = "";
		$anvil->data->{new_server}{cpu}           = "";
		$anvil->data->{new_server}{ram}           = "";
		$anvil->data->{new_server}{storage_group} = "";
		$anvil->data->{new_server}{storage_size}  = "";
		$anvil->data->{new_server}{install_media} = "";
		$anvil->data->{new_server}{driver_disc}   = "";
		
		my $problem = 0;
		if (not $anvil->data->{switches}{name})
		{
			my $print = $anvil->data->{switches}{options} ? 0 : 1;
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0438"});
			$problem = 1;
		}
		if (not $anvil->data->{switches}{os})
		{
			my $print = $anvil->data->{switches}{options} ? 0 : 1;
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0439"});
			$problem = 1;
		}
		else
		{
			# Make sure it's valid.
			my $os_key  = "os_list_".$anvil->data->{switches}{os};
			my $os_name = $anvil->Words->string({key => $os_key});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				os_key  => $os_key,
				os_name => $os_name, 
			}});
			if ($os_name =~ /#!not_found/)
			{
				my $print = $anvil->data->{switches}{options} ? 0 : 1;
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0440", variables => {
					os => $anvil->data->{switches}{os},
				}});
				$problem = 1;
			}
		}
		if (not $anvil->data->{switches}{cpu})
		{
			my $print = $anvil->data->{switches}{options} ? 0 : 1;
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0441", variables => {
				threads => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads},
			}});
			$problem = 1;
		}
		elsif (($anvil->data->{switches}{cpu} =~ /\D/) or ($anvil->data->{switches}{cpu} > $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}) or ($anvil->data->{switches}{cpu} < 1))
		{
			my $print = $anvil->data->{switches}{options} ? 0 : 1;
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0442", variables => {
				cores   => $anvil->data->{switches}{cpu},
				threads => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads},
			}});
			$problem = 1;
		}
		
		my $max_ram       = $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available};
		my $say_max_ram   = $anvil->Convert->bytes_to_human_readable({"bytes" => $max_ram});
		my $requested_ram = $anvil->Convert->human_readable_to_bytes({
			base2 => 1,
			size  => $anvil->data->{switches}{ram}, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			max_ram       => $max_ram, 
			say_max_ram   => $say_max_ram, 
			requested_ram => $requested_ram, 
		}});
		if (not $anvil->data->{switches}{ram})
		{
			my $print = $anvil->data->{switches}{options} ? 0 : 1;
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0443", variables => {
				max_ram => $say_max_ram,
			}});
			$problem = 1;
		}
		elsif (($requested_ram eq "!!error!!") or 
		       (not $requested_ram)            or 
		       ($requested_ram < (640*1024))   or 
		       ($requested_ram > $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}))
		{
			# Invalid
			my $print = $anvil->data->{switches}{options} ? 0 : 1;
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0444", variables => {
				ram     => $anvil->data->{switches}{ram},
				max_ram => $say_max_ram,
			}});
			$problem = 1;
		}
		
		my $max_storage_group_size = 0;
		my $storage_group_uuid     = "";
		if (not $anvil->data->{switches}{'storage-group'})
		{
			my $print = $anvil->data->{switches}{options} ? 0 : 1;
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0445"});
			foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}})
			{
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0446", variables => {
					name => $storage_group_name,
					uuid => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid},
				}});
			}
			$problem = 1;
		}
		else
		{
			# Make sure it's valid.
			my $storage_group = $anvil->data->{switches}{'storage-group'};
			if (exists $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group})
			{
				$storage_group_uuid     = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group}{storage_group_uuid};
				$max_storage_group_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					storage_group_uuid     => $storage_group_uuid, 
					max_storage_group_size => $max_storage_group_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $max_storage_group_size}).")", 
				}});
			}
			elsif (exists $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group})
			{
				$storage_group_uuid     = $storage_group;
				$max_storage_group_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					storage_group_uuid     => $storage_group_uuid, 
					max_storage_group_size => $max_storage_group_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $max_storage_group_size}).")", 
				}});
			}
			else
			{
				# Invalid
				my $print = $anvil->data->{switches}{options} ? 0 : 1;
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0447", variables => {
					storage_group => $storage_group,
				}});
				foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}})
				{
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0446", variables => {
						name => $storage_group_name,
						uuid => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid},
					}});
				}
				$problem = 1;
			}
		}
		my $say_max_storage_group_size = $max_storage_group_size ? $anvil->Convert->bytes_to_human_readable({"bytes" => $max_storage_group_size}) : "";
		if (not $anvil->data->{switches}{'storage-size'})
		{
			my $print = $anvil->data->{switches}{options} ? 0 : 1;
			if ($max_storage_group_size)
			{
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0448", variables => {
					storage_group_size => $say_max_storage_group_size,
				}});
			}
			else
			{
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0449"});
			}
			$problem = 1;
		}
		else
		{
			my $requested_disk = $anvil->Convert->human_readable_to_bytes({
				base2 => 1,
				size  => $anvil->data->{switches}{'storage-size'}, 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { requested_disk => $requested_disk }});
			if (($requested_disk eq "!!error!!") or 
			    ($requested_disk < (10*(2**20))) or 
			    ($requested_disk > $max_storage_group_size))
			{
				# Invalid
				my $print = $anvil->data->{switches}{options} ? 0 : 1;
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0450", variables => {
					storage_size => $anvil->data->{switches}{'storage-size'},
					max_size     => $say_max_storage_group_size,
				}});
				$problem = 1;
			}
		}
		if ($anvil->Validate->uuid({uuid => $anvil->data->{switches}{uuid}}))
		{
			$anvil->data->{new_server}{uuid} = $anvil->data->{switches}{uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				'new_server::uuid' => $anvil->data->{new_server}{uuid}, 
			}});
		}
		if (not $anvil->data->{switches}{'install-media'})
		{
			my $print = $anvil->data->{switches}{options} ? 0 : 1;
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0451"});
			foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}})
			{
				my $file_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid};
				my $file_type = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type};
				next if $file_type ne "iso";
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0452", variables => {
					name => $file_name,
					uuid => $file_uuid,
				}});
			}
			$problem = 1;
		}
		else
		{
			# Make sure the file is valid
			my $found = 0;
			foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}})
			{
				my $file_uuid      = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid};
				my $file_type      = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type};
				my $file_directory = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory};
				if (($file_name eq $anvil->data->{switches}{'install-media'}) or
				    ($file_uuid eq $anvil->data->{switches}{'install-media'}))
				{
					# Found it.
					$found = 1;
					if ($file_type eq "iso")
					{
						# We're good
						$anvil->data->{new_server}{install_media} = $file_uuid;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							'new_server::install_media' => $anvil->data->{new_server}{install_media}, 
						}});
					}
					else
					{
						# Not an ISO.
						my $print = $anvil->data->{switches}{options} ? 0 : 1;
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0453", variables => {
							file => $anvil->data->{switches}{'install-media'},
						}});
						$problem = 1;
					}
				}
				last if $found;
			}
			if (not $found)
			{
				if (not $anvil->data->{switches}{options})
				{
					my $print = $anvil->data->{switches}{options} ? 0 : 1;
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0454", variables => {
						file => $anvil->data->{switches}{'install-media'},
					}});
					foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}})
					{
						my $file_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid};
						my $file_type = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type};
						next if $file_type ne "iso";
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0452", variables => {
							name => $file_name,
							uuid => $file_uuid,
						}});
					}
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0455"});
				}
				$problem = 1;
			}
		}
		if ($anvil->data->{switches}{'driver-disc'})
		{
			# Make sure it exists
			my $found = 0;
			foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}})
			{
				my $file_uuid      = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid};
				my $file_type      = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type};
				my $file_directory = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory};
				if (($file_name eq $anvil->data->{switches}{'driver-disc'}) or
				    ($file_uuid eq $anvil->data->{switches}{'driver-disc'}))
				{
					# Found it.
					$found = 1;
					if ($file_type eq "iso")
					{
						# We're good
						$anvil->data->{new_server}{driver_disc} = $file_uuid;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							'new_server::driver_disc' => $anvil->data->{new_server}{driver_disc}, 
						}});
					}
					else
					{
						# Not an ISO.
						my $print = $anvil->data->{switches}{options} ? 0 : 1;
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0456", variables => {
							file => $anvil->data->{switches}{'driver-disc'},
						}});
						$problem = 1;
					}
				}
				last if $found;
			}
			if (not $found)
			{
				if (not $anvil->data->{switches}{options})
				{
					my $print = $anvil->data->{switches}{options} ? 0 : 1;
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0457", variables => {
						file => $anvil->data->{switches}{'driver-disc'},
					}});
					foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}})
					{
						my $file_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid};
						my $file_type = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type};
						next if $file_type ne "iso";
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0452", variables => {
							name => $file_name,
							uuid => $file_uuid,
						}});
					}
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0455"});
				}
				$problem = 1;
			}
		}
		if ($anvil->data->{switches}{options})
		{
			if ($anvil->data->{switches}{machine})
			{
				print "# Sizes are in bytes\n";
				print "ram:min=65536,max=".$max_ram."\n";
				print "cpu:min=1,max=".$anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}."\n";
				foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}})
				{
					my $storage_group_uuid = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid};
					print "storage_group:name=".$storage_group_name.",uuid=".$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid}.",free_space=".$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}."\n";
				}
				foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}})
				{
					my $file_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid};
					my $file_type = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type};
					my $file_size = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:file_name' => $file_name, 
						's2:file_uuid' => $file_uuid, 
						's3:file_type' => $file_type, 
						's4:file_size' => $file_size, 
					}});
					next if $file_type ne "iso";
					my $say_size = $anvil->Convert->bytes_to_human_readable({"bytes" => $file_size});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_size => $say_size }});
					print "iso_file:name=".$file_name.",uuid=".$file_uuid.",size=".$say_size."\n";
				}
			}
			else
			{
				# Show valid options to build a VM in a machine-parsable way.
				print $anvil->Words->string({key => "job_0458", variables => {
					threads => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads},
					max_ram => $say_max_ram,
				}})."\n";
				foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}})
				{
					# Make sure the group UUID really exists Autovivication could create
					# false groups.
					my $storage_group_uuid = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { storage_group_uuid => $storage_group_uuid }});
					next if not $storage_group_uuid;

					my $max_storage_group_size     = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
					my $say_max_storage_group_size = $anvil->Convert->bytes_to_human_readable({"bytes" => $max_storage_group_size});
					print $anvil->Words->string({key => "job_0459", variables => {
						name       => $storage_group_name,
						uuid       => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid},
						free_space => $say_max_storage_group_size,
					}})."\n";
				}
				
				print $anvil->Words->string({key => "job_0460"})."\n";
				foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}})
				{
					my $file_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid};
					my $file_type = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type};
					my $file_size = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size};
					next if $file_type ne "iso";
					my $say_size = $anvil->Convert->bytes_to_human_readable({"bytes" => $file_size});
					print $anvil->Words->string({key => "job_0461", variables => {
						name => $file_name,
						uuid => $file_uuid,
						size => $say_size,
					}})."\n";
				}
				print $anvil->Words->string({key => "job_0462"})."\n";
			}
			$anvil->nice_exit({exit_code => 0});
		}
		
		### TODO: Better sanity check this
		# If we were passed network info, validate it.
		if ($anvil->data->{switches}{network})
		{
			$problem = process_network($anvil, $anvil->data->{switches}{network});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		}
		
		if ($problem)
		{
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Save the values.
		$anvil->data->{new_server}{name}          = $anvil->data->{switches}{name};
		$anvil->data->{new_server}{os}            = $anvil->data->{switches}{os};
		$anvil->data->{new_server}{cpu}           = $anvil->data->{switches}{cpu};
		$anvil->data->{new_server}{ram}           = $anvil->data->{switches}{ram};
		$anvil->data->{new_server}{storage_group} = $storage_group_uuid;
		$anvil->data->{new_server}{storage_size}  = $anvil->data->{switches}{'storage-size'};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:new_server::name'          => $anvil->data->{new_server}{name},
			's2:new_server::uuid'          => $anvil->data->{new_server}{uuid},
			's3:new_server::os'            => $anvil->data->{new_server}{os},
			's4:new_server::cpu'           => $anvil->data->{new_server}{cpu},
			's5:new_server::ram'           => $anvil->data->{new_server}{ram},
			's6:new_server::storage_group' => $anvil->data->{new_server}{storage_group},
			's7:new_server::storage_size'  => $anvil->data->{new_server}{storage_size}." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $max_storage_group_size}).")",
			's8:new_server::install_media' => $anvil->data->{new_server}{install_media},
			's9:new_server::driver_disc'   => $anvil->data->{new_server}{driver_disc},
		}});
	}
	
	my $anvil_uuid      = $anvil->data->{new_server}{anvil_uuid};
	my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:anvil_uuid'      => $anvil_uuid,
		's2:node1_host_uuid' => $node1_host_uuid,
	}});
	
	my $storage_group_uuid      = $anvil->data->{new_server}{storage_group};
	my $say_ram                 = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}});
	my $say_storage_group       = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
	my $say_storage_size        = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{storage_size}});
	my $install_media_file_uuid = $anvil->data->{new_server}{install_media};
	my $say_install_media       = $anvil->data->{files}{file_uuid}{$install_media_file_uuid}{file_name};
	my $driver_disc_file_uuid   = $anvil->data->{new_server}{driver_disc};
	my $say_driver_disc         = $driver_disc_file_uuid eq "none" ? "#!string!unit_0005!#" : $anvil->data->{files}{file_uuid}{$driver_disc_file_uuid}{file_name};
	
	if (not $anvil->data->{switches}{'ci-test'})
	{
		print $terminal->Tputs('cl');
		print $anvil->Words->string({key => "job_0150"})."\n";
		print $anvil->Words->string({key => "job_0151", variables => { anvil_name => $anvil->data->{new_server}{anvil_name} }})."\n";
		print $anvil->Words->string({key => "job_0157", variables => { server_name => $anvil->data->{new_server}{name} }})."\n";
		print $anvil->Words->string({key => "job_0158", variables => { cpu_cores => $anvil->data->{new_server}{cpu} }})."\n";
		print $anvil->Words->string({key => "job_0165", variables => { ram => $say_ram }})."\n";
		print $anvil->Words->string({key => "job_0170", variables => { storage_group => $say_storage_group }})."\n";
		print $anvil->Words->string({key => "job_0173", variables => { storage_size => $say_storage_size }})."\n";
		print $anvil->Words->string({key => "job_0176", variables => { install_media => $say_install_media }})."\n";
		print $anvil->Words->string({key => "job_0180", variables => { driver_disc => $say_driver_disc }})."\n";
		print $anvil->Words->string({key => "job_0198", variables => { os => $anvil->data->{new_server}{say_os} }})."\n\n";
		print $anvil->Words->string({key => "message_0206"})." ";
		my $answer = <STDIN>;
		chomp $answer;
		
		$answer = "y" if $answer eq "";
		
		if ($answer !~ /^y/i)
		{
			# Abort
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	
	my $job_data = "server_name=".$anvil->data->{new_server}{name}."
os=".$anvil->data->{new_server}{os}." 
cpu_cores=".$anvil->data->{new_server}{cpu}."
ram=".$anvil->data->{new_server}{ram}."
storage_group_uuid=".$anvil->data->{new_server}{storage_group}."
storage_size=".$anvil->data->{new_server}{storage_size}."
install_iso=".$anvil->data->{new_server}{install_media}."
driver_iso=".$anvil->data->{new_server}{driver_disc};
	if ($anvil->data->{new_server}{uuid})
	{
		$job_data .= "
server_uuid=".$anvil->data->{new_server}{uuid};
	}
	print "\n".$anvil->Words->string({key => "job_0183", variables => { job_data => $job_data }})."\n";
	
	# Register the job with the primary node on the Anvil! (or node 1, if neither node is primary).
	my $target_host_uuid = $anvil->Cluster->get_primary_host_uuid({debug => 2, anvil_uuid => $anvil_uuid});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_uuid => $target_host_uuid }});
	if (not $target_host_uuid)
	{
		$target_host_uuid = $node1_host_uuid;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_uuid => $target_host_uuid }});
	}
	my $host_name = $anvil->Get->host_name_from_uuid({host_uuid => $target_host_uuid});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }});
	my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
		debug           => 2, 
		job_command     => $anvil->data->{path}{exe}{'anvil-provision-server'}.$anvil->Log->switches, 
		job_data        => $job_data, 
		job_name        => "server::provision", 
		job_title       => "job_0147", 
		job_description => "job_0148", 
		job_progress    => 0,
		job_host_uuid   => $target_host_uuid, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
	
	print $anvil->Words->string({key => "job_0184", variables => { job_uuid => $job_uuid }})."\n";
	
	return(0);
}