anvil/tools/anvil-join-anvil

#!/usr/bin/perl
# 
# This adds a node (or gets a node to join to a new/rebuilt peer or added DR) to an Anvil!.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Failed to connect to any database.
# 2 = Failed to load/parse the manifest.
# 3 = Failed to change the host name.
# 4 = Failed to reconnect to any database after the network was reconfigured
# 5 = Problem parsing job data or loading manifest or anvil data using job data.
# 6 = A pcs call failed.
# 7 = No job was found to run.
# 
# TODO: 
# - Check to see if this is a cluster node and/or running VMs, and if so, refuse to run.
# 

use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
use String::ShellQuote;
use Text::Diff;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

# Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is 
# passed directly, it will be used. Otherwise, the password will be read from the database.
$anvil->Get->switches;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0129"});
	sleep 2;
	$anvil->nice_exit({exit_code => 1});
}

# Get the job details
load_job($anvil);

# Update the user passwords
update_passwords($anvil);

# Check if we need to change any IPs or our hostname.
check_local_network($anvil);

# (wait for out peer and) Configure pacemaker
configure_pacemaker($anvil);

# Configure DRBD
configure_drbd($anvil);

# Enable anvil-safe-start
if (1)
{
	my ($return_code) = $anvil->System->enable_daemon({daemon => "anvil-safe-start.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	update_progress($anvil, 99, "job_0094,!!daemon!anvil-safe-start.service!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "anvil-safe-start.service" }});
}

update_progress($anvil, 100, "job_0128");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0128"});

# Record that we're done configuring pacemaker.
$anvil->Database->insert_or_update_variables({
	variable_name         => "system::all::configured", 
	variable_value        => 1, 
	variable_default      => "", 
	variable_description  => "striker_0297", 
	variable_section      => "system", 
	variable_source_uuid  => $anvil->data->{sys}{host_uuid}, 
	variable_source_table => "hosts", 
});

$anvil->nice_exit({exit_code => 0});

#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# Configure DRBD
sub configure_drbd
{
	my ($anvil) = @_;
	
	### TODO: See if there is a hardware RAID controller and, if so, auto-enable 
	update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0126");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0126"});
	my $updated = $anvil->DRBD->update_global_common({
		usage_count => $anvil->data->{sys}{privacy}{strong} ? 0 : 1,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { updated => $updated }});
	if ($updated)
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0127");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0127"});
	}
	else
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0128");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0128"});
	}
	
	# Record that we're done configuring pacemaker.
	$anvil->Database->insert_or_update_variables({
		variable_name         => "system::drbd::configured", 
		variable_value        => 1, 
		variable_default      => "", 
		variable_description  => "striker_0296", 
		variable_section      => "system", 
		variable_source_uuid  => $anvil->data->{sys}{host_uuid}, 
		variable_source_table => "hosts", 
	});
	
	return(0);
}

# Update the passwords for user accounts.
sub update_passwords
{
	my ($anvil) = @_;
	
	my $machine       = $anvil->data->{sys}{machine};
	my $manifest_uuid = $anvil->data->{sys}{manifest_uuid};
	my $anvil_name    = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{name};
	my $anvil_uuid    = $anvil->data->{sys}{anvil_uuid};
	my $new_password  = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $users         = $machine =~ /dr\d+/ ? ["root", "admin"] : ["root", "admin", "hacluster"];
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		machine          => $machine, 
		manifest_uuid    => $manifest_uuid,
		anvil_name       => $anvil_name, 
		anvil_uuid       => $anvil_uuid, 
		new_password     => $anvil->Log->is_secure($new_password), 
	}});
	
	foreach my $user (@{$users})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { user => $user }});
		
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0093,!!user!".$user."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0093", variables => { user => $user }});
		my ($return_code) = $anvil->System->change_shell_user_password({
			debug        => 3,
			user         => $user,
			new_password => $new_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
	}
	
	return(0);
}


# (wait for our peer and) Configure pacemaker. If this is a DR host, this is skipped.
sub configure_pacemaker
{
	my ($anvil) = @_;
	
	my $machine       = $anvil->data->{sys}{machine};
	my $manifest_uuid = $anvil->data->{sys}{manifest_uuid};
	
	### TODO: Move these to variables in the 'sys' hash
	my $anvil_name       =  $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{name};
	my $anvil_uuid       =  $anvil->data->{sys}{anvil_uuid};
	my $host_name        =  $anvil->Get->host_name;
	my $new_password     =  $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid  =  $anvil->data->{sys}{node1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name  =  $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	   $node1_host_name  =~ s/\..*$//;
	my $node2_host_uuid  =  $anvil->data->{sys}{node2_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name  =  $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	   $node2_host_name  =~ s/\..*$//;
	my $peer_host_name   =  $anvil->Get->host_uuid() eq $node1_host_uuid ? $node2_host_name : $node1_host_name;
	my $peer_host_uuid   =  $anvil->Get->host_uuid() eq $node1_host_uuid ? $node2_host_uuid : $node1_host_uuid;
	my $escaped_password =  shell_quote($new_password);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		machine          => $machine, 
		anvil_uuid       => $anvil_uuid, 
		anvil_name       => $anvil_name, 
		host_name        => $host_name, 
		manifest_uuid    => $manifest_uuid,
		node1_host_uuid  => $node1_host_uuid, 
		node1_host_name  => $node1_host_name, 
		node2_host_uuid  => $node2_host_uuid, 
		node2_host_name  => $node2_host_name, 
		peer_host_name   => $peer_host_name, 
		peer_host_uuid   => $peer_host_uuid, 
		new_password     => $anvil->Log->is_secure($new_password), 
		escaped_password => $anvil->Log->is_secure($escaped_password),
	}});
	
	# If this is a DR box, we don't use pacemaker.
	if ($machine =~ /dr\d+/)
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 10), "job_0096");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0096"});
		return(0);
	}
	
	### Run on both nodes.
	# Enable pcsd and start the pcsd daemon.
	my ($return_code) = $anvil->System->enable_daemon({daemon => "pcsd.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
	$return_code = undef;
	($return_code) = $anvil->System->start_daemon({daemon => "pcsd.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
	$return_code = undef;
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0094,!!daemon!pcsd.service!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "pcsd.service" }});
	
	# Enable libvirtd and start the libvirtd daemon.
	($return_code) = $anvil->System->enable_daemon({daemon => "libvirtd.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
	$return_code = undef;
	($return_code) = $anvil->System->start_daemon({daemon => "libvirtd.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
	$return_code = undef;
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0094,!!daemon!libvirtd.service!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "libvirtd.service" }});
	
	# Disabled and stop the drbd daemon.
	($return_code) = $anvil->System->disable_daemon({daemon => "drbd.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
	$return_code = undef;
	($return_code) = $anvil->System->stop_daemon({daemon => "drbd.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
	$return_code = undef;
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!drbd.service!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "drbd.service" }});
	
	# Disabled and stop the ksm and ksmtuned daemon.
	($return_code) = $anvil->System->disable_daemon({daemon => "ksm.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
	$return_code = undef;
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "ksm.service" }});
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!ksm.service!!");
	
	($return_code) = $anvil->System->stop_daemon({daemon => "ksmtuned.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
	$return_code = undef;
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "ksmtuned.service" }});
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!ksmtuned.service!!");
	
	# If there is no corosync.conf, see if the peer has it. If so, copy it. If not, we'll initialize the
	# cluster shortly.
	if (not -e $anvil->data->{path}{configs}{'corosync.conf'})
	{
		my $corosync_conf = $anvil->Storage->read_file({
			file   => $anvil->data->{path}{configs}{'corosync.conf'},
			target => $peer_host_name,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { corosync_conf => $corosync_conf }});
		if ($corosync_conf ne "!!error!!")
		{
			# Write the file out.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0100"});
			$anvil->Storage->write_file({
				debug => 2,
				body  => $corosync_conf,
				file  => $anvil->data->{path}{configs}{'corosync.conf'}, 
				user  => "root",
				group => "root",
				mode  => "0644",
			});
		}
	}
	
	# Node 1 initializes, node 2 waits.
	if ($machine eq "node2")
	{
		my $start_time = 0;
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0103");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0103"});
		
		# We loop until the peer finishes or the peer's job hit's 100.
		my $tried_starting = 0;
		my $both_online    = 0;
		until($both_online)
		{
			if (-e $anvil->data->{path}{configs}{'corosync.conf'})
			{
				if (not $start_time)
				{
					# Corosync is configured, we'll wait up to two minutes and then try
					# joining the cluster ourselves.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0108"});
					$start_time = time + 120;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { start_time => $start_time }});
				}
				elsif ((time > $start_time) && (not $tried_starting))
				{
					# We've waited a minute, time to try starting the cluster.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0106"});
					my $cluster_started = $anvil->Cluster->start_cluster({debug => 2, all => 1});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { cluster_started => $cluster_started }});
					
					# Mark that weve tried to start.
					$tried_starting = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { tried_starting => $tried_starting }});
				}
				
				my $problem = $anvil->Cluster->parse_cib({debug => 3});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
				if (not $problem)
				{
					# See if both nodes are online.
					my $node1_ready = $anvil->Cluster->check_node_status({node_name => $node1_host_name});
					my $node2_ready = $anvil->Cluster->check_node_status({node_name => $node2_host_name});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
						node1_ready => $node1_ready,
						node2_ready => $node2_ready, 
					}});
					if (($node1_ready) && ($node2_ready))
					{
						$both_online = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { both_online => $both_online }});
						
						update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0104");
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0104"});
					}
					else
					{
						# Not online yet, wait a bit.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0105", variables => {
							node1_name   => $node1_host_name,
							node1_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm},
							node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd},
							node1_crmd   => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{'join'},
							node1_join   => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{ready},
							node2_name   => $node2_host_name,
							node2_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{in_ccm},
							node2_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{crmd},
							node2_crmd   => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{'join'},
							node2_join   => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{ready},
						}});
					}
				}
			}
			else
			{
				# corosync.conf doesn't exist yet.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0107"});
			}
			sleep 5 if not $both_online;
		}
	}
	else
	{
		# We're node 1, proceed with cluster setup.
		my $waiting         = 1;
		my $warning_printed = 0;
		while($waiting)
		{
			# Try to authenticate against the peer.
			my $auth_shell_call = $anvil->data->{path}{exe}{pcs}." host auth ".$node1_host_name." ".$node2_host_name." -u hacluster -p ".$escaped_password;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { auth_shell_call => $auth_shell_call }});
			
			my ($output, $return_code) = $anvil->System->call({debug => 3, secure => 1, shell_call => $auth_shell_call});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output,
				return_code => $return_code, 
			}});
			if ($return_code)
			{
				# Something went wrong. 
				if (not $warning_printed)
				{
					# Update the job
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0097");
					$warning_printed = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { warning_printed => $warning_printed }});
				}
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0097"});
				sleep 5;
			}
			else
			{
				# We're good.
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0098");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0098"});
				$waiting = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
			}
		}
		
		# If there is no corosync.conf, see if the peer has it. If so, copy it. If not, initialize 
		# the cluster.
		if (not -e $anvil->data->{path}{configs}{'corosync.conf'})
		{
			# There's no cluster yet, initialize it.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0099,!!anvil_name!".$anvil_name."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0099", variables => { anvil_name => $anvil_name }});
			
			my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster setup ".$anvil_name." ".$node1_host_name." ".$node2_host_name;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output,
				return_code => $return_code, 
			}});
			if ($return_code)
			{
				# Something went wrong
				update_progress($anvil, 100, "job_0101,!!error!".$output."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0101", variables => { error => $output }});
				$anvil->nice_exit({exit_code => 5});
			}
		}
		
		# If we can parse the CIB, then pcsd is running. 
		my $problem = $anvil->Cluster->parse_cib({debug => 3});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		if ($problem)
		{
			# Start the cluster.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0102");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0102"});
			
			my $cluster_started = $anvil->Cluster->start_cluster({debug => 3, all => 1});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cluster_started => $cluster_started }});
		}
		
		# Now wait for both nodes to come online.
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0109");
		my $both_online = 0;
		my $start_again = time + 60;
		until ($both_online)
		{
			### TODO: If we're waiting more that five minutes, call 'pcs cluster start --all' again.
			my $problem = $anvil->Cluster->parse_cib({debug => 3});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
			if (not $problem)
			{
				# See if both nodes are online.
				my $node1_ready = $anvil->Cluster->check_node_status({node_name => $node1_host_name});
				my $node2_ready = $anvil->Cluster->check_node_status({node_name => $node2_host_name});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					node1_ready => $node1_ready,
					node2_ready => $node2_ready, 
				}});
				if (($node1_ready) && ($node2_ready))
				{
					$both_online = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { both_online => $both_online }});
					
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0104");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0104"});
				}
				else
				{
					# Not online yet, wait a bit.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0105", variables => {
						node1_name   => $node1_host_name,
						node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm},
						node1_crmd   => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd},
						node1_join   => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{'join'},
						node1_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{ready},
						node2_name   => $node2_host_name,
						node2_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{in_ccm},
						node2_crmd   => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{crmd},
						node2_join   => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{'join'},
						node2_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{ready},
					}});
				}
			}
			if (time > $start_again)
			{
				# Call cluster start again.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0272"});
				   $start_again = time + 60;
				my $shell_call  = $anvil->data->{path}{exe}{pcs}." cluster start --all";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					start_again => $start_again, 
					shell_call  => $shell_call,
				}});
				
				my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output,
					return_code => $return_code, 
				}});
			}
			sleep 5 if not $both_online;
		}
		
		### Setup fencing! 
		$anvil->Striker->get_fence_data();
		
		# IPMI first, if applicable.
		my $something_changed = {};
		my $fence_order       = {};
		my $node1_use_delay   = 0;
		my $node2_use_delay   = 0;
		my $fence_devices     = {};
		foreach my $node ("node1", "node2")
		{
			my $node_name         = $node eq "node1" ? $node1_host_name : $node2_host_name;
			my $host_uuid         = $node eq "node1" ? $node1_host_uuid : $node2_host_uuid;
			my $host_ipmi         = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi};
			my $ipmi_stonith_name = "ipmilan_".$node; 
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				node_name         => $node_name, 
				host_uuid         => $host_uuid, 
				host_ipmi         => $host_ipmi =~ /passw/ ? $anvil->Log->is_secure($host_ipmi) : $host_ipmi,
				ipmi_stonith_name => $ipmi_stonith_name, 
			}});
			
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0115,!!node!".$node_name."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0115", variables => { node => $node_name }});
			
			# This will store the fence level order. If something changes
			$fence_order->{$node_name} = [];
			
			# This will switch to '1' if something changed, triggering a reconfig of the fencing levels.
			$something_changed->{$node_name} = 0;
			
			# Does this stonith method already exist?
			my $create_entry    = 0;
			my $delete_old      = 0;
			my $pcs_add_command = "";
			if ($host_ipmi)
			{
				push @{$fence_order->{$node_name}}, "fence_ipmilan";
				$fence_devices->{$node_name}{fence_ipmilan} = [$ipmi_stonith_name];
				
				# The --action switch needs to be 'pcmk_off_action' in pcs, so we convert it here.
				$host_ipmi =~ s/--action status//;
				$host_ipmi =~ s/--action/--pcmk_off_action/;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
					host_ipmi => $host_ipmi =~ /passw/ ? $anvil->Log->is_secure($host_ipmi) : $host_ipmi,
				}});
				
				# We have IPMI, so we also want fence_delay for this node.
				if ($node eq "node1")
				{
					$node1_use_delay = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { node1_use_delay => $node1_use_delay }});
				}
				else
				{
					$node2_use_delay = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { node2_use_delay => $node2_use_delay }});
				}
				
				# If we're here, break up the command and turn it into the pcs call.
				my $old_switches              = {};
				my ($fence_agent, $arguments) = ($host_ipmi =~ /^\/.*\/(.*?)\s+(.*)$/);
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					fence_agent  => $fence_agent,
					arguments    => $arguments =~ /passw/ ? $anvil->Log->is_secure($arguments) : $arguments,
				}});
				
				   $pcs_add_command = $anvil->data->{path}{exe}{pcs}." stonith create ".$ipmi_stonith_name." ".$fence_agent." pcmk_host_list=\"".$node_name."\" ";
				my $switches = $anvil->System->parse_arguments({debug => 2, arguments => $arguments});
				foreach my $switch (sort {$a cmp $b} keys %{$switches})
				{
					# Ignore 'delay', we handle that in Cluster->set_delay(); Also, 
					# convert '#!SET!#' to 'true'.
					my $value =  $switches->{$switch};
					   $value =~ s/"/\\"/g;
					   $value =~ s/#!SET!#/true/g;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						switch => $switch,
						value  => $value,
					}});
					next if $anvil->data->{fence_data}{$fence_agent}{switch}{$switch}{name} eq "delay";
					next if $anvil->data->{fence_data}{$fence_agent}{switch}{$switch}{name} eq "action";
					
					# Find the argument=value version.
					my $argument        =  $anvil->data->{fence_data}{$fence_agent}{switch}{$switch}{name};
					   $pcs_add_command .= $argument."=\"".$value."\" ";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						argument        => $argument,
						value           => $argument        =~ /passw/ ? $anvil->Log->is_secure($value)           : $value,
						pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command,
					}});
					
					# Store this to see if it's different from what's already in the CIB.
					$old_switches->{$argument} = $value;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
						"old_switches->{$argument}" => $old_switches->{$argument},
					}});
				}
				$pcs_add_command .= "op monitor interval=\"60\"";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
					pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command,
				}});
				
				# If there's an entry in the CIB, so if it's different somehow
				if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name})
				{
					foreach my $argument (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name}{argument}})
					{
						next if $argument eq "delay";
						next if $argument eq "action";
						my $old_entry = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name}{argument}{$argument}{value};
						my $new_entry = exists $old_switches->{$argument} ? $old_switches->{$argument} : "";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
							's1:argument'  => $argument, 
							's2:old_entry' => $old_entry,
							's3:new_entry' => $new_entry,
						}});
						
						if ($old_entry ne $new_entry)
						{
							# Changed, delete and recreate.
							$delete_old   = 1;
							$create_entry = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
								delete_old   => $delete_old,
								create_entry => $create_entry,
							}});
							
							update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0117");
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0117"});
							last;
						}
						
						# Delete the old switch.
						delete $old_switches->{$argument};
					}
					
					# Are there any old switches left?
					my $old_switch_count = keys %{$old_switches};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
						delete_old       => $delete_old, 
						old_switch_count => $old_switch_count,
					}});
					if ((not $delete_old) && ($old_switch_count))
					{
						# Delete and recreate. 
						$delete_old   = 1;
						$create_entry = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
							delete_old   => $delete_old,
							create_entry => $create_entry,
						}});
						
						update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0117");
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0117"});
					}
				}
				else
				{
					# No existing entry, add a new one.
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0116");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0116"});
			
					$create_entry = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { create_entry => $create_entry }});
				}
			}
			elsif (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name})
			{
				# There was an existing fence config, but there's no entry in 'host_ipmi'. 
				# Remove the stonith entry.
				$delete_old = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { delete_old => $delete_old }});
				
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0118");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0118"});
			}
			
			# Process the IPMI entry.
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				delete_old   => $delete_old,
				create_entry => $create_entry, 
			}});
			if ($delete_old)
			{
				# Delete
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0119,!!device!".$ipmi_stonith_name."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0119", variables => { device => $ipmi_stonith_name }});
				
				my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith delete ".$ipmi_stonith_name;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
				
				my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
					output      => $output,
					return_code => $return_code, 
				}});
				if ($return_code)
				{
					# Something went wrong.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
						shell_call  => $shell_call, 
						output      => $output, 
						return_code => $return_code, 
					}});
					update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
					sleep 2;
					$anvil->nice_exit({exit_code => 6});
				}
				
				$something_changed->{$node_name} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
			}
			if ($create_entry)
			{
				# Create.
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0120,!!device!".$ipmi_stonith_name."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0120", variables => { device => $ipmi_stonith_name }});
				
				my $shell_call = $pcs_add_command;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
				
				my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
					output      => $output,
					return_code => $return_code, 
				}});
				if ($return_code)
				{
					# Something went wrong.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
						shell_call  => $shell_call, 
						output      => $output, 
						return_code => $return_code, 
					}});
					update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
					sleep 2;
					$anvil->nice_exit({exit_code => 6});
				}
				
				$something_changed->{$node_name} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
			}
			
			
			### Now any other fence devices.
			foreach my $device (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$node}{fence}})
			{
				my $delete_old      = 0;
				my $create_entry    = 0;
				my $old_switches    = {};
				my $fence_uuid      = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{fences}{$device}{uuid};
				my $fence_name      = $anvil->data->{fences}{fence_uuid}{$fence_uuid}{fence_name};
				my $fence_arguments = $anvil->data->{fences}{fence_uuid}{$fence_uuid}{fence_arguments};
				my $fence_agent     = $anvil->data->{fences}{fence_uuid}{$fence_uuid}{fence_agent};
				my $stonith_name    = ($fence_agent =~ /^fence_(.*)$/)[0]."_".$node."_".$fence_name; 
				my $port            = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$node}{fence}{$device}{port};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					device          => $device, 
					fence_uuid      => $fence_uuid, 
					fence_name      => $fence_name, 
					fence_arguments => $fence_arguments =~ /passw/ ? $anvil->Log->is_secure($fence_arguments) : $fence_arguments,
					stonith_name    => $stonith_name, 
					port            => $port, 
				}});
				
				# We use this to tell if there are two or more entries per agent. If there
				# are, we link them later when setting up the fence levels.
				if (not exists $fence_devices->{$node_name}{$fence_agent})
				{
					push @{$fence_order->{$node_name}}, $fence_agent;
					$fence_devices->{$node_name}{$fence_agent} = [];
				}
				push @{$fence_devices->{$node_name}{$fence_agent}}, $stonith_name;
				
				# Fence arguments use 'action', but pcs deprecated it in favour of 'pcmk_off_action', so rename it.
				$fence_arguments =~ s/action=/pcmk_off_action=/;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					fence_arguments => $fence_arguments =~ /passw/ ? $anvil->Log->is_secure($fence_arguments) : $fence_arguments, 
				}});
				
				# Build the pcs command
				my $pcs_add_command = $anvil->data->{path}{exe}{pcs}." stonith create ".$stonith_name." ".$fence_agent." pcmk_host_list=\"".$node_name."\" ".$fence_arguments." ";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, 
				}});
				while ($fence_arguments =~ /=/)
				{
					# Ignore 'delay', we handle that in Cluster->set_delay();
					my $pair               =  ($fence_arguments =~ /(\S*?=".*?")/)[0];
					   $fence_arguments    =~ s/$pair//;
					   $fence_arguments    =~ s/^\s+//;
					   $fence_arguments    =~ s/\s+$//;
					my ($argument, $value) =  ($pair =~ /(.*)="(.*)"/);
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:fence_arguments' => $fence_arguments, 
						's2:pair'            => $pair =~ /passw/ ? $anvil->Log->is_secure($pair) : $pair,
						's3:argument'        => $argument,
						's4:value'           => $argument =~ /passw/ ? $anvil->Log->is_secure($value) : $value,
					}});
					
					# Ignore 'delay', we handle that in Cluster->set_delay();
					if (($argument ne "pcmk_off_action")                                           && 
					    (exists $anvil->data->{fence_data}{$fence_agent}{switch}{$argument}{name}) && 
					    ($anvil->data->{fence_data}{$fence_agent}{switch}{$argument}{name} eq "delay"))
					{
						next;
					}
					
					# Store this to see if it's different from what's already in the CIB.
					$old_switches->{$argument} = $value;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"old_switches->{$argument}" => $old_switches->{$argument},
					}});
				}
				if ($port)
				{
					$port                 =~ s/"/\\"/g;
					$pcs_add_command      .= "port=\"".$port."\" ";
					$old_switches->{port} =  $port;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, 
						"old_switches->{port}" => $old_switches->{port},
					}});
				}
				$pcs_add_command .= "op monitor interval=\"60\"";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, 
				}});
				
				# Does this device exist already?
				if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name})
				{
					foreach my $argument (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name}{argument}})
					{
						next if $argument eq "delay";
						my $old_entry = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name}{argument}{$argument}{value};
						my $new_entry = exists $old_switches->{$argument} ? $old_switches->{$argument} : "";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							's1:argument'  => $argument, 
							's2:old_entry' => $old_entry,
							's3:new_entry' => $new_entry,
						}});
						
						if ($old_entry ne $new_entry)
						{
							# Changed, delete and recreate.
							$delete_old   = 1;
							$create_entry = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								delete_old   => $delete_old,
								create_entry => $create_entry,
							}});
							
							update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0121,!!device!".$stonith_name."!!");
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0121", variables => { device => $stonith_name }});
							last;
						}
						
						# Delete the old switch.
						delete $old_switches->{$argument};
					}
					
					# Are there any old switches left?
					my $old_switch_count = keys %{$old_switches};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_switch_count => $old_switch_count }});
					if ((not $delete_old) && ($old_switch_count))
					{
						# Delete and recreate. 
						$delete_old   = 1;
						$create_entry = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							delete_old   => $delete_old,
							create_entry => $create_entry,
						}});
						
						update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0121,!!device!".$stonith_name."!!");
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0121", variables => { device => $stonith_name }});
					}
				}
				else
				{
					# No existing entry, add a new one.
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0122,!!device!".$stonith_name."!!");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0122", variables => { device => $stonith_name }});
					
					$create_entry = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_entry => $create_entry }});
				}
				
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					delete_old   => $delete_old,
					create_entry => $create_entry, 
				}});
				if ($delete_old)
				{
					# Delete
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0119,!!device!".$stonith_name."!!");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0119", variables => { device => $stonith_name }});
					
					my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith delete ".$stonith_name;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					if ($return_code)
					{
						# Something went wrong.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
							shell_call  => $shell_call, 
							output      => $output, 
							return_code => $return_code, 
						}});
						update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
						sleep 2;
						$anvil->nice_exit({exit_code => 6});
					}
					
					$something_changed->{$node_name} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
				}
				if ($create_entry)
				{
					# Create.
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0120,!!device!".$stonith_name."!!");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0120", variables => { device => $stonith_name }});
					
					my $shell_call = $pcs_add_command;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					if ($return_code)
					{
						# Something went wrong.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
							shell_call  => $shell_call, 
							output      => $output, 
							return_code => $return_code, 
						}});
						update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
						sleep 2;
						$anvil->nice_exit({exit_code => 6});
					}
					
					$something_changed->{$node_name} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
				}
			}
			
			
			### If we had a fence_ipmilan entry, add a 'fence_delay' entry, if needed.
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				node            => $node, 
				node1_use_delay => $node1_use_delay,
				node2_use_delay => $node2_use_delay, 
			}});
			if ((($node eq "node1") && ($node1_use_delay)) or 
			    (($node eq "node2") && ($node2_use_delay)))
			{
				my $stonith_name = "delay_".$node; 
				push @{$fence_order->{$node_name}}, "fence_delay";
				$fence_devices->{$node_name}{fence_delay} = [$stonith_name];
				
				# Add the fence delay if it doesn't exist yet.
				if (not exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name})
				{
					my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith create ".$stonith_name." fence_delay pcmk_host_list=\"".$node_name."\" wait=\"60\" op monitor interval=\"60\"";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					if ($return_code)
					{
						# Something went wrong.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
							shell_call  => $shell_call, 
							output      => $output, 
							return_code => $return_code, 
						}});
						update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
						sleep 2;
						$anvil->nice_exit({exit_code => 6});
					}
				}
			}
		}
		
		# Setup fence levels.
		foreach my $node_name (sort {$a cmp $b} keys %{$fence_order})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
			if ($something_changed->{$node_name})
			{
				# Update our view of the cluster.
				my $problem = $anvil->Cluster->parse_cib({debug => 2});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
				
				# Delete any existing fence levels
				if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name})
				{
					foreach my $index (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{order}})
					{
						# pcs stonith level delete <index> <target>
						my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith level delete ".$index." ".$node_name;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
						
						my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							output      => $output,
							return_code => $return_code, 
						}});
						if ($return_code)
						{
							# Something went wrong.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
								shell_call  => $shell_call, 
								output      => $output, 
								return_code => $return_code, 
							}});
							update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
							sleep 2;
							$anvil->nice_exit({exit_code => 6});
						}
					}
				}
				
				# Create the new fence levels
				my $i = 1;
				foreach my $fence_agent (@{$fence_order->{$node_name}})
				{
					my $devices = "";
					foreach my $device (sort {$a cmp $b} @{$fence_devices->{$node_name}{$fence_agent}})
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { device => $device }});
						$devices .= $device.",";
					}
					$devices =~ s/,$//;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { devices => $devices }});
					
					my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith level add ".$i." ".$node_name." ".$devices;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					if ($return_code)
					{
						# Something went wrong.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
							shell_call  => $shell_call, 
							output      => $output, 
							return_code => $return_code, 
						}});
						update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
						sleep 2;
						$anvil->nice_exit({exit_code => 6});
					}
					
					$i++;
				}
			}
		}
	}

	# Make sure logind is update to handle fencing properly
	# see - https://access.redhat.com/solutions/1578823
	$anvil->Cluster->configure_logind({debug => 2});

	# Enable fencing and set the retry to INFINITY, if needed.
	$anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'} = "" if not defined $anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"cib::parsed::data::stonith::max-attempts" => $anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'}, 
	}});
	if ($anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'} ne "INFINITY")
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0124");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0124"});
		
		my $shell_call = $anvil->data->{path}{exe}{pcs}." property set stonith-max-attempts=INFINITY";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
		if ($return_code)
		{
			# Something went wrong.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
				shell_call  => $shell_call, 
				output      => $output, 
				return_code => $return_code, 
			}});
			update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
			sleep 2;
			$anvil->nice_exit({exit_code => 6});
		}
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"cib::parsed::data::stonith::enabled" => $anvil->data->{cib}{parsed}{data}{stonith}{enabled}, 
	}});
	if (not $anvil->data->{cib}{parsed}{data}{stonith}{enabled})
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0125");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0125"});
		
		my $shell_call = $anvil->data->{path}{exe}{pcs}." property set stonith-enabled=true";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
		if ($return_code)
		{
			# Something went wrong.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
				shell_call  => $shell_call, 
				output      => $output, 
				return_code => $return_code, 
			}});
			update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
			sleep 2;
			$anvil->nice_exit({exit_code => 6});
		}
	}
	
	# Update (if needed) corosync.conf to use the BCN1 and SN1 as knet networks.
	if ($machine eq "node1")
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0344");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0344"});
		
		check_corosync($anvil);
	}
	
	# Record that we're done configuring pacemaker.
	$anvil->Database->insert_or_update_variables({
		variable_name         => "system::pacemaker::configured", 
		variable_value        => 1, 
		variable_default      => "", 
		variable_description  => "striker_0295", 
		variable_section      => "system", 
		variable_source_uuid  => $anvil->data->{sys}{host_uuid}, 
		variable_source_table => "hosts", 
	});
	
	return(0);
}
	
sub check_corosync
{
	my ($anvil) = @_;
	
	update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0345");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0345"});
	my $waiting      = 1;
	my $anvil_uuid   = $anvil->data->{sys}{anvil_uuid};
	my $new_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	while ($waiting)
	{
		my $problem          =  $anvil->Cluster->parse_cib({debug => 3});
		my $peer_ready       =  $anvil->data->{cib}{parsed}{peer}{ready};
		my $peer_name        =  $anvil->data->{cib}{parsed}{peer}{name};
		my $peer_short_name  =  $peer_name;
		   $peer_short_name  =~ s/\..*$//;
		my $peer_bcn_name    =  $peer_short_name.".bcn1";
		my $peer_sn_name     =  $peer_short_name.".sn1";
		my $local_ready      =  $anvil->data->{cib}{parsed}{'local'}{ready};
		my $local_name       =  $anvil->data->{cib}{parsed}{'local'}{name};
		my $local_short_name =  $anvil->Get->short_host_name;
		my $local_bcn_name   =  $local_short_name.".bcn1";
		my $local_sn_name    =  $local_short_name.".sn1";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			problem          => $problem, 
			peer_ready       => $peer_ready,
			peer_name        => $peer_name, 
			peer_short_name  => $peer_short_name, 
			peer_bcn_name    => $peer_bcn_name, 
			peer_sn_name     => $peer_sn_name, 
			local_ready      => $local_ready,
			local_name       => $local_name, 
			local_short_name => $local_short_name, 
			local_bcn_name   => $local_bcn_name, 
			local_sn_name    => $local_sn_name, 
		}});
		if ((not $problem) && ($peer_ready) && ($local_ready))
		{
			update_progress($anvil, $anvil->data->{job}{progress}, "job_0346");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0346"});
			
			# Verify we can ping the peer on the BCN and SN.
			my $bcn_access = $anvil->Remote->test_access({
				target   => $peer_bcn_name,
				password => $new_password,
			});
			my $sn_access = $anvil->Remote->test_access({
				target   => $peer_sn_name,
				password => $new_password,
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				bcn_access => $bcn_access, 
				sn_access  => $sn_access, 
			}});
			if (($bcn_access) && ($sn_access))
			{
				# We're ready!
				update_progress($anvil, ($anvil->data->{job}{progress}+1), "job_0347");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0347"});
				
				$waiting = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				
				my $in_totem          = 0;
				my $token_seen        = 0;
				my $in_nodelist       = 0;
				my $in_node           = 0;
				my $ring0_addr        = "";
				my $ring1_addr        = "";
				my $in_node_name      = "";
				my $nodelist_body     = "";
				my $old_corosync_conf = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'corosync.conf'}});
				my $new_corosync_conf = "";
				foreach my $line (split/\n/, $old_corosync_conf)
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
					if ($line =~ /totem \{/)
					{
						$in_totem = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_totem => $in_totem }});
						
						$new_corosync_conf .= $line."\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
						next;
					}
					if ($line =~ /nodelist \{/)
					{
						$in_nodelist = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
						
						$new_corosync_conf .= $line."\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
						next;
					}
					
					if ($in_nodelist)
					{
						if ($line =~ /node \{/)
						{
							$in_node = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_node => $in_node }});
							
							$new_corosync_conf .= $line."\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
							next;
						}
						if ($in_node)
						{
							if ($line =~ /name:(.*)$/)
							{
								$in_node_name = $anvil->Words->clean_spaces({string => $1});
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_node_name => $in_node_name }});
								
								$nodelist_body .= $line."\n"; 
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }});
								next;
							}
							elsif ($line =~ /ring0_addr:(.*)$/)
							{
								$ring0_addr = $anvil->Words->clean_spaces({string => $1});
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ring0_addr => $ring0_addr }});
								
								$nodelist_body .= $line."\n"; 
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }});
								next;
							}
							elsif ($line =~ /ring1_addr:(.*)$/)
							{
								$ring1_addr = $anvil->Words->clean_spaces({string => $1});
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ring1_addr => $ring1_addr }});
								
								$nodelist_body .= $line."\n"; 
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }});
								next;
							}
							elsif ($line =~ /}/)
							{
								$in_node = 0;
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_node => $in_node }});
								
								# Is this the local or peer node?
								my $test_ring0_addr = $peer_bcn_name;
								my $test_ring1_addr = $peer_sn_name;
								if (($in_node_name eq $anvil->Get->host_name) or ($in_node_name eq $anvil->Get->short_host_name))
								{
									# It's us
									$test_ring0_addr = $local_bcn_name;
									$test_ring1_addr = $local_sn_name;
								}
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:ring0_addr'      => $ring0_addr, 
									's2:test_ring0_addr' => $test_ring0_addr,
									's3:ring1_addr'      => $ring1_addr, 
									's4:test_ring1_addr' => $test_ring1_addr, 
								}});
								
								if ((not $ring1_addr) or ($ring0_addr ne $test_ring0_addr) or ($ring1_addr ne $test_ring1_addr))
								{
									# Rewrite the nodelist body.
									my $new_nodelist_body  = "        ring0_addr: ".$test_ring0_addr."\n";
									$new_nodelist_body .= "        ring1_addr: ".$test_ring1_addr."\n";
									foreach my $nodelist_line (split/\n/, $nodelist_body)
									{
										$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_line => $nodelist_line }});
										next if $nodelist_line =~ /ring\d_addr/;
										
										$new_nodelist_body .= $nodelist_line."\n";
										$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_nodelist_body => $new_nodelist_body }});
									}
									
									$nodelist_body = $new_nodelist_body;
									$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_nodelist_body => $new_nodelist_body }});
								}
								
								$new_corosync_conf .= $nodelist_body;
								$new_corosync_conf .= $line."\n";
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
								
								$ring0_addr         = "";
								$ring1_addr         = "";
								$in_node_name       = "";
								$nodelist_body      = "";
								next;
							}
							else
							{
								# Normal line, stash it.
								$nodelist_body .= $line."\n"; 
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }});
								next;
							}
						}
						elsif ($line =~ /}/)
						{
							$in_nodelist = 0;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_nodelist => $in_nodelist }});
							
							$new_corosync_conf .= $line."\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
							next;
						}
					}
					if ($in_totem)
					{
						if ($line =~ /}/)
						{
							if (not $token_seen)
							{
								$new_corosync_conf .= "    token: 10000\n";
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
							}
							$new_corosync_conf .= $line."\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
							
							$in_totem   = 0;
							$token_seen = 0;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								in_totem   => $in_totem,
								token_seen => $token_seen,
							}});
							next;
						}
						if ($line =~ /token:/)
						{
							$token_seen = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
						}
						$new_corosync_conf .= $line."\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
					}
					else
					{
						# Normal line 
						$new_corosync_conf .= $line."\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
					}
				}
				
				# Take the last new lines of the file bodies.
				$old_corosync_conf =~ s/\n$//g;
				$new_corosync_conf =~ s/\n$//g;
				
				my $difference = diff \$old_corosync_conf, \$new_corosync_conf, { STYLE => 'Unified' };
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
				
				if ($difference)
				{
					# Update the corosync.conf, sync it and reload corosync.
					update_progress($anvil, ($anvil->data->{job}{progress}+1), "log_0643,!!difference!".$difference."!!");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0643", variables => { difference => $difference }});
					$anvil->Storage->write_file({
						file      => $anvil->data->{path}{configs}{'corosync.conf'}, 
						body      => $new_corosync_conf,
						overwrite => 1,
						backup    => 1,
					});
					
					# Sync
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0644"});
					my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster sync";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					# RC 0 is OK
					my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					update_progress($anvil, ($anvil->data->{job}{progress}+1), "job_0348,!!return_code!".$return_code."!!,!!output!".$output."!!");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0348", variables => { 
						output      => $output,
						return_code => $return_code, 
					}});
					
					# Reload
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0645"});
					$shell_call = $anvil->data->{path}{exe}{pcs}." cluster reload corosync";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					# RC 0 is OK
					($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					update_progress($anvil, ($anvil->data->{job}{progress}+1), "job_0349,!!return_code!".$return_code."!!,!!output!".$output."!!");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0349", variables => { 
						output      => $output,
						return_code => $return_code, 
					}});
				}
			}
		}
		
		if ($waiting)
		{
			sleep 5;
		}
	}
	
	return(0);
}

# Check if we need to change any IPs or our hostname.
sub check_local_network
{
	my ($anvil) = @_;
	
	# What host name and IP(s) should I have?
	my $local_host    = $anvil->Get->short_host_name();
	my $machine       = $anvil->data->{sys}{machine};
	my $manifest_uuid = $anvil->data->{sys}{manifest_uuid};
	
	my $domain        = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{domain};
	my $old_host_name = $anvil->Get->host_name;
	my $new_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{name};
	if ($domain)
	{
		$new_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{name}.".".$domain;
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		's2:domain'        => $domain, 
		's3:old_host_name' => $old_host_name, 
		's4:new_host_name' => $new_host_name, 
	}});
	
	$anvil->data->{sys}{host_name} = $new_host_name;
	
	# If the hostname isn't the same, change it.
	if ($old_host_name ne $new_host_name)
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0061,!!host_name!".$new_host_name."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0061", variables => { host_name => $new_host_name }});
		my ($now_host_name) = $anvil->System->host_name({
			debug => 2,
			set   => $new_host_name,
		});
		if ($now_host_name eq $new_host_name)
		{
			# Success!
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0045");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0045"});
		}
		else
		{
			# Failed
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0063", variables => {
				host_name         => $new_host_name, 
				current_host_name => $now_host_name, 
			}});
			update_progress($anvil, 0, "job_0063,!!host_name!".$new_host_name."!!,!!current_host_name!".$now_host_name."!!");
			sleep 2;
			$anvil->nice_exit({exit_code => 3});
		}
	}
	else
	{
		# No need to change
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0077,!!host_name!".$new_host_name."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0077", variables => { host_name => $new_host_name }});
	}
	
	# Read the local network manager data.
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0080");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0080"});
	$anvil->Network->read_nmcli({debug => 2});
	
	# Now check IP addresses.
	$anvil->Network->get_ips({debug => 2});
	
	# This will be set when the first IFN with a gateway is set.
	my $default_gateway_interface = "";
	foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
	{
		if ($anvil->data->{network}{$local_host}{interface}{$in_iface}{default_gateway})
		{
			$default_gateway_interface = $in_iface;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { default_gateway_interface => $default_gateway_interface }});
			last;
		}
	}
	
	my $dns = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{dns};
	my $mtu = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{mtu};
	my $ntp = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{ntp};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		dns => $dns,
		mtu => $mtu,
		ntp => $ntp,
	}});
	
	### TODO: sorting the array seems inconsistent, so sorting in a hash
	# The DNS are comma-separated lists, that may or may not have spaces and may or may not be in 
	# alphabetical order. To properly compare, we'll rebuild the CSV string of the current and desired 
	# DNS settings.
	my @dns_array       = split/,/, $dns;
	my $wanted_dns_hash = {};
	foreach my $this_dns (sort {$a cmp $b} @dns_array)
	{
		$this_dns                     = $anvil->Words->clean_spaces({ string => $this_dns });
		$wanted_dns_hash->{$this_dns} = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"wanted_dns_hash->${this_dns}" => $wanted_dns_hash->{$this_dns}, 
		}});
	}
	my $cleaned_wanted_dns = "";
	foreach my $wanted_dns (sort {$a cmp $b} keys %{$wanted_dns_hash})
	{
		$cleaned_wanted_dns .= $wanted_dns.",";
	}
	$cleaned_wanted_dns =~ s/,$//;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cleaned_wanted_dns => $cleaned_wanted_dns }});
	
	# If any network interface is changed, we'll write out the config file then, when done, disconnect 
	# from the database, restart networking and then reconnect before moving on.
	my $restart_interfaces = [];
	
	foreach my $network (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}})
	{
		my $ip      = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}{$network}{ip};
		my $subnet  = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network}{subnet};
		my $cidr    = $anvil->Convert->cidr({subnet_mask => $subnet});
		my $gateway = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network}{gateway};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
			's1:network' => $network, 
			's2:ip'      => $ip,
			's3:subnet'  => $subnet, 
			's4:cidr'    => $cidr, 
			's5:gateway' => $gateway, 
		}});
		foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
		{
			# Only one interface will start with the network name and have an IP address.
			next if $in_iface !~ /^${network}_/;
			next if not $anvil->data->{network}{$local_host}{interface}{$in_iface}{ip};
			my $current_ip      = $anvil->data->{network}{$local_host}{interface}{$in_iface}{ip};
			my $current_subnet  = $anvil->data->{network}{$local_host}{interface}{$in_iface}{subnet_mask};
			my $current_gateway = $anvil->data->{network}{$local_host}{interface}{$in_iface}{gateway};
			my $current_dns     = $anvil->data->{network}{$local_host}{interface}{$in_iface}{dns};
			my $current_mtu     = $anvil->data->{network}{$local_host}{interface}{$in_iface}{mtu};
			my $mac_address     = $anvil->data->{network}{$local_host}{interface}{$in_iface}{mac_address};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:in_iface'        => $in_iface,
				's2:current_ip'      => $current_ip,
				's4:current_subnet'  => $current_subnet, 
				's5:current_gateway' => $current_gateway, 
				's6:current_dns'     => $current_dns, 
				's7:current_mtu'     => $current_mtu, 
			}});
			
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0081,!!name!".$in_iface."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0081", variables => { name => $in_iface }});
			
			if ((not $default_gateway_interface) && ($in_iface =~ /^ifn/) && ($gateway))
			{
				# No existing default gateway, but this is the first IFN we've seen with a 
				# gateway defined, so we'll use this one.
				$default_gateway_interface = $in_iface;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_gateway_interface => $default_gateway_interface }});
			}
			
			my $change            = 0;
			my $current_dns_hash  = {};
			my @current_dns_array = split/,/, $current_dns;
			foreach my $this_current_dns (sort {$a cmp $b} @current_dns_array)
			{
				$this_current_dns                      = $anvil->Words->clean_spaces({ string => $this_current_dns });
				$current_dns_hash->{$this_current_dns} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"current_dns_hash->::${this_current_dns}" => $current_dns_hash->{$this_current_dns}, 
				}});
			}
			my $cleaned_current_dns = "";
			foreach my $current_dns (sort {$a cmp $b} keys %{$current_dns_hash})
			{
				$cleaned_current_dns .= $current_dns.",";
			}
			$cleaned_current_dns =~ s/,$//;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				cleaned_wanted_dns  => $cleaned_wanted_dns, 
				cleaned_current_dns => $cleaned_current_dns,
			}});
			
			if (($current_ip ne $ip) or ($current_subnet ne $subnet))
			{
				# IP / subnet changed.
				#print "IP: .... [".$current_ip."] -> [".$ip."]\n";
				#print "Subnet:  [".$current_subnet."] -> [".$subnet."]\n";
				$change = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
			}
			
			# gateway?
			if ($current_gateway ne $gateway)
			{
				#print "Gateway: [".$current_gateway."] -> [".$gateway."]\n";
				$change = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
			}
			
			# Check DNS only if this is the default gateway interface.
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				in_iface                  => $in_iface,
				default_gateway_interface => $default_gateway_interface, 
				cleaned_current_dns       => $cleaned_current_dns, 
			}});
			if ($in_iface eq $default_gateway_interface)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					cleaned_current_dns => $cleaned_current_dns,
					cleaned_wanted_dns  => $cleaned_wanted_dns, 
				}});
				if ($cleaned_wanted_dns ne $cleaned_current_dns)
				{
					$change = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
				}
			}
			elsif ($cleaned_current_dns)
			{
				# Remove the old DNS entries.
				$change = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
			}
			
			if (not $change)
			{
				# No change
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0082");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0082"});
			}
			else
			{
				# Update the config. 
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0078,!!interface!".$in_iface."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0078", variables => { interface => $in_iface }});
				
				my $interface_uuid = $anvil->data->{nmcli}{$local_host}{device_to_uuid}{$in_iface};
				my $filename       = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{filename};
				my $interface_name = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{name};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:in_iface'       => $in_iface, 
					's2:interface_name' => $interface_name, 
					's3:filename'       => $filename, 
					's4:interface_uuid' => $interface_uuid, 
				}});
				
				# Record the name to restart
				push @{$restart_interfaces}, $interface_name;
				
				# Read in the file. We'll update the date stamp and change the variables as 
				# needed and write it back out.
				my $ip_seen         = 0;
				my $subnet_seen     = 0;
				my $gateway_seen    = 0;
				my $dns_seen        = 0;
				my $mtu_seen        = 0;
				my $boot_proto_seen = 0;
				my $defroute_seen   = 0;
				my $say_default     = $default_gateway_interface eq $in_iface ? "YES" : "NO";
				my $new_config      = "";
				my $old_config      = $anvil->Storage->read_file({file => $filename});
				foreach my $line (split/\n/, $old_config)
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
					if ($line =~ /^BOOTPROTO="(.*?)"/)
					{
						my $boot_protocol   =  $1;
						   $boot_proto_seen =  1;
						   $new_config      .= "BOOTPROTO=\"none\"\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							's1:boot_proto_seen' => $boot_proto_seen, 
							's2:boot_protocol'   => $boot_protocol,
							's3:new_config'      => $new_config,
						}});
						if ($boot_protocol eq "dhcp")
						{
							# Inject the network config.
							if (not $ip_seen)
							{
								$ip_seen    =  1;
								$new_config .= "IPADDR=\"".$ip."\"\n";
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:ip_seen'    => $boot_proto_seen, 
									's2:new_config' => $new_config,
								}});
							}
							if (not $subnet_seen)
							{
								if ($cidr)
								{
									$new_config .= "PREFIX=\"".$cidr."\"\n";
								}
								else
								{
									$new_config .= "NETMASK=\"".$subnet."\"\n";
								}
								$subnet_seen = 1;
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:subnet_seen' => $subnet_seen, 
									's2:new_config'  => $new_config,
								}});
							}
							if (($gateway) && (not $gateway_seen))
							{
								$gateway_seen =  1;
								$new_config   .= "GATEWAY=\"".$gateway."\"\n";
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:gateway_seen' => $gateway_seen, 
									's2:new_config'   => $new_config,
								}});
							}
							if (not $defroute_seen)
							{
								$defroute_seen =  1;
								$new_config    .= "DEFROUTE=\"".$say_default."\"\n";
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:defroute_seen' => $defroute_seen, 
									's2:new_config'    => $new_config,
								}});
							}
							if (($default_gateway_interface eq $in_iface) && ($cleaned_wanted_dns))
							{
								if (not $dns_seen)
								{
									my $i = 1;
									foreach my $this_dns (split/,/, $cleaned_wanted_dns)
									{
										$new_config .= "DNS".$i."=\"".$this_dns."\"\n";
										$i++;
									}

									$dns_seen = 1;
									$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
										's1:dns_seen'   => $dns_seen, 
										's2:new_config' => $new_config,
									}});
								}
							}
						}
					}
					elsif ($line =~ /^IPADDR=".*?"/)
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ip_seen => $ip_seen }});
						if (not $ip_seen)
						{
							$ip_seen    =  1;
							$new_config .= "IPADDR=\"".$ip."\"\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:ip_seen'    => $boot_proto_seen, 
								's2:new_config' => $new_config,
							}});
						}
					}
					elsif (($line =~ /^PREFIX=".*?"/) or ($line =~ /^NETMASK=".*?"/))
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { subnet_seen => $subnet_seen }});
						if (not $subnet_seen)
						{
							$subnet_seen = 1;
							if ($cidr)
							{
								$new_config .= "PREFIX=\"".$cidr."\"\n";
							}
							else
							{
								$new_config .= "NETMASK=\"".$subnet."\"\n";
							}
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:subnet_seen' => $subnet_seen, 
								's2:new_config'  => $new_config,
							}});
						}
					}
					elsif ($line =~ /^GATEWAY=".*"/)
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { gateway_seen => $gateway_seen }});
						if ($in_iface ne $default_gateway_interface)
						{
							# This interface shouldn't have DNS.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0111", variables => { 
								interface    => $in_iface,
								gateway_line => $line,
							}});
							next;
						}
						if (not $gateway_seen)
						{
							$gateway_seen =  1;
							$new_config   .= "GATEWAY=\"".$gateway."\"\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:gateway_seen' => $gateway_seen, 
								's2:new_config'   => $new_config,
							}});
						}
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { gateway_seen => $gateway_seen }});
					}
					elsif ($line =~ /^DNS\d+=".*"/)
					{
						# The first time we see a DNS entry, we inject the DNS we've 
						# got and then ignore the rest.
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dns_seen => $dns_seen }});
						if ($in_iface ne $default_gateway_interface)
						{
							# This interface shouldn't have DNS.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0110", variables => { 
								interface => $in_iface,
								dns_line  => $line,
							}});
							next;
						}
						if (not $dns_seen)
						{
							
							# If there's no DNS, this will do nothing.
							my $i = 1;
							foreach my $this_dns (split/,/, $cleaned_wanted_dns)
							{
								$new_config .= "DNS".$i."=\"".$this_dns."\"\n";
								$i++;
							}
							$dns_seen = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:dns_seen'   => $dns_seen, 
								's2:new_config' => $new_config,
							}});
						}
					}
					elsif ($line =~ /^MTU=".*"/)
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mtu_seen => $mtu_seen }});
						if (not $mtu_seen)
						{
							$mtu_seen   =  1;
							$new_config .= "MTU=\"".$mtu."\"\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:mtu_seen'   => $mtu_seen, 
								's2:new_config' => $new_config,
							}});
						}
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mtu_seen => $mtu_seen }});
					}
					elsif ($line =~ /^DEFROUTE=".*"/)
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }});
						if (not $defroute_seen)
						{
							$defroute_seen =  1;
							$new_config   .= "DEFROUTE=\"".$say_default."\"\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:defroute_seen' => $defroute_seen, 
								's2:new_config'    => $new_config,
							}});
						}
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }});
					}
					else
					{
						$new_config .= $line."\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }});
					}
				}
				
				# Add any values I've not yet seen.
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					ip_seen       => $ip_seen, 
					subnet_seen   => $subnet_seen, 
					gateway_seen  => $gateway_seen, 
					dns_seen      => $dns_seen, 
					mtu_seen      => $mtu_seen, 
					defroute_seen => $defroute_seen, 
				}});
				if (not $ip_seen)
				{
					$new_config .= "IPADDR=\"".$ip."\"\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				if (not $subnet_seen)
				{
					if ($cidr)
					{
						$new_config .= "PREFIX=\"".$cidr."\"\n";
					}
					else
					{
						$new_config .= "NETMASK=\"".$subnet."\"\n";
					}
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				if ($in_iface eq $default_gateway_interface)
				{
					if (not $gateway_seen)
					{
						$new_config .= "GATEWAY=\"".$gateway."\"\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
					}
					if (not $dns_seen)
					{
						# If there's no DNS, this will do nothing.
						my $i = 1;
						foreach my $this_dns (split/,/, $cleaned_wanted_dns)
						{
							$new_config .= "DNS".$i."=\"".$this_dns."\"\n";
							$i++;
						}
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
					}
				}
				if (not $mtu_seen)
				{
					$new_config .= "MTU=\"".$mtu."\"\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				if (not $defroute_seen)
				{
					$new_config .= "DEFROUTE=\"".$say_default."\"\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				
				# Write out the new file.
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0083,!!file!".$filename."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0083", variables => { file => $filename }});
				$anvil->Storage->write_file({
					debug     => 2,
					file      => $filename, 
					body      => $new_config, 
					user      => "root", 
					group     => "root", 
					mode      => "0644", 
					overwrite => 1,
				});
			}
			last;
		}
	}
	
	# If there are any entries in '$restart_interfaces', restart
	my $restart_interface_count = @{$restart_interfaces};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { restart_interface_count => $restart_interface_count }});
	if ($restart_interface_count)
	{
		# Disconnect from the database, as we're about to tear down our connection.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0079"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0079");
		$anvil->Database->disconnect();
		
		# Tell nmcli to re-read the config files.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0463"});
		my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection reload"});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
		
		foreach my $interface_name (sort {$a cmp $b} @{$restart_interfaces})
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0462", variables => { interface => $interface_name }});
			$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection down \"".$interface_name."\""});
			
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0464", variables => { interface => $interface_name }});
			$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection up \"".$interface_name."\""});
		}
		
		# Wait for a DB connection. We'll wait up to 130 seconds (updelay is 120 seconds, plus a small buffer).
		my $wait_until = time + 130;
		until ($anvil->data->{sys}{database}{connections})
		{
			$anvil->refresh();
			$anvil->Database->connect();
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0132"});
			if (not $anvil->data->{sys}{database}{connections})
			{
				if (time > $wait_until)
				{
					# Failed to reconnect, exit. 
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0107"});
					$anvil->nice_exit({exit_code => 4});
				}
				
				# No databases, sleep and then try again.
				sleep 2;
			}
			
			# reload the job data.
			load_job($anvil);
		}
		
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0084"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0084");
	}
	
	# Remove virbr0 if it exists.
	if (exists $anvil->data->{network}{$local_host}{interface}{virbr0})
	{
		# Remove the NAT'ed bridge
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0085"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0085");
		
		$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-destroy default"});
		$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-undefine default "});
		
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0034"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0034");
	}
	
	# Update network view
	$anvil->Network->read_nmcli({debug => 3});
	$anvil->Network->get_ips({debug => 3});
	
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0086"});
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0086");
	
	# Update MTUs (running interface and config) if needed.
	foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
	{
		# Only one interface will start with the network name and have an IP address.
		my $current_mtu = $anvil->data->{network}{$local_host}{interface}{$in_iface}{mtu};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
			's1:in_iface'    => $in_iface, 
			's2:current_mtu' => $current_mtu,
		}});
		
		if (($mtu) && ($current_mtu eq $mtu))
		{
			# It's fine
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0087,!!interface!".$in_iface."!!,!!mtu!".$mtu."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0087", variables => { 
				interface => $in_iface,
				mtu       => $mtu,
			}});
		}
		else
		{
			# Change the MTU both on the running interface and in the config file.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0088,!!interface!".$in_iface."!!,!!old_mtu!".$current_mtu."!!,!!mtu!".$mtu."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0088", variables => { 
				interface => $in_iface, 
				old_mtu   => $current_mtu, 
				mtu       => $mtu, 
			}});
			
			# Change the live MTU.
			$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{ip}." link set ".$in_iface." mtu ".$mtu});
			
			# Now update the config file.
			my $interface_uuid = $anvil->data->{nmcli}{$local_host}{device_to_uuid}{$in_iface};
			my $filename       = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{filename};
			my $interface_name = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				's1:interface_name' => $interface_name, 
				's2:filename'       => $filename, 
				's3:interface_uuid' => $interface_uuid, 
			}});
			
			my $mtu_seen   = 0;
			my $new_config = "";
			my $old_config = $anvil->Storage->read_file({file => $filename});
			foreach my $line (split/\n/, $old_config)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
				if ($line =~ /^MTU=".*?"/)
				{
					$mtu_seen   =  1;
					$new_config .= "MTU=\"".$mtu."\"\n";
				}
				else
				{
					$new_config .= $line."\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { new_config => $new_config }});
				}
			}
			if (not $mtu_seen)
			{
				# Inject the MTU variable
				$new_config .= "MTU=\"".$mtu."\"\n";
			}
			
			# Write out the new file.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0083,!!file!".$filename."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0083", variables => { file => $filename }});
			$anvil->Storage->write_file({
				debug     => 2,
				file      => $filename, 
				body      => $new_config, 
				user      => "root", 
				group     => "root", 
				mode      => "0644", 
				overwrite => 1,
			});
			
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0034"});
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0034");
		}
	}
	
	# Update NTP if set and needed.
	if ($ntp)
	{
		# Break up the NTP servers into a list, we'll set to '1' the ones we find.
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0089");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0089"});
		my $new_ntp = {};
		foreach my $ntp_server (split/,/, $ntp)
		{
			$ntp_server = $anvil->Words->clean_spaces({ string => $ntp_server });
			$new_ntp->{$ntp_server} = 0;
		}
		
		# Call chrony to see what servers are setup already.
		my $change     = 0;
		my $new_config = "";
		my $old_config = $anvil->Storage->read_file({file => $anvil->data->{path}{data}{'chrony.conf'}});
		foreach my $line (split/\n/, $old_config)
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
			$new_config .= $line."\n";
			if ($line =~ /^Server (.*)$/)
			{
				# This is a source.
				my $source = $1;
				$new_ntp->{$source} = 1;
			}
		}
		
		# Are any sources not enabled?
		foreach my $source (sort {$a cmp $b} keys %{$new_ntp})
		{
			if (not $new_ntp->{$source})
			{
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0090,!!server!".$source."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0090", variables => { server => $source }});
				$change = 1;
				$new_config .= "Server ".$source."\n";
			}
		}
		
		if ($change)
		{
			# Write out the updated file.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0055,!!file!".$anvil->data->{path}{data}{'chrony.conf'}."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0055", variables => { file => $anvil->data->{path}{data}{'chrony.conf'} }});
			$anvil->Storage->write_file({
				debug     => 2,
				file      => $anvil->data->{path}{data}{'chrony.conf'}, 
				body      => $new_config, 
				user      => "root", 
				group     => "root", 
				mode      => "0644", 
				overwrite => 1,
			});
			
			# Restart the daemon.
			my $daemon = "chronyd";
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0091,!!daemon!".$daemon."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0091", variables => { daemon => $daemon }});
			$anvil->System->restart_daemon({
				debug  => 3,
				daemon => $daemon,
			})
		}
	}
	
	# Update the hosts file.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0112"});
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0112");
	$anvil->System->update_hosts({debug => 3});
	
	# Configure SSH by adding ours and our peer's SSH keys to ~/.ssh/known_hosts
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0113"});
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0113");
	$anvil->System->check_ssh_keys({debug => 2});
	
	# Setup IPMI, if needed.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0114"});
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0114");
	$anvil->System->configure_ipmi({debug => 2, manifest_uuid => $manifest_uuid});
	
	return(0);
}

# Load the job details.
sub load_job
{
	my ($anvil) = @_;
	
	# See if we can find the job details. This method checks for the 'job-uuid' switch if it was used.
	$anvil->data->{switches}{'job-uuid'} = "" if not exists $anvil->data->{switches}{'job-uuid'};
	$anvil->data->{jobs}{job_uuid}       = "" if not exists $anvil->data->{jobs}{job_uuid};
	
	$anvil->Job->get_job_details({debug => 3});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		"jobs::job_uuid" => $anvil->data->{jobs}{job_uuid},
	}});
	
	if (not $anvil->data->{jobs}{job_uuid})
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0130"});
		sleep 1;
		$anvil->nice_exit({exit_code => 7});
	}
	
	$anvil->data->{job}{progress} = 0;
	update_progress($anvil, $anvil->data->{job}{progress}, "clear");
	update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0074,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0074", variables => { 'job-uuid' => $anvil->data->{switches}{'job-uuid'} }});
	
	my ($machine, $manifest_uuid, $anvil_uuid) = ($anvil->data->{jobs}{job_data} =~ /as_machine=(.*?),manifest_uuid=(.*?),anvil_uuid=(.*?)$/);
	$machine       = "" if not defined $machine;
	$manifest_uuid = "" if not defined $manifest_uuid;
	$anvil_uuid    = "" if not defined $anvil_uuid;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 
		's1:job_data'      => $anvil->data->{jobs}{job_data},
		's2:machine'       => $machine,
		's3:manifest_uuid' => $manifest_uuid, 
		's4:anvil_uuid'    => $anvil_uuid, 
	}});
	
	if ((not $machine)       or 
	    (not $manifest_uuid) or 
	    (not $anvil_uuid))
	{
		# This occassionally is hit, but then works when tried again. 
		update_progress($anvil, 100, "error_0308,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!,!!raw!".$anvil->data->{jobs}{job_data}."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0308", variables => { 
			'job-uuid' => $anvil->data->{switches}{'job-uuid'},
			raw        => $anvil->data->{jobs}{job_data},
		}});	
		sleep 2;
		$anvil->nice_exit({exit_code => 5});
	}
	
	$anvil->data->{sys}{machine}       = $machine;
	$anvil->data->{sys}{manifest_uuid} = $manifest_uuid;
	$anvil->data->{sys}{anvil_uuid}    = $anvil_uuid;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"sys::machine"         => $anvil->data->{sys}{machine},
		"sys::manifest_uuid"   => $anvil->data->{sys}{manifest_uuid},
		"sys::anvil_uuid"      => $anvil->data->{sys}{anvil_uuid},
	}});
	
	# Load in the host, manifest and anvil data.
	$anvil->Database->get_hosts();
	$anvil->Database->get_manifests();
	$anvil->Database->get_anvils();
	
	# Parse the manifest
	if (not exists $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid})
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "warning_0046", variables => { uuid => $anvil->data->{sys}{manifest_uuid} }});
		update_progress($anvil, 0, "warning_0046,!!uuid!".$anvil->data->{sys}{manifest_uuid}."!!");
		sleep 10;
		$anvil->nice_exit({exit_code => 2});
	}
	
	# Parse the manifest
	my $problem = $anvil->Striker->load_manifest({manifest_uuid => $anvil->data->{sys}{manifest_uuid}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		manifest_uuid => $anvil->data->{sys}{manifest_uuid},
		problem       => $problem, 
	}});
	if ($problem)
	{
		# Something went wrong, fatally. Abort the job.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0076", variables => { uuid => $anvil->data->{sys}{manifest_uuid} }});
		update_progress($anvil, 100, "job_0076,!!uuid!".$anvil->data->{sys}{manifest_uuid}."!!");
		$anvil->nice_exit({exit_code => 2});
	}
	
	# Make sure we have a valid Anvil! 
	if (not exists $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid})
	{
		# Odd. Error out, the Anvil! might not be loaded yet.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "warning_0121", variables => { uuid => $anvil->data->{sys}{anvil_uuid} }});
		update_progress($anvil, 0, "warning_0121,!!uuid!".$anvil->data->{sys}{anvil_uuid}."!!");
		sleep 10;
		$anvil->nice_exit({exit_code => 5});
	}
	
	# Load the anvil
	$anvil->data->{sys}{node1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	$anvil->data->{sys}{node2_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		"sys::node1_host_uuid" => $anvil->data->{sys}{node1_host_uuid},
		"sys::node2_host_uuid" => $anvil->data->{sys}{node2_host_uuid},
	}});
	update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0075,!!machine!".$anvil->data->{sys}{machine}."!!,!!manifest_uuid!".$anvil->data->{sys}{manifest_uuid}."!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0075", variables => { 
		machine       => $anvil->data->{sys}{machine},
		manifest_uuid => $anvil->data->{sys}{manifest_uuid},
	}});	
	
	return(0);
}

# If this is being called as a job, this will allow the progress to be updated.
sub update_progress
{
	my ($anvil, $progress, $message) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		's1:progress' => $progress,
		's2:message'  => $message, 
	}});

	$progress = 98 if $progress > 100;
	if (not $anvil->data->{switches}{'job-uuid'})
	{
		return(0);
	}
	
	$anvil->Job->update_progress({
		debug    => 3,
		progress => $progress, 
		message  => $message,
		job_uuid => $anvil->data->{switches}{'job-uuid'},
	});
	
	return(0);
}