anvil/tools/anvil-join-anvil

#!/usr/bin/perl
# 
# This adds a node (or gets a node to join to a new/rebuilt peer or added DR) to an Anvil!.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Failed to connect to any database.
# 2 = Failed to load/parse the manifest.
# 3 = Failed to change the host name.
# 4 = Failed to reconnect to any database after the network was reconfigured
# 5 = Problem parsing job data or loading manifest or anvil data using job data.
# 
# TODO: 
# - 
# 

use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
use String::ShellQuote;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

# Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is 
# passed directly, it will be used. Otherwise, the password will be read from the database.
$anvil->Get->switches;

$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0129"});
	sleep 2;
	$anvil->nice_exit({exit_code => 1});
}

# Get the job details
load_job($anvil);

# Check if we need to change any IPs or our hostname.
check_local_network($anvil);

# (wait for out peer and) Configure pacemaker
configure_pacemaker($anvil);

$anvil->nice_exit({exit_code => 0});

#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# (wait for out peer and) Configure pacemaker. If this is a DR host, this is skipped.
sub configure_pacemaker
{
	my ($anvil) = @_;
	
	my $machine       = $anvil->data->{sys}{machine};
	my $manifest_uuid = $anvil->data->{sys}{manifest_uuid};
	
	### TODO: Move these to variables in the 'sys' hash
	my $anvil_name       =  $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{name};
	my $anvil_uuid       =  $anvil->data->{sys}{anvil_uuid};
	my $host_name        =  $anvil->data->{sys}{host_name};
	my $new_password     =  $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid  =  $anvil->data->{sys}{node1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name  =  $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	   $node1_host_name  =~ s/\..*$//;
	my $node2_host_uuid  =  $anvil->data->{sys}{node2_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name  =  $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	   $node2_host_name  =~ s/\..*$//;
	my $peer_host_name   =  $anvil->Get->host_uuid() eq $node1_host_uuid ? $node2_host_name : $node1_host_name;
	my $peer_host_uuid   =  $anvil->Get->host_uuid() eq $node1_host_uuid ? $node2_host_uuid : $node1_host_uuid;
	my $escaped_password =  shell_quote($new_password);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		machine          => $machine, 
		anvil_uuid       => $anvil_uuid, 
		anvil_name       => $anvil_name, 
		host_name        => $host_name, 
		manifest_uuid    => $manifest_uuid,
		node1_host_uuid  => $node1_host_uuid, 
		node1_host_name  => $node1_host_name, 
		node2_host_uuid  => $node2_host_uuid, 
		node2_host_name  => $node2_host_name, 
		peer_host_name   => $peer_host_name, 
		peer_host_uuid   => $peer_host_uuid, 
		new_password     => $anvil->Log->is_secure($new_password), 
		escaped_password => $anvil->Log->is_secure($escaped_password),
	}});
	
	# If this is a DR box, we don't use pacemaker.
	if ($machine =~ /dr\d+/)
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 10), "job_0096");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0096"});
		return(0);
	}
	
	### Run on both nodes.
	# Set the 'hacluster' user password.
	my ($return_code) = $anvil->System->change_shell_user_password({
		debug        => 3,
		user         => "hacluster",
		new_password => $new_password,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	$return_code = undef;
	
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0093,!!user!hacluster!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0093", variables => { user => "hascluster" }});
	
	# Enable pcsd and start the pcsd daemons.
	($return_code) = $anvil->System->enable_daemon({daemon => "pcsd"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	$return_code = undef;
	($return_code) = $anvil->System->start_daemon({daemon => "pcsd"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	$return_code = undef;
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0094,!!daemon!pcsd!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0094", variables => { daemon => "pcsd" }});
	
	# Disabled and stop the libvirtd daemon.
	foreach my $daemon ("libvirtd.service", "drbd.service")
	{
		my ($return_code) = $anvil->System->disable_daemon({daemon => $daemon});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
		$return_code = undef;
		($return_code) = $anvil->System->stop_daemon({daemon => $daemon});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
		$return_code = undef;
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!".$daemon."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0095", variables => { daemon => $daemon }});
	}
	
	# If there is no corosync.conf, see if the peer has it. If so, copy it. If not, we'll initialize the
	# cluster shortly.
	if (not -e $anvil->data->{path}{configs}{'corosync.conf'})
	{
		my $corosync_conf = $anvil->Storage->read_file({
			file   => $anvil->data->{path}{configs}{'corosync.conf'},
			target => $peer_host_name,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { corosync_conf => $corosync_conf }});
		if ($corosync_conf ne "!!error!!")
		{
			# Write the file out.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0100"});
			$anvil->Storage->write_file({
				body  => $corosync_conf,
				file  => $anvil->data->{path}{configs}{'corosync.conf'}, 
				user  => "root",
				group => "root",
				mode  => "0644",
			});
		}
	}
	
	# Node 1 initializes, node 2 waits.
	if ($machine eq "node2")
	{
		my $start_time = 0;
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0103");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0103"});
		
		# We loop until the peer finishes or the peer's job hit's 100.
		my $tried_starting = 0;
		my $both_online    = 0;
		until($both_online)
		{
			if (-e $anvil->data->{path}{configs}{'corosync.conf'})
			{
				if (not $start_time)
				{
					# Corosync is configured, we'll wait up to two minutes and then try
					# joining the cluster ourselves.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0108"});
					$start_time = time + 120;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { start_time => $start_time }});
				}
				elsif ((time > $start_time) && (not $tried_starting))
				{
					# We've waited a minute, time to try starting the cluster.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0106"});
					my $cluster_started = $anvil->Cluster->start_cluster({debug => 2, all => 1});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cluster_started => $cluster_started }});
					
					# Mark that weve tried to start.
					$tried_starting = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { tried_starting => $tried_starting }});
				}
				
				my $problem = $anvil->Cluster->parse_cib({debug => 2});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
				if (not $problem)
				{
					# See if both nodes are online.
					my $node1_ready = $anvil->Cluster->check_node_status({node_name => $node1_host_name});
					my $node2_ready = $anvil->Cluster->check_node_status({node_name => $node2_host_name});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						node1_ready => $node1_ready,
						node2_ready => $node2_ready, 
					}});
					if (($node1_ready) && ($node2_ready))
					{
						$both_online = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { both_online => $both_online }});
						
						update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0104");
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0104"});
					}
					else
					{
						# Not online yet, wait a bit.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0105", variables => {
							node1_name   => $node1_host_name,
							node1_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm},
							node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd},
							node1_crmd   => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{'join'},
							node1_join   => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{ready},
							node2_name   => $node2_host_name,
							node2_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{in_ccm},
							node2_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{crmd},
							node2_crmd   => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{'join'},
							node2_join   => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{ready},
						}});
					}
				}
			}
			else
			{
				# corosync.conf doesn't exist yet.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0107"});
			}
			sleep 5 if not $both_online;
		}
	}
	else
	{
		# We're node 1, proceed with cluster setup.
		my $waiting         = 1;
		my $warning_printed = 0;
		while($waiting)
		{
			# Try to authenticate against the peer.
			my $auth_shell_call        = $anvil->data->{path}{exe}{pcs}." host auth ".$node1_host_name." ".$node2_host_name." -u hacluster -p ".$escaped_password;
			my ($output, $return_code) = $anvil->System->call({debug => 3, secure => 1, shell_call => $auth_shell_call});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output,
				return_code => $return_code, 
			}});
			if ($return_code)
			{
				# Something went wrong. 
				if (not $warning_printed)
				{
					# Update the job
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0097");
					$warning_printed = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { warning_printed => $warning_printed }});
				}
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0097"});
				sleep 5;
			}
			else
			{
				# We're good.
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0098");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0098"});
				$waiting = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
			}
		}
		
		# If there is no corosync.conf, see if the peer has it. If so, copy it. If not, initialize 
		# the cluster.
		if (not -e $anvil->data->{path}{configs}{'corosync.conf'})
		{
			# There's no cluster yet, initialize it.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0099,!!anvil_name!".$anvil_name."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0099", variables => { anvil_name => $anvil_name }});
			
			my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster setup ".$anvil_name." ".$node1_host_name." ".$node2_host_name;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output,
				return_code => $return_code, 
			}});
			if ($return_code)
			{
				# Something went wrong
				update_progress($anvil, 100, "job_0101,!!error!".$output."!!");
				sleep 2;
				$anvil->nice_exit({exit_code => 5});
			}
		}
		
		# If we can parse the CIB, then pcsd is running. 
		my $problem = $anvil->Cluster->parse_cib({debug => 2});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		if ($problem)
		{
			# Start the cluster.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0102");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0102"});
			
			my $cluster_started = $anvil->Cluster->start_cluster({debug => 2, all => 1});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cluster_started => $cluster_started }});
		}
		
		# Now wait for both nodes to come online.
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0109");
		my $both_online = 0;
		until ($both_online)
		{
			my $problem = $anvil->Cluster->parse_cib({debug => 2});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
			if (not $problem)
			{
				# See if both nodes are online.
				my $node1_ready = $anvil->Cluster->check_node_status({node_name => $node1_host_name});
				my $node2_ready = $anvil->Cluster->check_node_status({node_name => $node2_host_name});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					node1_ready => $node1_ready,
					node2_ready => $node2_ready, 
				}});
				if (($node1_ready) && ($node2_ready))
				{
					$both_online = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { both_online => $both_online }});
					
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0104");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0104"});
				}
				else
				{
					# Not online yet, wait a bit.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0105", variables => {
						node1_name   => $node1_host_name,
						node1_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm},
						node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd},
						node1_crmd   => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{'join'},
						node1_join   => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{ready},
						node2_name   => $node2_host_name,
						node2_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{in_ccm},
						node2_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{crmd},
						node2_crmd   => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{'join'},
						node2_join   => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{ready},
					}});
				}
			}
			sleep 5 if not $both_online;
		}
	}
	
	
	die;
=cut
$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed};
$VAR1 = {
          'fences' => {
                        'pulsar' => 'c2be8d2e-045e-45d0-80e7-7443f2314fb1',
                        'el8-pdu02' => '2681049b-1ae7-444a-890b-39aca262e79e',
                        'el8-pdu01' => 'bad1fc5c-5342-4385-8d0d-a87a342dd87d',
                        'an-nas02' => '4117a862-f58f-4676-991a-9ca257a3c612'
                      },
          'domain' => 'digimer.ca',
          'networks' => {
                          'count' => {
                                       'ifn' => 1,
                                       'bcn' => 1,
                                       'sn' => 1
                                     },
                          'dns' => '8.8.8.8,8.8.4.4',
                          'mtu' => '1500',
                          'ntp' => '10.201.14.1,10.201.14.2',
                          'name' => {
                                      'ifn1' => {
                                                  'subnet' => '255.255.255.0',
                                                  'gateway' => '192.168.122.1',
                                                  'network' => '192.168.122.0'
                                                },
                                      'bcn1' => {
                                                  'network' => '10.201.0.0',
                                                  'gateway' => '',
                                                  'subnet' => '255.255.0.0'
                                                },
                                      'sn1' => {
                                                 'gateway' => '',
                                                 'network' => '10.101.0.0',
                                                 'subnet' => '255.255.0.0'
                                               }
                                    }
                        },
          'upses' => {
                       'el8-ups01' => {
                                        'uuid' => '7ebecdda-782d-4624-841d-98d912ed3d50'
                                      },
                       'el8-ups02' => {
                                        'uuid' => '7ffb4dc2-8b96-4ca7-80bb-49e309fb2f5f'
                                      }
                     },
          'machine' => {
                         'node2' => {
                                      'network' => {
                                                     'bcn1' => {
                                                                 'ip' => '10.201.10.2'
                                                               },
                                                     'sn1' => {
                                                                'ip' => '10.101.10.2'
                                                              },
                                                     'ifn1' => {
                                                                 'ip' => '192.168.122.12'
                                                               }
                                                   },
                                      'ups' => {
                                                 'el8-ups01' => {
                                                                  'used' => '0'
                                                                },
                                                 'el8-ups02' => {
                                                                  'used' => '0'
                                                                }
                                               },
                                      'fence' => {
                                                   'pulsar' => {
                                                                 'port' => ''
                                                               },
                                                   'el8-pdu02' => {
                                                                    'port' => ''
                                                                  },
                                                   'el8-pdu01' => {
                                                                    'port' => ''
                                                                  },
                                                   'an-nas02' => {
                                                                   'port' => 'el8-a01n02'
                                                                 }
                                                 },
                                      'name' => 'el8-a01n02',
                                      'ipmi_ip' => ''
                                    },
                         'node1' => {
                                      'ipmi_ip' => '',
                                      'name' => 'el8-a01n01',
                                      'ups' => {
                                                 'el8-ups02' => {
                                                                  'used' => '0'
                                                                },
                                                 'el8-ups01' => {
                                                                  'used' => '0'
                                                                }
                                               },
                                      'network' => {
                                                     'ifn1' => {
                                                                 'ip' => '192.168.122.11'
                                                               },
                                                     'bcn1' => {
                                                                 'ip' => '10.201.10.1'
                                                               },
                                                     'sn1' => {
                                                                'ip' => '10.101.10.1'
                                                              }
                                                   },
                                      'fence' => {
                                                   'el8-pdu02' => {
                                                                    'port' => ''
                                                                  },
                                                   'pulsar' => {
                                                                 'port' => ''
                                                               },
                                                   'an-nas02' => {
                                                                   'port' => 'el8-a01n01'
                                                                 },
                                                   'el8-pdu01' => {
                                                                    'port' => ''
                                                                  }
                                                 }
                                    },
                         'dr1' => {
                                    'fence' => {
                                                 'pulsar' => {
                                                               'port' => ''
                                                             },
                                                 'el8-pdu02' => {
                                                                  'port' => ''
                                                                },
                                                 'el8-pdu01' => {
                                                                  'port' => ''
                                                                },
                                                 'an-nas02' => {
                                                                 'port' => ''
                                                               }
                                               },
                                    'network' => {
                                                   'bcn1' => {
                                                               'ip' => '10.201.10.3'
                                                             },
                                                   'sn1' => {
                                                              'ip' => '10.101.10.3'
                                                            },
                                                   'ifn1' => {
                                                               'ip' => '192.168.122.13'
                                                             }
                                                 },
                                    'ups' => {
                                               'el8-ups02' => {
                                                                'used' => '0'
                                                              },
                                               'el8-ups01' => {
                                                                'used' => '0'
                                                              }
                                             },
                                    'name' => 'el8-a01dr01',
                                    'ipmi_ip' => ''
                                  }
                       },
          'sequence' => '01',
          'name' => 'el8-anvil-01',
          'prefix' => 'el8'
        };

	
909; xxx::upses::el8-ups01::uuid: [7ebecdda-782d-4624-841d-98d912ed3d50]
909; xxx::upses::el8-ups02::uuid: [7ffb4dc2-8b96-4ca7-80bb-49e309fb2f5f]
918; xxx::fences::an-nas02::uuid: [4117a862-f58f-4676-991a-9ca257a3c612]

949; xxx::networks::name::bcn1::gateway: [], xxx::networks::name::bcn1::network: [10.201.0.0], xxx::networks::name::bcn1::subnet: [255.255.0.0]
949; xxx::networks::name::sn1::gateway: [], xxx::networks::name::sn1::network: [10.101.0.0], xxx::networks::name::sn1::subnet: [255.255.0.0]
949; xxx::networks::name::ifn1::gateway: [10.255.255.254], xxx::networks::name::ifn1::network: [10.255.0.0], xxx::networks::name::ifn1::subnet: [255.255.0.0]
961; xxx::networks::count::bcn: [1], xxx::networks::count::ifn: [1], xxx::networks::count::sn: [1]

971; xxx::machine::node1::ipmi_ip: [], xxx::machine::node1::type: [!!undef!!]
980; xxx::machine::node1::fence::an-nas02::port: [el8-a01n01]
980; xxx::machine::node1::fence::el8-pdu01::port: []
980; xxx::machine::node1::fence::el8-pdu02::port: []
980; xxx::machine::node1::fence::pulsar::port: []
989; xxx::machine::node1::ups::el8-ups01::used: [0]
989; xxx::machine::node1::ups::el8-ups02::used: [0]
998; xxx::machine::node1::network::bcn1::ip: [10.201.10.1]
998; xxx::machine::node1::network::ifn1::ip: [10.255.10.1]
998; xxx::machine::node1::network::sn1::ip: [10.101.10.1]
=cut
	
	
	return(0);
}

# Check if we need to change any IPs or our hostname.
sub check_local_network
{
	my ($anvil) = @_;
	
	# What host name and IP(s) should I have?
	my $machine       = $anvil->data->{sys}{machine};
	my $manifest_uuid = $anvil->data->{sys}{manifest_uuid};
	
	my $domain        = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{domain};
	my $old_host_name = $anvil->_host_name;
	my $new_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{name};
	if ($domain)
	{
		$new_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{name}.".".$domain;
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's2:domain'        => $domain, 
		's3:old_host_name' => $old_host_name, 
		's4:new_host_name' => $new_host_name, 
	}});
	
	$anvil->data->{sys}{host_name} = $new_host_name;
	
	# If the hostname isn't the same, change it.
	if ($old_host_name ne $new_host_name)
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0061,!!host_name!".$new_host_name."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0061", variables => { host_name => $new_host_name }});
		my ($now_host_name) = $anvil->System->host_name({
			debug => 2,
			set   => $new_host_name,
		});
		if ($now_host_name eq $new_host_name)
		{
			# Success!
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0045");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0045"});
		}
		else
		{
			# Failed
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0063", variables => {
				host_name         => $new_host_name, 
				current_host_name => $now_host_name, 
			}});
			update_progress($anvil, 0, "job_0063,!!host_name!".$new_host_name."!!,!!current_host_name!".$now_host_name."!!");
			sleep 2;
			$anvil->nice_exit({exit_code => 3});
		}
	}
	else
	{
		# No need to change
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0077,!!host_name!".$new_host_name."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0077", variables => { host_name => $new_host_name }});
	}
	
	# Read the local network manager data.
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0080");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0080"});
	$anvil->Network->read_nmcli({debug => 3});
	
	# Now check IP addresses.
	$anvil->Network->get_ips({debug => 3});
	
	# This will be set when the first IFN with a gateway is set.
	my $default_gateway_interface = "";
	foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{'local'}{interface}})
	{
		if ($anvil->data->{network}{'local'}{interface}{$in_iface}{default_gateway})
		{
			$default_gateway_interface = $in_iface;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_gateway_interface => $default_gateway_interface }});
			last;
		}
	}
	
	my $dns = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{dns};
	my $mtu = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{mtu};
	my $ntp = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{ntp};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		dns => $dns,
		mtu => $mtu,
		ntp => $ntp,
	}});
	
	# The DNS are comma-separated lists, that may or may not have spaces and may or may not be in 
	# alphabetical order. To properly compare, we'll rebuild the CSV string of the current and desired 
	# DNS settings.
	my $cleaned_dns = "";
	my @dns_array   = split/,/, $dns;
	foreach my $this_dns (sort {$a cmp $b} @dns_array)
	{
		$this_dns    =  $anvil->Words->clean_spaces({ string => $this_dns });
		$cleaned_dns .= $this_dns.",";
	}
	$cleaned_dns =~ s/,$//;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cleaned_dns => $cleaned_dns }});
	
	# If any network interface is changed, we'll write out the config file then, when done, disconnect 
	# from the database, restart networking and then reconnect before moving on.
	my $restart_interfaces = [];
	
	foreach my $network (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}})
	{
		my $ip      = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}{$network}{ip};
		my $subnet  = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network}{subnet};
		my $cidr    = $anvil->Convert->cidr({subnet_mask => $subnet});
		my $gateway = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network}{gateway};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:network' => $network, 
			's2:ip'      => $ip,
			's3:subnet'  => $subnet, 
			's4:cidr'    => $cidr, 
			's5:gateway' => $gateway, 
		}});
		foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{'local'}{interface}})
		{
			### BUG: DNS gets replicated, but updated.
			# Only one interface will start with the network name and have an IP address.
			next if $in_iface !~ /^${network}_/;
			next if not $anvil->data->{network}{'local'}{interface}{$in_iface}{ip};
			my $current_ip      = $anvil->data->{network}{'local'}{interface}{$in_iface}{ip};
			my $current_subnet  = $anvil->data->{network}{'local'}{interface}{$in_iface}{subnet_mask};
			my $current_gateway = $anvil->data->{network}{'local'}{interface}{$in_iface}{gateway};
			my $current_dns     = $anvil->data->{network}{'local'}{interface}{$in_iface}{dns};
			my $current_mtu     = $anvil->data->{network}{'local'}{interface}{$in_iface}{mtu};
			my $mac_address     = $anvil->data->{network}{'local'}{interface}{$in_iface}{mac_address};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:in_iface'        => $in_iface,
				's2:current_ip'      => $current_ip,
				's4:current_subnet'  => $current_subnet, 
				's5:current_gateway' => $current_gateway, 
				's6:current_dns'     => $current_dns, 
				's7:current_mtu'     => $current_mtu, 
			}});
			
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0081,!!name!".$in_iface."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0081", variables => { name => $in_iface }});
			
			if ((not $default_gateway_interface) && ($in_iface =~ /^ifn/) && ($gateway))
			{
				# No existing default gateway, but this is the first IFN we've seen with a 
				# gateway defined, so we'll use this one.
				$default_gateway_interface = $in_iface;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_gateway_interface => $default_gateway_interface }});
			}
			
			my $cleaned_current_dns = "";
			my @current_dns_array   = split/,/, $current_dns;
			foreach my $this_current_dns (sort {$a cmp $b} @current_dns_array)
			{
				$this_current_dns    =  $anvil->Words->clean_spaces({ string => $this_current_dns });
				$cleaned_current_dns .= $this_current_dns.",";
			}
			$cleaned_current_dns =~ s/,$//;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cleaned_current_dns => $cleaned_current_dns }});
			
			my $change = 0;
			if (($current_ip ne $ip) or ($current_subnet ne $subnet))
			{
				# IP / subnet changed.
				#print "IP: .... [".$current_ip."] -> [".$ip."]\n";
				#print "Subnet:  [".$current_subnet."] -> [".$subnet."]\n";
				$change = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
			}
			
			# gateway?
			if ($current_gateway ne $gateway)
			{
				#print "Gateway: [".$current_gateway."] -> [".$gateway."]\n";
				$change = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
			}
			
			# Check DNS only if gateway is set.
			if ($gateway)
			{
				if ($cleaned_dns ne $cleaned_current_dns)
				{
					#print "DNS: ... [".$cleaned_current_dns."] -> [".$cleaned_dns."]\n";
					$change = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
				}
				elsif (($in_iface eq $default_gateway_interface) && (not $anvil->data->{network}{'local'}{interface}{$in_iface}{default_gateway}))
				{
					# This isn't the default gateway yet, but we'll make it so.
					#print "Will set as default gateway\n";
					$change = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
				}
			}
			
			if (not $change)
			{
				# No change
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0082");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0082"});
			}
			else
			{
				# Update the config. 
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0078,!!interface!".$in_iface."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0078", variables => { interface => $in_iface }});
				
				my $interface_uuid = $anvil->data->{nmcli}{'local'}{device_to_uuid}{$in_iface};
				my $filename       = $anvil->data->{nmcli}{'local'}{uuid}{$interface_uuid}{filename};
				my $interface_name = $anvil->data->{nmcli}{'local'}{uuid}{$interface_uuid}{name};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:in_iface'       => $in_iface, 
					's2:interface_name' => $interface_name, 
					's3:filename'       => $filename, 
					's4:interface_uuid' => $interface_uuid, 
				}});
				
				# Record the name to restart
				push @{$restart_interfaces}, $interface_name;
				
				# Read in the file. We'll update the date stamp and change the variables as 
				# needed and write it back out.
				my $ip_seen         = 0;
				my $subnet_seen     = 0;
				my $gateway_seen    = 0;
				my $dns_seen        = 0;
				my $mtu_seen        = 0;
				my $boot_proto_seen = 0;
				my $defroute_seen   = 0;
				my $say_default     = $default_gateway_interface eq $in_iface ? "YES" : "NO";
				my $new_config      = "";
				my $old_config      = $anvil->Storage->read_file({file => $filename});
				foreach my $line (split/\n/, $old_config)
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
					if ($line =~ /^BOOTPROTO="(.*?)"/)
					{
						my $boot_protocol   =  $1;
						   $boot_proto_seen =  1;
						   $new_config      .= "BOOTPROTO=\"none\"\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							's1:boot_proto_seen' => $boot_proto_seen, 
							's2:boot_protocol'   => $boot_protocol,
							's3:new_config'      => $new_config,
						}});
						if ($boot_protocol eq "dhcp")
						{
							# Inject the network config.
							if (not $ip_seen)
							{
								$ip_seen    =  1;
								$new_config .= "IPADDR=\"".$ip."\"\n";
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:ip_seen'    => $boot_proto_seen, 
									's2:new_config' => $new_config,
								}});
							}
							if (not $subnet_seen)
							{
								if ($cidr)
								{
									$new_config .= "PREFIX=\"".$cidr."\"\n";
								}
								else
								{
									$new_config .= "NETMASK=\"".$subnet."\"\n";
								}
								$subnet_seen = 1;
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:subnet_seen' => $subnet_seen, 
									's2:new_config'  => $new_config,
								}});
							}
							if (($gateway) && (not $gateway_seen))
							{
								$gateway_seen =  1;
								$new_config   .= "GATEWAY=\"".$gateway."\"\n";
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:gateway_seen' => $gateway_seen, 
									's2:new_config'   => $new_config,
								}});
							}
							if (not $defroute_seen)
							{
								$defroute_seen =  1;
								$new_config    .= "DEFROUTE=\"".$say_default."\"\n";
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:defroute_seen' => $defroute_seen, 
									's2:new_config'    => $new_config,
								}});
							}
							if (($default_gateway_interface eq $in_iface) && ($cleaned_dns))
							{
								if (not $dns_seen)
								{
									my $i = 1;
									foreach my $this_dns (split/,/, $cleaned_dns)
									{
										$new_config .= "DNS".$i."=\"".$this_dns."\"\n";
										$i++;
									}

									$dns_seen = 1;
									$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
										's1:dns_seen'   => $dns_seen, 
										's2:new_config' => $new_config,
									}});
								}
							}
						}
					}
					elsif ($line =~ /^IPADDR=".*?"/)
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ip_seen => $ip_seen }});
						if (not $ip_seen)
						{
							$ip_seen    =  1;
							$new_config .= "IPADDR=\"".$ip."\"\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:ip_seen'    => $boot_proto_seen, 
								's2:new_config' => $new_config,
							}});
						}
					}
					elsif (($line =~ /^PREFIX=".*?"/) or ($line =~ /^NETMASK=".*?"/))
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { subnet_seen => $subnet_seen }});
						if (not $subnet_seen)
						{
							$subnet_seen = 1;
							if ($cidr)
							{
								$new_config .= "PREFIX=\"".$cidr."\"\n";
							}
							else
							{
								$new_config .= "NETMASK=\"".$subnet."\"\n";
							}
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:subnet_seen' => $subnet_seen, 
								's2:new_config'  => $new_config,
							}});
						}
					}
					elsif ($line =~ /^GATEWAY=".*"/)
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { gateway_seen => $gateway_seen }});
						if (not $gateway_seen)
						{
							$gateway_seen =  1;
							$new_config   .= "GATEWAY=\"".$gateway."\"\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:gateway_seen' => $gateway_seen, 
								's2:new_config'   => $new_config,
							}});
						}
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { gateway_seen => $gateway_seen }});
					}
					elsif ($line =~ /^DNS=".*"/)
					{
						# The first time we see a DNS entry, we inject the DNS we've 
						# got and then ignore the rest.
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dns_seen => $dns_seen }});
						if (not $dns_seen)
						{
							# If there's no DNS, this will do nothing.
							my $i = 1;
							foreach my $this_dns (split/,/, $cleaned_dns)
							{
								$new_config .= "DNS".$i."=\"".$this_dns."\"\n";
								$i++;
							}

							$dns_seen = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:dns_seen'   => $dns_seen, 
								's2:new_config' => $new_config,
							}});
						}
					}
					elsif ($line =~ /^MTU=".*"/)
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mtu_seen => $mtu_seen }});
						if (not $mtu_seen)
						{
							$mtu_seen =  1;
							$new_config   .= "MTU=\"".$mtu."\"\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:mtu_seen'   => $mtu_seen, 
								's2:new_config' => $new_config,
							}});
						}
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mtu_seen => $mtu_seen }});
					}
					elsif ($line =~ /^DEFROUTE=".*"/)
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }});
						if (not $defroute_seen)
						{
							$defroute_seen =  1;
							$new_config   .= "DEFROUTE=\"".$say_default."\"\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:defroute_seen' => $defroute_seen, 
								's2:new_config'    => $new_config,
							}});
						}
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }});
					}
					else
					{
						$new_config .= $line."\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }});
					}
				}
				
				# Add any values I've not yet seen.
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					ip_seen       => $ip_seen, 
					subnet_seen   => $subnet_seen, 
					gateway_seen  => $gateway_seen, 
					dns_seen      => $dns_seen, 
					mtu_seen      => $mtu_seen, 
					defroute_seen => $defroute_seen, 
				}});
				if (not $ip_seen)
				{
					$new_config .= "IPADDR=\"".$ip."\"\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				if (not $subnet_seen)
				{
					if ($cidr)
					{
						$new_config .= "PREFIX=\"".$cidr."\"\n";
					}
					else
					{
						$new_config .= "NETMASK=\"".$subnet."\"\n";
					}
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				if (not $gateway_seen)
				{
					$new_config .= "GATEWAY=\"".$gateway."\"\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				if (not $dns_seen)
				{
					# If there's no DNS, this will do nothing.
					my $i = 1;
					foreach my $this_dns (split/,/, $cleaned_dns)
					{
						$new_config .= "DNS".$i."=\"".$this_dns."\"\n";
						$i++;
					}
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				if (not $mtu_seen)
				{
					$new_config .= "MTU=\"".$mtu."\"\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				if (not $defroute_seen)
				{
					$new_config .= "DEFROUTE=\"".$say_default."\"\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				
				# Write out the new file.
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0083,!!file!".$filename."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0083", variables => { file => $filename }});
# 				print "File about to write: [".$filename."]\n";
# 				print "============================\n";
# 				print $new_config."\n";
# 				print "============================\n";
# 				die "There's a bug here that is re-writting the config with duplicate DNS and something the wrong IP for this host.\n";
				$anvil->Storage->write_file({
					debug     => 3,
					file      => $filename, 
					body      => $new_config, 
					user      => "root", 
					group     => "root", 
					mode      => "0644", 
					overwrite => 1,
				});
			}
			last;
		}
	}
	
	# If there are any entries in '$restart_interfaces', restart
	my $restart_interface_count = @{$restart_interfaces};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { restart_interface_count => $restart_interface_count }});
	if ($restart_interface_count)
	{
		# Disconnect from the database, as we're about to tear down our connection.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0079"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0079");
		$anvil->Database->disconnect();
		
		# Tell nmcli to re-read the config files.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0463"});
		my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{nmcli}." connection reload"});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
		
		foreach my $interface_name (sort {$a cmp $b} @{$restart_interfaces})
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0462", variables => { interface => $interface_name }});
			$anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{nmcli}." connection down \"".$interface_name."\""});
			
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0464", variables => { interface => $interface_name }});
			$anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{nmcli}." connection up \"".$interface_name."\""});
		}
	
		# Wait for a DB connection. We'll wait up to 130 seconds (updelay is 120 seconds, plus a small buffer).
		my $wait_until = time + 130;
		until ($anvil->data->{sys}{database}{connections})
		{
			$anvil->refresh();
			$anvil->Database->connect();
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "log_0132"});
			if (not $anvil->data->{sys}{database}{connections})
			{
				if (time > $wait_until)
				{
					# Failed to reconnect, exit. 
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0107"});
					$anvil->nice_exit({exit_code => 4});
				}
				
				# No databases, sleep and then try again.
				sleep 2;
			}
		}
		
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0084"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0084");
	}
	
	# Remove virbr0 if it exists.
	if (exists $anvil->data->{network}{'local'}{interface}{virbr0})
	{
		# Remove the NAT'ed bridge
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0085"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0085");
		
		$anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{virsh}." net-destroy default"});
		$anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{virsh}." net-undefine default "});
		
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0034"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0034");
	}
	
	# Update network view
	$anvil->Network->read_nmcli({debug => 2});
	$anvil->Network->get_ips({debug => 3});
	
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0086"});
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0086");
	
	# Update MTUs (running interface and config) if needed.
	foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{'local'}{interface}})
	{
		# Only one interface will start with the network name and have an IP address.
		my $current_mtu = $anvil->data->{network}{'local'}{interface}{$in_iface}{mtu};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:in_iface'    => $in_iface, 
			's2:current_mtu' => $current_mtu,
		}});
		
		if ($current_mtu eq $mtu)
		{
			# It's fine
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0087,!!interface!".$in_iface."!!,!!mtu!".$mtu."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0087", variables => { 
				interface => $in_iface,
				mtu       => $mtu,
			}});
		}
		else
		{
			# Change the MTU both on the running interface and in the config file.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0088,!!interface!".$in_iface."!!,!!old_mtu!".$current_mtu."!!,!!mtu!".$mtu."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0088", variables => { 
				interface => $in_iface, 
				old_mtu   => $current_mtu, 
				mtu       => $mtu, 
			}});
			
			# Change the live MTU.
			$anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{ip}." link set ".$in_iface." mtu ".$mtu});
			
			# Now update the config file.
			my $interface_uuid = $anvil->data->{nmcli}{'local'}{device_to_uuid}{$in_iface};
			my $filename       = $anvil->data->{nmcli}{'local'}{uuid}{$interface_uuid}{filename};
			my $interface_name = $anvil->data->{nmcli}{'local'}{uuid}{$interface_uuid}{name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:interface_name' => $interface_name, 
				's2:filename'       => $filename, 
				's3:interface_uuid' => $interface_uuid, 
			}});
			
			my $mtu_seen   = 0;
			my $new_config = "";
			my $old_config = $anvil->Storage->read_file({file => $filename});
			foreach my $line (split/\n/, $old_config)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
				if ($line =~ /^MTU=".*?"/)
				{
					$mtu_seen   =  1;
					$new_config .= "MTU=\"".$mtu."\"\n";
				}
				else
				{
					$new_config .= $line."\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
			}
			if (not $mtu_seen)
			{
				# Inject the MTU variable
				$new_config .= "MTU=\"".$mtu."\"\n";
			}
			
			# Write out the new file.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0083,!!file!".$filename."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0083", variables => { file => $filename }});
# 			print "File about to write: [".$filename."]\n";
# 			print "============================\n";
# 			print $new_config."\n";
# 			print "============================\n";
# 			die "There's a bug here that is re-writting the config with duplicate DNS and something the wrong IP for this host.\n";
			$anvil->Storage->write_file({
				debug     => 2,
				file      => $filename, 
				body      => $new_config, 
				user      => "root", 
				group     => "root", 
				mode      => "0644", 
				overwrite => 1,
			});
			
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0034"});
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0034");
		}
	}
	
	# Update NTP if set and needed.
	if ($ntp)
	{
		# Break up the NTP servers into a list, we'll set to '1' the ones we find.
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0089");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0089"});
		my $new_ntp = {};
		foreach my $ntp_server (split/,/, $ntp)
		{
			$ntp_server = $anvil->Words->clean_spaces({ string => $ntp_server });
			$new_ntp->{$ntp_server} = 0;
		}
		
		# Call chrony to see what servers are setup already.
		my $change     = 0;
		my $new_config = "";
		my $old_config = $anvil->Storage->read_file({file => $anvil->data->{path}{data}{'chrony.conf'}});
		foreach my $line (split/\n/, $old_config)
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
			$new_config .= $line."\n";
			if ($line =~ /^Server (.*)$/)
			{
				# This is a source.
				my $source = $1;
				$new_ntp->{$source} = 1;
			}
		}
		
		# Are any sources not enabled?
		foreach my $source (sort {$a cmp $b} keys %{$new_ntp})
		{
			if (not $new_ntp->{$source})
			{
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0090,!!server!".$source."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0090", variables => { server => $source }});
				$change = 1;
				$new_config .= "Server ".$source."\n";
			}
		}
		
		if ($change)
		{
			# Write out the updated file.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0055,!!file!".$anvil->data->{path}{data}{'chrony.conf'}."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0055", variables => { file => $anvil->data->{path}{data}{'chrony.conf'} }});
			$anvil->Storage->write_file({
				debug     => 2,
				file      => $anvil->data->{path}{data}{'chrony.conf'}, 
				body      => $new_config, 
				user      => "root", 
				group     => "root", 
				mode      => "0644", 
				overwrite => 1,
			});
			
			# Restart the daemon.
			my $daemon = "chronyd";
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0091,!!daemon!".$daemon."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0091", variables => { daemon => $daemon }});
			$anvil->System->restart_daemon({
				debug  => 2,
				daemon => $daemon,
			})
		}
	}
	
	# Make sure the IPs of the dashboard(s) and our peers are in the hosts file.
	
	
	### TODO: Do we really need passwordless SSH anymore?
	# Configure SSH by adding ours and our peer's SSH keys to ~/.ssh/known_hosts
	$anvil->System->check_ssh_keys({debug => 3});
	
	# Update the hosts file.
	$anvil->System->update_hosts({debug => 3});
	
	# Setup IPMI, if needed.
	### TODO: Do this when on real hardware
	
	return(0);
}

# Load the job details.
sub load_job
{
	my ($anvil) = @_;
	
	# See if we can find the job details. This method checks for the 'job-uuid' switch if it was used.
	$anvil->data->{switches}{'job-uuid'} = "" if not exists $anvil->data->{switches}{'job-uuid'};
	$anvil->data->{jobs}{job_uuid}       = "" if not exists $anvil->data->{jobs}{job_uuid};
	
	$anvil->Job->get_job_details({debug => 2});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		"jobs::job_uuid" => $anvil->data->{jobs}{job_uuid},
	}});
	
	$anvil->data->{job}{progress} = 0;
	update_progress($anvil, $anvil->data->{job}{progress}, "clear");
	update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0074,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0074", variables => { 'job-uuid' => $anvil->data->{switches}{'job-uuid'} }});
	
	my ($machine, $manifest_uuid, $anvil_uuid) = ($anvil->data->{jobs}{job_data} =~ /as_machine=(.*?),manifest_uuid=(.*?),anvil_uuid=(.*?)$/);
	$machine       = "" if not defined $machine;
	$manifest_uuid = "" if not defined $manifest_uuid;
	$anvil_uuid    = "" if not defined $anvil_uuid;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		machine       => $machine,
		manifest_uuid => $manifest_uuid, 
		anvil_uuid    => $anvil_uuid, 
	}});
	
	if ((not $machine)       or 
	    (not $manifest_uuid) or 
	    (not $anvil_uuid))
	{
		# Terminate the job entirely, it's likely an unrecoverable problem.
		update_progress($anvil, 100, "job_0092,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!,!!raw!".$anvil->data->{jobs}{job_data}."!!");
		sleep 2;
		$anvil->nice_exit({exit_code => 5});
	}
	
	$anvil->data->{sys}{machine}       = $machine;
	$anvil->data->{sys}{manifest_uuid} = $manifest_uuid;
	$anvil->data->{sys}{anvil_uuid}    = $anvil_uuid;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"sys::machine"         => $anvil->data->{sys}{machine},
		"sys::manifest_uuid"   => $anvil->data->{sys}{manifest_uuid},
		"sys::anvil_uuid"      => $anvil->data->{sys}{anvil_uuid},
	}});
	
	# Load in the manifest 
	load_manifest($anvil);
	
	$anvil->Database->get_anvils();
	
	# Load the manifest and anvil data.
	if ((not exists $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}) or (not exists $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}))
	{
		# Terminate the job entirely, it's likely an unrecoverable problem.
		update_progress($anvil, 100, "job_0092,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!,!!raw!".$anvil->data->{jobs}{job_data}."!!");
		sleep 2;
		$anvil->nice_exit({exit_code => 5});
	}
	
	# Load the anvil
	$anvil->data->{sys}{node1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	$anvil->data->{sys}{node2_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	$anvil->data->{sys}{dr1_host_uuid}   = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"sys::node1_host_uuid" => $anvil->data->{sys}{node1_host_uuid},
		"sys::node2_host_uuid" => $anvil->data->{sys}{node2_host_uuid},
		"sys::dr1_host_uuid"   => $anvil->data->{sys}{dr1_host_uuid},
	}});
	update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0075,!!machine!".$anvil->data->{sys}{machine}."!!,!!manifest_uuid!".$anvil->data->{sys}{manifest_uuid}."!!");
	
	return(0);
}

# Load in the manifest 
sub load_manifest
{
	my ($anvil) = @_;
	
	$anvil->Database->get_hosts();
	my $problem = $anvil->Striker->load_manifest({
		debug         => 2,
		manifest_uuid => $anvil->data->{sys}{manifest_uuid}, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		manifest_uuid => $anvil->data->{sys}{manifest_uuid},
		problem       => $problem, 
	}});
	
	if ($problem)
	{
		# Something went wrong
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "warning_0046", variables => { uuid => $anvil->data->{sys}{manifest_uuid} }});
		update_progress($anvil, 0, "job_0076");
		sleep 2;
		$anvil->nice_exit({exit_code => 2});
	}
	
	return(0);
}

# If this is being called as a job, this will allow the progress to be updated.
sub update_progress
{
	my ($anvil, $progress, $message) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		's1:progress' => $progress,
		's2:message'  => $message, 
	}});
	
	# Disabled for the moment
	return(0);

	$progress = 95 if $progress > 100;
	if (not $anvil->data->{switches}{'job-uuid'})
	{
		return(0);
	}
	
	$anvil->Job->update_progress({
		debug    => 3,
		progress => $progress, 
		message  => $message,
		job_uuid => $anvil->data->{switches}{'job-uuid'},
	});
	
	return(0);
}