anvil/tools/anvil-join-anvil

#!/usr/bin/perl
# 
# This adds a node (or gets a node to join to a new/rebuilt peer or added DR) to an Anvil!.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Failed to connect to any database.
# 2 = Failed to load/parse the manifest.
# 3 = Failed to change the host name.
# 4 = Failed to reconnect to any database after the network was reconfigured
# 
# TODO: 
#

use strict;
use warnings;
use Anvil::Tools;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});

# Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is 
# passed directly, it will be used. Otherwise, the password will be read from the database.
$anvil->Get->switches;

$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0129"});
	sleep 10;
	$anvil->nice_exit({exit_code => 1});
}

# Get the job details
load_job($anvil);

# Load in the manifest 
load_manifest($anvil);

# Check if we need to change any IPs or our hostname.
check_local_network($anvil);


$anvil->nice_exit({code => 0});

#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# Check if we need to change any IPs or our hostname.
sub check_local_network
{
	my ($anvil) = @_;
	
	# What host name and IP(s) should I have?
	my $machine       = $anvil->data->{sys}{machine};
	my $manifest_uuid = $anvil->data->{sys}{manifest_uuid};
	
	my $anvil_name    = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{name};
	my $domain        = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{domain};
	my $old_host_name = $anvil->_host_name;
	my $new_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{name};
	if ($domain)
	{
		$new_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{name}.".".$domain;
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:anvil_name'    => $anvil_name, 
		's2:domain'        => $domain, 
		's3:old_host_name' => $old_host_name, 
		's4:new_host_name' => $new_host_name, 
	}});
	
	# If the hostname isn't the same, change it.
	if ($old_host_name ne $new_host_name)
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0061,!!host_name!".$new_host_name."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0061", variables => { host_name => $new_host_name }});
		my ($now_host_name) = $anvil->System->host_name({
			debug => 2,
			set   => $new_host_name,
			
		});
		if ($now_host_name eq $new_host_name)
		{
			# Success!
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0045");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0045"});
		}
		else
		{
			# Failed
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0063", variables => {
				host_name         => $new_host_name, 
				current_host_name => $now_host_name, 
			}});
			update_progress($anvil, 0, "job_0063,!!host_name!".$new_host_name."!!,!!current_host_name!".$now_host_name."!!");
			sleep 10;
			$anvil->nice_exit({code => 3});
		}
	}
	else
	{
		# No need to change
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0077,!!host_name!".$new_host_name."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0077", variables => { host_name => $new_host_name }});
	}
	
	# Read the local network manager data.
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0080");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0080"});
	$anvil->Network->read_nmcli({debug => 3});
	
	# Now check IP addresses.
	$anvil->Network->get_ips({debug => 3});
	
	# This will be set when the first IFN with a gateway is set.
	my $default_gateway_interface = "";
	foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{'local'}{interface}})
	{
		if ($anvil->data->{network}{'local'}{interface}{$in_iface}{default_gateway})
		{
			$default_gateway_interface = $in_iface;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_gateway_interface => $default_gateway_interface }});
			last;
		}
	}
	
	my $dns = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{dns};
	my $mtu = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{mtu};
	my $ntp = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{ntp};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		dns => $dns,
		mtu => $mtu,
		ntp => $ntp,
	}});
	
	# The DNS are comma-separated lists, that may or may not have spaces and may or may not be in 
	# alphabetical order. To properly compare, we'll rebuild the CSV string of the current and desired 
	# DNS settings.
	my $cleaned_dns = "";
	my @dns_array   = split/,/, $dns;
	foreach my $this_dns (sort {$a cmp $b} @dns_array)
	{
		$this_dns    =  $anvil->Words->clean_spaces({ string => $this_dns });
		$cleaned_dns .= $this_dns.",";
	}
	$cleaned_dns =~ s/,$//;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cleaned_dns => $cleaned_dns }});
	
	# If any network interface is changed, we'll write out the config file then, when done, disconnect 
	# from the database, restart networking and then reconnect before moving on.
	my $restart_interfaces = [];
	
	foreach my $network (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}})
	{
		my $ip      = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}{$network}{ip};
		my $subnet  = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network}{subnet};
		my $cidr    = $anvil->Convert->cidr({subnet_mask => $subnet});
		my $gateway = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network}{gateway};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:network' => $network, 
			's2:ip'      => $ip,
			's3:subnet'  => $subnet, 
			's4:cidr'    => $cidr, 
			's5:gateway' => $gateway, 
		}});
		foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{'local'}{interface}})
		{
			# Only one interface will start with the network name and have an IP address.
			next if $in_iface !~ /^${network}_/;
			next if not $anvil->data->{network}{'local'}{interface}{$in_iface}{ip};
			my $current_ip      = $anvil->data->{network}{'local'}{interface}{$in_iface}{ip};
			my $current_subnet  = $anvil->data->{network}{'local'}{interface}{$in_iface}{subnet_mask};
			my $current_gateway = $anvil->data->{network}{'local'}{interface}{$in_iface}{gateway};
			my $current_dns     = $anvil->data->{network}{'local'}{interface}{$in_iface}{dns};
			my $current_mtu     = $anvil->data->{network}{'local'}{interface}{$in_iface}{mtu};
			my $mac_address     = $anvil->data->{network}{'local'}{interface}{$in_iface}{mac_address};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:in_iface'        => $in_iface,
				's2:current_ip'      => $current_ip,
				's4:current_subnet'  => $current_subnet, 
				's5:current_gateway' => $current_gateway, 
				's6:current_dns'     => $current_dns, 
				's7:current_mtu'     => $current_mtu, 
			}});
			
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0081,!!name!".$in_iface."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0081", variables => { name => $in_iface }});
			
			if ((not $default_gateway_interface) && ($in_iface =~ /^ifn/) && ($gateway))
			{
				# No existing default gateway, but this is the first IFN we've seen with a 
				# gateway defined, so we'll use this one.
				$default_gateway_interface = $in_iface;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_gateway_interface => $default_gateway_interface }});
			}
			
			my $cleaned_current_dns = "";
			my @current_dns_array   = split/,/, $current_dns;
			foreach my $this_current_dns (sort {$a cmp $b} @current_dns_array)
			{
				$this_current_dns    =  $anvil->Words->clean_spaces({ string => $this_current_dns });
				$cleaned_current_dns .= $this_current_dns.",";
			}
			$cleaned_current_dns =~ s/,$//;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cleaned_current_dns => $cleaned_current_dns }});
			
			my $change = 0;
			if (($current_ip ne $ip) or ($current_subnet ne $subnet))
			{
				# IP / subnet changed.
				print "IP: .... [".$current_ip."] -> [".$ip."]\n";
				print "Subnet:  [".$current_subnet."] -> [".$subnet."]\n";
				$change = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
			}
			
			# gateway?
			if ($current_gateway ne $gateway)
			{
				print "Gateway: [".$current_gateway."] -> [".$gateway."]\n";
				$change = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
			}
			
			# Check DNS only if gateway is set.
			if ($gateway)
			{
				if ($cleaned_dns ne $cleaned_current_dns)
				{
					print "DNS: ... [".$cleaned_current_dns."] -> [".$cleaned_dns."]\n";
					$change = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
				}
				elsif (($in_iface eq $default_gateway_interface) && (not $anvil->data->{network}{'local'}{interface}{$in_iface}{default_gateway}))
				{
					# This isn't the default gateway yet, but we'll make it so.
					print "Will set as default gateway\n";
					$change = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
				}
			}
			
			if (not $change)
			{
				# No change
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0082");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0082"});
			}
			else
			{
				# Update the config. 
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0078,!!interface!".$in_iface."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0078", variables => { interface => $in_iface }});
				
				my $interface_uuid = $anvil->data->{nmcli}{'local'}{device_to_uuid}{$in_iface};
				my $filename       = $anvil->data->{nmcli}{'local'}{uuid}{$interface_uuid}{filename};
				my $interface_name = $anvil->data->{nmcli}{'local'}{uuid}{$interface_uuid}{name};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:in_iface'       => $in_iface, 
					's2:interface_name' => $interface_name, 
					's3:filename'       => $filename, 
					's4:interface_uuid' => $interface_uuid, 
				}});
				
				# Record the name to restart
				push @{$restart_interfaces}, $interface_name;
				
				# Read in the file. We'll update the date stamp and change the variables as 
				# needed and write it back out.
				my $ip_seen         = 0;
				my $subnet_seen     = 0;
				my $gateway_seen    = 0;
				my $dns_seen        = 0;
				my $mtu_seen        = 0;
				my $boot_proto_seen = 0;
				my $defroute_seen   = 0;
				my $say_default     = $default_gateway_interface eq $in_iface ? "YES" : "NO";
				my $new_config      = "";
				my $old_config      = $anvil->Storage->read_file({file => $filename});
				foreach my $line (split/\n/, $old_config)
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
					if ($line =~ /^BOOTPROTO="(.*?)"/)
					{
						my $boot_protocol   =  $1;
						   $boot_proto_seen =  1;
						   $new_config      .= "BOOTPROTO=\"none\"\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							's1:boot_proto_seen' => $boot_proto_seen, 
							's2:boot_protocol'   => $boot_protocol,
							's3:new_config'      => $new_config,
						}});
						if ($boot_protocol eq "dhcp")
						{
							# Inject the network config.
							if (not $ip_seen)
							{
								$ip_seen    =  1;
								$new_config .= "IPADDR=\"".$ip."\"\n";
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:ip_seen'    => $boot_proto_seen, 
									's2:new_config' => $new_config,
								}});
							}
							if (not $subnet_seen)
							{
								if ($cidr)
								{
									$new_config .= "PREFIX=\"".$cidr."\"\n";
								}
								else
								{
									$new_config .= "NETMASK=\"".$subnet."\"\n";
								}
								$subnet_seen = 1;
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:subnet_seen' => $subnet_seen, 
									's2:new_config'  => $new_config,
								}});
							}
							if (($gateway) && (not $gateway_seen))
							{
								$gateway_seen =  1;
								$new_config   .= "GATEWAY=\"".$gateway."\"\n";
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:gateway_seen' => $gateway_seen, 
									's2:new_config'   => $new_config,
								}});
							}
							if (not $defroute_seen)
							{
								$defroute_seen =  1;
								$new_config    .= "DEFROUTE=\"".$say_default."\"\n";
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:defroute_seen' => $defroute_seen, 
									's2:new_config'    => $new_config,
								}});
							}
							if (($default_gateway_interface eq $in_iface) && ($cleaned_dns))
							{
								if (not $dns_seen)
								{
									my $i = 1;
									foreach my $this_dns (split/,/, $cleaned_dns)
									{
										$new_config .= "DNS".$i."=\"".$this_dns."\"\n";
										$i++;
									}

									$dns_seen = 1;
									$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
										's1:dns_seen'   => $dns_seen, 
										's2:new_config' => $new_config,
									}});
								}
							}
						}
					}
					elsif ($line =~ /^IPADDR=".*?"/)
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ip_seen => $ip_seen }});
						if (not $ip_seen)
						{
							$ip_seen    =  1;
							$new_config .= "IPADDR=\"".$ip."\"\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:ip_seen'    => $boot_proto_seen, 
								's2:new_config' => $new_config,
							}});
						}
					}
					elsif (($line =~ /^PREFIX=".*?"/) or ($line =~ /^NETMASK=".*?"/))
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { subnet_seen => $subnet_seen }});
						if (not $subnet_seen)
						{
							$subnet_seen = 1;
							if ($cidr)
							{
								$new_config .= "PREFIX=\"".$cidr."\"\n";
							}
							else
							{
								$new_config .= "NETMASK=\"".$subnet."\"\n";
							}
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:subnet_seen' => $subnet_seen, 
								's2:new_config'  => $new_config,
							}});
						}
					}
					elsif ($line =~ /^GATEWAY=".*"/)
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { gateway_seen => $gateway_seen }});
						if (not $gateway_seen)
						{
							$gateway_seen =  1;
							$new_config   .= "GATEWAY=\"".$gateway."\"\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:gateway_seen' => $gateway_seen, 
								's2:new_config'   => $new_config,
							}});
						}
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { gateway_seen => $gateway_seen }});
					}
					elsif ($line =~ /^DNS=".*"/)
					{
						# The first time we see a DNS entry, we inject the DNS we've 
						# got and then ignore the rest.
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dns_seen => $dns_seen }});
						if (not $dns_seen)
						{
							# If there's no DNS, this will do nothing.
							my $i = 1;
							foreach my $this_dns (split/,/, $cleaned_dns)
							{
								$new_config .= "DNS".$i."=\"".$this_dns."\"\n";
								$i++;
							}

							$dns_seen = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:dns_seen'   => $dns_seen, 
								's2:new_config' => $new_config,
							}});
						}
					}
					elsif ($line =~ /^MTU=".*"/)
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mtu_seen => $mtu_seen }});
						if (not $mtu_seen)
						{
							$mtu_seen =  1;
							$new_config   .= "MTU=\"".$mtu."\"\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:mtu_seen'   => $mtu_seen, 
								's2:new_config' => $new_config,
							}});
						}
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mtu_seen => $mtu_seen }});
					}
					elsif ($line =~ /^DEFROUTE=".*"/)
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }});
						if (not $defroute_seen)
						{
							$defroute_seen =  1;
							$new_config   .= "DEFROUTE=\"".$say_default."\"\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:defroute_seen' => $defroute_seen, 
								's2:new_config'    => $new_config,
							}});
						}
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }});
					}
					else
					{
						$new_config .= $line."\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }});
					}
				}
				
				# Add any values I've not yet seen.
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					ip_seen       => $ip_seen, 
					subnet_seen   => $subnet_seen, 
					gateway_seen  => $gateway_seen, 
					dns_seen      => $dns_seen, 
					mtu_seen      => $mtu_seen, 
					defroute_seen => $defroute_seen, 
				}});
				if (not $ip_seen)
				{
					$new_config .= "IPADDR=\"".$ip."\"\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				if (not $subnet_seen)
				{
					if ($cidr)
					{
						$new_config .= "PREFIX=\"".$cidr."\"\n";
					}
					else
					{
						$new_config .= "NETMASK=\"".$subnet."\"\n";
					}
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				if (not $gateway_seen)
				{
					$new_config .= "GATEWAY=\"".$gateway."\"\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				if (not $dns_seen)
				{
					# If there's no DNS, this will do nothing.
					my $i = 1;
					foreach my $this_dns (split/,/, $cleaned_dns)
					{
						$new_config .= "DNS".$i."=\"".$this_dns."\"\n";
						$i++;
					}
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				if (not $mtu_seen)
				{
					$new_config .= "MTU=\"".$mtu."\"\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				if (not $defroute_seen)
				{
					$new_config .= "DEFROUTE=\"".$say_default."\"\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
				
				# Write out the new file.
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0083,!!file!".$filename."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0083", variables => { file => $filename }});
				$anvil->Storage->write_file({
					debug     => 2,
					file      => $filename, 
					body      => $new_config, 
					user      => "root", 
					group     => "root", 
					mode      => "0644", 
					overwrite => 1,
				});
			}
			last;
		}
	}
	
	# If there are any entries in '$restart_interfaces', restart
	my $restart_interface_count = @{$restart_interfaces};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { restart_interface_count => $restart_interface_count }});
	if ($restart_interface_count)
	{
		# Disconnect from the database, as we're about to tear down our connection.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, secure => 0, key => "job_0079"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0079");
		$anvil->Database->disconnect();
		
		# Tell nmcli to re-read the config files.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0463"});
		my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{nmcli}." connection reload"});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
		
		foreach my $interface_name (sort {$a cmp $b} @{$restart_interfaces})
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0462", variables => { interface => $interface_name }});
			$anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{nmcli}." connection down \"".$interface_name."\""});
			
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0464", variables => { interface => $interface_name }});
			$anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{nmcli}." connection up \"".$interface_name."\""});
		}
	
		# Wait for a DB connection. We'll wait up to 130 seconds (updelay is 120 seconds, plus a small buffer).
		my $wait_until = time + 130;
		until ($anvil->data->{sys}{database}{connections})
		{
			$anvil->refresh();
			$anvil->Database->connect();
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, secure => 0, key => "log_0132"});
			if (not $anvil->data->{sys}{database}{connections})
			{
				if (time > $wait_until)
				{
					# Failed to reconnect, exit. 
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0107"});
					$anvil->nice_exit({code => 4});
				}
				
				# No databases, sleep and then try again.
				sleep 2;
			}
		}
		
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, secure => 0, key => "job_0084"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0084");
	}
	
	# Remove virbr0 if it exists.
	if (exists $anvil->data->{network}{'local'}{interface}{virbr0})
	{
		# Remove the NAT'ed bridge
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, secure => 0, key => "job_0085"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0085");
		
		$anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{virsh}." net-destroy default"});
		$anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{virsh}." net-undefine default "});
		
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, secure => 0, key => "job_0034"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0034");
	}
	
	# Update network view
	$anvil->Network->read_nmcli({debug => 2});
	$anvil->Network->get_ips({debug => 3});
	
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, secure => 0, key => "job_0086"});
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0086");
	
	# Update MTUs (running interface and config) if needed.
	foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{'local'}{interface}})
	{
		# Only one interface will start with the network name and have an IP address.
		my $current_mtu = $anvil->data->{network}{'local'}{interface}{$in_iface}{mtu};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:in_iface'    => $in_iface, 
			's2:current_mtu' => $current_mtu,
		}});
		
		if ($current_mtu eq $mtu)
		{
			# It's fine
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0087,!!interface!".$in_iface."!!,!!mtu!".$mtu."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0087", variables => { 
				interface => $in_iface,
				mtu       => $mtu,
			}});
		}
		else
		{
			# Change the MTU both on the running interface and in the config file.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0088,!!interface!".$in_iface."!!,!!old_mtu!".$current_mtu."!!,!!mtu!".$mtu."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0088", variables => { 
				interface => $in_iface, 
				old_mtu   => $current_mtu, 
				mtu       => $mtu, 
			}});
			
			# Change the live MTU.
			$anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{ip}." link set ".$in_iface." mtu ".$mtu});
			
			# Now update the config file.
			my $interface_uuid = $anvil->data->{nmcli}{'local'}{device_to_uuid}{$in_iface};
			my $filename       = $anvil->data->{nmcli}{'local'}{uuid}{$interface_uuid}{filename};
			my $interface_name = $anvil->data->{nmcli}{'local'}{uuid}{$interface_uuid}{name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:interface_name' => $interface_name, 
				's2:filename'       => $filename, 
				's3:interface_uuid' => $interface_uuid, 
			}});
			
			my $mtu_seen   = 0;
			my $new_config = "";
			my $old_config = $anvil->Storage->read_file({file => $filename});
			foreach my $line (split/\n/, $old_config)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
				if ($line =~ /^MTU=".*?"/)
				{
					$mtu_seen   =  1;
					$new_config .= "MTU=\"".$mtu."\"\n";
				}
				else
				{
					$new_config .= $line."\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
			}
			if (not $mtu_seen)
			{
				# Inject the MTU variable
				$new_config .= "MTU=\"".$mtu."\"\n";
			}
			
			# Write out the new file.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0083,!!file!".$filename."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0083", variables => { file => $filename }});
			$anvil->Storage->write_file({
				debug     => 2,
				file      => $filename, 
				body      => $new_config, 
				user      => "root", 
				group     => "root", 
				mode      => "0644", 
				overwrite => 1,
			});
			
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, secure => 0, key => "job_0034"});
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0034");
		}
	}
	
	# Update NTP if set and needed.
	if ($ntp)
	{
		# Break up the NTP servers into a list, we'll set to '1' the ones we find.
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0089");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0089"});
		my $new_ntp = {};
		foreach my $ntp_server (split/,/, $ntp)
		{
			$ntp_server = $anvil->Words->clean_spaces({ string => $ntp_server });
			$new_ntp->{$ntp_server} = 0;
		}
		
		# Call chrony to see what servers are setup already.
		my $change     = 0;
		my $new_config = "";
		my $old_config = $anvil->Storage->read_file({file => $anvil->data->{path}{data}{'chrony.conf'}});
		foreach my $line (split/\n/, $old_config)
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, list => { line => $line }});
			$new_config .= $line."\n";
			if ($line =~ /^Server (.*)$/)
			{
				# This is a source.
				my $source = $1;
				$new_ntp->{$source} = 1;
			}
		}
		
		# Are any sources not enabled?
		foreach my $source (sort {$a cmp $b} keys %{$new_ntp})
		{
			if (not $new_ntp->{$source})
			{
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0090,!!server!".$source."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0090", variables => { server => $source }});
				$change = 1;
				$new_config .= "Server ".$source."\n";
			}
		}
		
		if ($change)
		{
			# Write out the updated file.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0055,!!file!".$anvil->data->{path}{data}{'chrony.conf'}."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0055", variables => { file => $anvil->data->{path}{data}{'chrony.conf'} }});
			$anvil->Storage->write_file({
				debug     => 2,
				file      => $anvil->data->{path}{data}{'chrony.conf'}, 
				body      => $new_config, 
				user      => "root", 
				group     => "root", 
				mode      => "0644", 
				overwrite => 1,
			});
			
			# Restart the daemon.
			my $daemon = "chronyd";
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0091,!!daemon!".$daemon."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0091", variables => { daemon => $daemon }});
			$anvil->System->restart_daemon({
				debug  => 2,
				daemon => $daemon,
			})
		}
	}
	
	# Setup IPMI, if needed.
=cut
909; xxx::upses::el8-ups01::uuid: [7ebecdda-782d-4624-841d-98d912ed3d50]
909; xxx::upses::el8-ups02::uuid: [7ffb4dc2-8b96-4ca7-80bb-49e309fb2f5f]
918; xxx::fences::an-nas02::uuid: [4117a862-f58f-4676-991a-9ca257a3c612]

949; xxx::networks::name::bcn1::gateway: [], xxx::networks::name::bcn1::network: [10.201.0.0], xxx::networks::name::bcn1::subnet: [255.255.0.0]
949; xxx::networks::name::sn1::gateway: [], xxx::networks::name::sn1::network: [10.101.0.0], xxx::networks::name::sn1::subnet: [255.255.0.0]
949; xxx::networks::name::ifn1::gateway: [10.255.255.254], xxx::networks::name::ifn1::network: [10.255.0.0], xxx::networks::name::ifn1::subnet: [255.255.0.0]
961; xxx::networks::count::bcn: [1], xxx::networks::count::ifn: [1], xxx::networks::count::sn: [1]

971; xxx::machine::node1::ipmi_ip: [], xxx::machine::node1::type: [!!undef!!]
980; xxx::machine::node1::fence::an-nas02::port: [el8-a01n01]
980; xxx::machine::node1::fence::el8-pdu01::port: []
980; xxx::machine::node1::fence::el8-pdu02::port: []
980; xxx::machine::node1::fence::pulsar::port: []
989; xxx::machine::node1::ups::el8-ups01::used: [0]
989; xxx::machine::node1::ups::el8-ups02::used: [0]
998; xxx::machine::node1::network::bcn1::ip: [10.201.10.1]
998; xxx::machine::node1::network::ifn1::ip: [10.255.10.1]
998; xxx::machine::node1::network::sn1::ip: [10.101.10.1]
=cut
	
	return(0);
}

# Load the job details.
sub load_job
{
	my ($anvil) = @_;
	
	# See if we can find the job details. This method checks for the 'job-uuid' switch if it was used.
	$anvil->data->{switches}{'job-uuid'} = "" if not exists $anvil->data->{switches}{'job-uuid'};
	$anvil->data->{jobs}{job_uuid}       = "" if not exists $anvil->data->{jobs}{job_uuid};
	
	$anvil->Job->get_job_details({debug => 2});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		"jobs::job_uuid" => $anvil->data->{jobs}{job_uuid},
	}});
	
	$anvil->data->{job}{progress} = 0;
	update_progress($anvil, $anvil->data->{job}{progress}, "clear");
	update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0074,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0074", variables => { 'job-uuid' => $anvil->data->{switches}{'job-uuid'} }});
	
	my ($machine, $manifest_uuid) = ($anvil->data->{jobs}{job_data} =~ /as_machine=(.*?),manifest_uuid=(.*)$/);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		machine       => $machine,
		manifest_uuid => $manifest_uuid, 
	}});
	
	$anvil->data->{sys}{machine}       = $machine;
	$anvil->data->{sys}{manifest_uuid} = $manifest_uuid;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"sys::machine"       => $anvil->data->{sys}{machine},
		"sys::manifest_uuid" => $anvil->data->{sys}{manifest_uuid},
	}});
	update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0075,!!machine!".$anvil->data->{sys}{machine}."!!,!!manifest_uuid!".$anvil->data->{sys}{manifest_uuid}."!!");
	
	return(0);
}

# Load in the manifest 
sub load_manifest
{
	my ($anvil) = @_;
	
	$anvil->Database->get_hosts();
	my $problem = $anvil->Striker->load_manifest({
		debug         => 2,
		manifest_uuid => $anvil->data->{sys}{manifest_uuid}, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		manifest_uuid => $anvil->data->{sys}{manifest_uuid},
		problem       => $problem, 
	}});
	
	if ($problem)
	{
		# Report a problem and send the user back to the manifests page.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "warning_0046"});
		update_progress($anvil, 0, "job_0076");
		sleep 10;
		$anvil->nice_exit({exit_code => 2});
	}
	
	return(0);
}

# If this is being called as a job, this will allow the progress to be updated.
sub update_progress
{
	my ($anvil, $progress, $message) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		's1:progress' => $progress,
		's2:message'  => $message, 
	}});

	$progress = 95 if $progress > 100;
	if (not $anvil->data->{switches}{'job-uuid'})
	{
		return(0);
	}
	
	$anvil->Job->update_progress({
		debug    => 3,
		progress => $progress, 
		message  => $message,
		job_uuid => $anvil->data->{switches}{'job-uuid'},
	});
	
	return(0);
}