anvil/tools/anvil-configure-host

#!/usr/bin/perl
# 
# This is called when striker, an node or a DR host needs to configure the local network and user accounts.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = The program is not running as root.
# 2 = Failed to connect to database(s).
# 3 = Job was already picked up by another running instance.
# 4 = The host name did not update properly.
# 5 = Failed to write the temp file with the new password needed to call anvil-change-password.
# 6 = The job-uuid was not found.
# 

use strict;
use warnings;
use Data::Dumper;
use Anvil::Tools;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});

# Read switches
$anvil->data->{switches}{'no-reboot'} = 0;
$anvil->Get->switches;

# Paths
$anvil->Storage->read_config({file => $anvil->data->{path}{configs}{'anvil.conf'}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});

# Make sure we're running as 'root'
# $< == real UID, $> == effective UID
if (($< != 0) && ($> != 0))
{
	# Not root
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "error_0005"});
	$anvil->nice_exit({code => 1});
}

# Connect
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0031"});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "error_0003"});
	$anvil->nice_exit({exit_code => 2});
}

# Set maintenance mode
$anvil->System->maintenance_mode({set => 1});

pickup_job_details($anvil);

my ($reboot_needed) = reconfigure_network($anvil);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, list => { reboot_needed => $reboot_needed }});

# Record that we've configured this machine.
$anvil->Database->insert_or_update_variables({
	variable_name         => "system::configured", 
	variable_value        => 1, 
	variable_default      => "", 
	variable_description  => "striker_0048", 
	variable_section      => "system", 
	variable_source_uuid  => $anvil->Get->host_uuid, 
	variable_source_table => "hosts", 
});

update_passwords($anvil);
$anvil->Job->update_progress({
	debug    => 3, 
	progress => 100, 
	message  => $anvil->data->{switches}{'no-reboot'} ? "message_0065" : "",
	job_uuid => $anvil->data->{job}{uuid},
});

# Clear maintenance mode.
$anvil->System->maintenance_mode({set => 0});

### TODO: This is only until we can get the damn networking stable on reconfigure.
# Set reboot needed so that things clean up properly on reboot.
if ($reboot_needed)
{
	$anvil->System->reboot_needed({set => 1});
	if (not $anvil->data->{switches}{'no-reboot'})
	{
		# Reboot, after waiting a few seconds to let the user's browser pick up the last messages in 
		# jobs.json. We'll also log the user out, in case we were re-configuring.
		sleep 5;
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0215"});
		$anvil->Account->logout({debug => 2});
		$anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{'anvil-manage-power'}." --reboot -y"});
	}
}
$anvil->nice_exit({code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This updates the passwords on the root user, admin user, database and apache user.
sub update_passwords
{
	my ($anvil) = @_;
	
	# Is this a striker dashboard, node or DR host?
	my $type = $anvil->System->get_host_type();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, list => { type => $type }});
	
	if ($type eq "dashboard")
	{
		# Set the passwords
		my $password  = $anvil->data->{variables}{form}{config_step2}{striker_password}{value};
		my $temp_file = "/tmp/anvil-".$anvil->Get->uuid;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }});
		
		# Write the password into a temporary file.
		my $error = $anvil->Storage->write_file({
			body      => $password,
			file      => $temp_file,
			group     => "root", 
			mode      => "0600",
			overwrite => 1,
			secure    => 1,
			user      => "root",
		});
		
		# Call anvil-change-password
		if ($error)
		{
			# Couldn't write the temp file.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "message_0030", variables => { file => $temp_file }});
			$anvil->nice_exit({code => 5});
		}
		else
		{
			my $return_code            = "";
			my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{'anvil-change-password'}." -y --password-file ".$temp_file });
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, list => { output => $output, return_code => $return_code }});
			foreach my $line (split/\n/, $output)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, list => { line => $line }});
			}
			
			# Unlink the temp file.
			unlink $temp_file;
			
			if ($return_code)
			{
				# Something went wrong
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "error_0011", variables => { return_code => $return_code }});
			}
		}
	}
	else
	{
		if ($type eq "node")
		{
			# We'll need to update pcsd's password as well.
		}
		
		# Set passwords that are common to nodes and DR hosts. 
		
		
	}
	
	$anvil->Job->update_progress({
		progress => 95, 
		job_uuid => $anvil->data->{job}{uuid},
	});
	
	return(0);
}

# This does the work of reconfiguring the network
sub reconfigure_network
{
	my ($anvil) = @_;
	
	my $reboot_needed   = 0;
	my $prefix          = $anvil->data->{variables}{form}{config_step1}{prefix}{value};
	my $sequence        = $anvil->data->{variables}{form}{config_step1}{sequence}{value};
	my $domain          = $anvil->data->{variables}{form}{config_step1}{domain}{value};
	my $organization    = $anvil->data->{variables}{form}{config_step1}{organization}{value};
	my $bcn_count       = 1;	# TODO: This should be coming from the form, even though it's only '1' for now.
	my $sn_count        = 0;	# TODO: This should be coming from the form, even though it's always '0' for Strikers.
	my $ifn_count       = $anvil->data->{variables}{form}{config_step1}{ifn_count}{value};
	my $new_hostname    = defined $anvil->data->{variables}{form}{config_step2}{hostname}{value} ? $anvil->data->{variables}{form}{config_step2}{hostname}{value} : $prefix."-striker".sprintf("%02d", $sequence).".".$domain;
	my $pretty_hostname = $organization." - Striker ".sprintf("%02d", $sequence);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, list => { 
		prefix          => $prefix, 
		sequence        => $sequence,
		domain          => $domain, 
		organization    => $organization, 
		bcn_count       => $bcn_count, 
		ifn_count       => $ifn_count, 
		new_hostname    => $new_hostname, 
		pretty_hostname => $pretty_hostname, 
	}});
	
	# Set the hostname
	my ($hostname, $descriptive_hostname) = $anvil->System->hostname({set => $new_hostname, pretty => $pretty_hostname, debug => 3});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		hostname             => $hostname,
		descriptive_hostname => $descriptive_hostname, 
	}});
	if ($hostname eq $new_hostname)
	{
		# Success
		$anvil->Job->update_progress({
			progress => 10, 
			message  => "message_0016,!!hostname!$new_hostname!!", 
			job_uuid => $anvil->data->{job}{uuid},
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0016", variables => { hostname => $new_hostname }});
	}
	else
	{
		# Failed
		$anvil->Job->update_progress({
			progress => 0, 
			message  => "message_0017,!!hostname!$new_hostname!!,!!bad_hostname!$hostname!!", 
			job_uuid => $anvil->data->{job}{uuid},
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "message_0017", variables => { 
			hostname     => $new_hostname,
			bad_hostname => $hostname,
		}});
		$anvil->nice_exit({code => 4});
	}
	
	# Get the current list of IPs and MAC addresses.
	$anvil->System->get_ips();
	
	# Now configure the network.
	my $dns = defined $anvil->data->{variables}{form}{config_step2}{dns}{value} ? [split/,/, $anvil->data->{variables}{form}{config_step2}{dns}{value}] : [];
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { dns => $dns }});
	for (my $i = 0; $i < @{$dns}; $i++)
	{
		$dns->[$i] = $anvil->Words->clean_spaces({ string => $dns->[$i] });
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "dns->[$i]" => $dns->[$i] }});
	}
	
	my $gateway           = defined $anvil->data->{variables}{form}{config_step2}{gateway}{value}           ? $anvil->data->{variables}{form}{config_step2}{gateway}{value}           : "";
	my $gateway_interface = defined $anvil->data->{variables}{form}{config_step2}{gateway_interface}{value} ? $anvil->data->{variables}{form}{config_step2}{gateway_interface}{value} : "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		gateway           => $gateway, 
		gateway_interface => $gateway_interface, 
	}});
	foreach my $network_type ("bcn", "sn", "ifn")
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { network_type => $network_type }});
		
		my $count = 0;
		if    ($network_type eq "bcn") { $count = $bcn_count; }
		elsif ($network_type eq "sn")  { $count = $sn_count;  }
		elsif ($network_type eq "ifn") { $count = $ifn_count; }
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { count => $count }});
		
		next if not $count;
		foreach my $network_count (1..$count)
		{
			my $this_network    =         $network_type.$network_count;
			my $link1_key       =         $this_network."_link1_mac_to_set";
			my $link2_key       =         $this_network."_link2_mac_to_set";
			my $subnet_key      =         $this_network."_subnet";
			my $ip_key          =         $this_network."_ip";
			my $is_gateway      =         $this_network eq $gateway_interface ? 1 : 0;
			my $link1_mac       =         $anvil->data->{variables}{form}{config_step2}{$link1_key}{value};
			my $link2_mac       = defined $anvil->data->{variables}{form}{config_step2}{$link2_key}{value} ? $anvil->data->{variables}{form}{config_step2}{$link2_key}{value} : "";
			my $old_link1_iface =         $anvil->data->{sys}{mac}{$link1_mac}{iface} ? $anvil->data->{sys}{mac}{$link1_mac}{iface} : "";
			my $old_link2_iface = defined $anvil->data->{sys}{mac}{$link2_mac}{iface} ? $anvil->data->{sys}{mac}{$link2_mac}{iface} : "";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				ip_key          => $ip_key, 
				is_gateway      => $is_gateway, 
				link1_key       => $link1_key, 
				link1_mac       => $link1_mac, 
				link2_key       => $link2_key, 
				link2_mac       => $link2_mac, 
				old_link1_iface => $old_link1_iface, 
				old_link2_iface => $old_link2_iface, 
				subnet_key      => $subnet_key, 
				this_network    => $this_network, 
			}});
			
			# Skip if this doesn't exist or isn't a valid IPv4 address.
			if (not exists $anvil->data->{variables}{form}{config_step2}{$ip_key}{value})
			{
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0176", variables => { ip_key => $ip_key }});
				next;
			}
			else
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "variables::form::config_step2::${ip_key}::value" => $anvil->data->{variables}{form}{config_step2}{$ip_key}{value} }});
			}
			if (($anvil->data->{variables}{form}{config_step2}{$ip_key}{value}) and (not $anvil->Validate->is_ipv4({ip => $anvil->data->{variables}{form}{config_step2}{$ip_key}{value}})))
			{
				# Something was set, but it isn't valid.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "log_0148", variables => { 
					network => $this_network,
					ip      => $anvil->data->{variables}{form}{config_step2}{$ip_key}{value},
				}});
				next;
			}
			
			my $ip     = $anvil->data->{variables}{form}{config_step2}{$ip_key}{value};
			my $subnet = $anvil->data->{variables}{form}{config_step2}{$subnet_key}{value};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				ip     => $ip, 
				subnet => $subnet, 
			}});
			
			# Are we building bonded interfaces?
			if ($anvil->Validate->is_mac({mac => $link2_mac}))
			{
				### TODO: Handle when bridges exist. Detect when the host is a node and/or have a "use as bridge" option?
				# Yup! Build the configs.
				my $say_network      = "";
				my $say_interface    = "";
				my $interface_prefix = "";
				if ($network_type eq "bcn")
				{
					$say_network      = "Back-Channel Network ".$network_count;
					$say_interface    = "bcn".$network_count;
					$interface_prefix = "BCN";
				}
				elsif ($network_type eq "sn")
				{
					$say_network      = "Storage Network ".$network_count; 
					$say_interface    = "sn".$network_count;
					$interface_prefix = "SN";
				}
				elsif ($network_type eq "ifn")
				{
					$say_network      = "Internet-Facing Network ".$network_count; 
					$say_interface    = "ifn".$network_count;
					$interface_prefix = "IFN";
				}
				my $say_defroute    = $is_gateway ? "yes" : "no";
				my $cidr            = $anvil->Convert->cidr({subnet => $subnet});
				my $bond_file       = $anvil->data->{path}{directories}{ifcfg}."/ifcfg-".$interface_prefix."_".$network_count."_-_Bond_1";
				my $new_link1_file  = $anvil->data->{path}{directories}{ifcfg}."/ifcfg-".$interface_prefix."_".$network_count."_-_Link_1";
				my $new_link2_file  = $anvil->data->{path}{directories}{ifcfg}."/ifcfg-".$interface_prefix."_".$network_count."_-_Link_2";
				my $old_link1_file  = $new_link1_file;
				my $old_link2_file  = $new_link2_file;
				my $new_bond_iface  = $say_interface."_bond1";
				my $new_link1_iface = $say_interface."_link1";
				my $new_link2_iface = $say_interface."_link2";
				if ((exists $anvil->data->{sys}{mac}{$link1_mac}{iface}) && ($anvil->data->{sys}{mac}{$link1_mac}{iface}))
				{
					$old_link1_file = $anvil->data->{path}{directories}{ifcfg}."/ifcfg-".$anvil->data->{sys}{mac}{$link1_mac}{iface};
				}
				if ((exists $anvil->data->{sys}{mac}{$link2_mac}{iface}) && ($anvil->data->{sys}{mac}{$link2_mac}{iface}))
				{
					$old_link2_file = $anvil->data->{path}{directories}{ifcfg}."/ifcfg-".$anvil->data->{sys}{mac}{$link2_mac}{iface};
				}
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
					say_defroute    => $say_defroute, 
					cidr            => $cidr, 
					bond_file       => $bond_file, 
					new_link1_file  => $new_link1_file,
					new_link2_file  => $new_link2_file, 
					old_link1_file  => $old_link1_file, 
					old_link2_file  => $old_link2_file, 
					new_bond_iface  => $new_bond_iface, 
					new_link1_iface => $new_link1_iface, 
					new_link2_iface => $new_link2_iface, 
				}});
				
				# Gather (or create) UUIDs
				my $bond_uuid  = get_uuid_from_interface_file($anvil, $bond_file);
				my $link1_uuid = get_uuid_from_interface_file($anvil, $old_link1_file);
				my $link2_uuid = get_uuid_from_interface_file($anvil, $old_link2_file);
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
					bond_uuid  => $bond_uuid, 
					link1_uuid => $link1_uuid, 
					link2_uuid => $link2_uuid, 
				}});
				
				### TODO: Set the firewall Zone appropriately.
				# Build the Bond config.
				my $bond_config =  "# $say_network - Bond 1\n";
				   $bond_config .= "UUID=\"".$bond_uuid."\"\n";
				   $bond_config .= "NAME=\"".$interface_prefix." ".$network_count." - Bond 1\"\n";
				   $bond_config .= "DEVICE=\"".$new_bond_iface."\"\n";
				   $bond_config .= "BONDING_OPTS=\"mode=active-backup primary=".$say_interface."_link1 updelay=120000 downdelay=0 miimon=100 primary_reselect=better\"\n";
				   $bond_config .= "TYPE=\"Bond\"\n";
				   $bond_config .= "BONDING_MASTER=\"yes\"\n";
				   $bond_config .= "BOOTPROTO=\"none\"\n";
				   $bond_config .= "IPV6INIT=\"no\"\n";
				   $bond_config .= "ONBOOT=\"yes\"\n";
				   $bond_config .= "IPADDR=\"".$ip."\"\n";
				   $bond_config .= $cidr ? "PREFIX=\"".$cidr."\"\n" : "NETMASK=\"".$subnet."\"\n";
				if ($is_gateway)
				{
					$bond_config .= "GATEWAY=\"".$gateway."\"\n";
					for (my $i = 0; $i < @{$dns}; $i++)
					{
						$bond_config .= "DNS".($i+1)."=\"".$dns->[$i]."\"\n";
					}
				}
				$bond_config .= "DEFROUTE=\"".$say_defroute."\"\n";
				$bond_config .= "ZONE=\"".uc($say_interface)."\"";
				   
				my $link1_config =  "# $say_network - Link 1\n";
				   $link1_config .= "HWADDR=\"".uc($link1_mac)."\"\n";
				   $link1_config .= "UUID=\"".$link1_uuid."\"\n";
				   $link1_config .= "NAME=\"".$interface_prefix." ".$network_count." - Link 1\"\n";
				   $link1_config .= "DEVICE=\"".$new_link1_iface."\"\n";
				   $link1_config .= "TYPE=\"Ethernet\"\n";
				   $link1_config .= "BOOTPROTO=\"none\"\n";
				   $link1_config .= "IPV6INIT=\"no\"\n";
				   $link1_config .= "ONBOOT=\"yes\"\n";
				   $link1_config .= "USERCTL=\"no\"\n";
				   $link1_config .= "MTU=\"1500\"\n";	# TODO: Make the MTU user-adjustable
				   $link1_config .= "NM_CONTROLLED=\"yes\"\n";
				   $link1_config .= "SLAVE=\"yes\"\n";
				   $link1_config .= "MASTER=\"".$say_interface."_bond1\"\n";
				   $link1_config .= "ZONE=\"".uc($say_interface)."\"";
				   
				my $link2_config =  "# $say_network - Link 2\n";
				   $link2_config .= "HWADDR=\"".uc($link2_mac)."\"\n";
				   $link2_config .= "UUID=\"".$link2_uuid."\"\n";
				   $link2_config .= "NAME=\"".$interface_prefix." ".$network_count." - Link 2\"\n";
				   $link2_config .= "DEVICE=\"".$new_link2_iface."\"\n";
				   $link2_config .= "TYPE=\"Ethernet\"\n";
				   $link2_config .= "BOOTPROTO=\"none\"\n";
				   $link2_config .= "IPV6INIT=\"no\"\n";
				   $link2_config .= "ONBOOT=\"yes\"\n";
				   $link2_config .= "USERCTL=\"no\"\n";
				   $link2_config .= "MTU=\"1500\"\n";	# TODO: Make the MTU user-adjustable
				   $link2_config .= "NM_CONTROLLED=\"yes\"\n";
				   $link2_config .= "SLAVE=\"yes\"\n";
				   $link2_config .= "MASTER=\"".$say_interface."_bond1\"\n";
				   $link2_config .= "ZONE=\"".uc($say_interface)."\"";
				   
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					bond_config  => $bond_config, 
					link1_config => $link1_config, 
					link2_config => $link2_config, 
				}});
				
				# Make backups of existing files
				if (-e $bond_file)
				{
					$anvil->Storage->backup({debug => 2, file => $bond_file});
				}
				if (-e $old_link1_file)
				{
					$anvil->Storage->backup({debug => 2, file => $old_link1_file});
				}
				if (-e $old_link2_file)
				{
					$anvil->Storage->backup({debug => 2, file => $old_link2_file});
				}
				if (($old_link1_file ne $new_link1_file) && (-e $new_link1_file))
				{
					$anvil->Storage->backup({debug => 2, file => $new_link1_file});
				}
				if (($old_link2_file ne $new_link2_file) && (-e $new_link2_file))
				{
					$anvil->Storage->backup({debug => 2, file => $new_link2_file});
				}

				### Write out the new configs
				# Bond, Link 1 and Link 2
				$anvil->Storage->write_file({file => $bond_file,      body => $bond_config,  user => "root", group => "root", mode => "0644", overwrite => 1});
				$anvil->Storage->write_file({file => $new_link1_file, body => $link1_config, user => "root", group => "root", mode => "0644", overwrite => 1});
				$anvil->Storage->write_file({file => $new_link2_file, body => $link2_config, user => "root", group => "root", mode => "0644", overwrite => 1});
				
				### NOTE: Everything except the unlink is disabled until we sort out the reload
				# Shut down (and rename) Link 1
				if ($old_link1_iface)
 				{
					# Take down and rename the old link 1 interface
					#print "Downing: ..... [$old_link1_iface]\n";
					#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ifdown}." ".$old_link1_iface});
					#print "Disconnecting: [$old_link1_iface]\n";
					#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ip}." link set ".$old_link1_iface." down"});
				}
				if ($old_link1_iface ne $new_link1_iface)
				{
					# Rename it
					#print "Renaming: .... [$old_link1_iface] -> [$new_link1_iface]\n";
					#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ip}." link set ".$old_link1_iface." name ".$new_link1_iface});
					
					# Unllnk the old one, if it exists.
					if (-e $old_link1_file)
					{
						#print "Deleting: .... [$old_link1_file]\n";
						unlink $old_link1_file;
					}
				}
				# Drop the new link, too, in case it still has the old config
				#print "Downing: ..... [$new_link1_iface]\n";
				#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ifdown}." ".$new_link1_iface});
				
				# Shut down (and rename) Link 2
				if ($old_link2_iface)
				{
					# Take down and rename the old link 1 interface
					#print "Downing: ..... [$old_link2_iface]\n";
					#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ifdown}." ".$old_link2_iface});
					#print "Disconnecting: [$old_link2_iface]\n";
					#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ip}." link set ".$old_link2_iface." down"});
				}
				if ($old_link2_iface ne $new_link2_iface)
				{
					# Rename it
					#print "Renaming ..... [$old_link2_iface] -> [$new_link2_iface]\n";
					#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ip}." link set ".$old_link2_iface." name ".$new_link2_iface});
				}
				# Drop the new link, too, in case it still has the old config
				#print "Downing: ..... [$new_link2_iface]\n";
				#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ifdown}." ".$new_link2_iface});
				
				# Start the bond.
				#print "Uping: ....... [$new_bond_iface]\n";
				#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ifup}." ".$new_bond_iface});
				
				# Reconnect and up link 1 
				#print "Uping: ....... [$new_link1_iface]\n";
				#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ifup}." ".$new_link1_iface});
				#print "Reconnecting:  [$new_link1_iface]\n";
				#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ip}." link set ".$new_link1_iface." up"});
				
				# Reconnect and up link 2
				#print "Uping: ....... [$new_link2_iface]\n";
				#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ifup}." ".$new_link2_iface});
				#print "Reconnecting:  [$new_link2_iface]\n";
				#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ip}." link set ".$new_link2_iface." up"});
				
				# Connect the bond (in case it isn't already)r
				#print "Connecting: .. [$new_bond_iface]\n";
				#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ip}." link set ".$new_bond_iface." up"});
			}
			elsif ((exists $anvil->data->{variables}{form}{config_step2}{$link1_key}{value}) && ($anvil->Validate->is_mac({mac => $anvil->data->{variables}{form}{config_step2}{$link1_key}{value}})))
			{
				# Single interface, set it up
				my $link1_mac = $anvil->data->{variables}{form}{config_step2}{$link1_key}{value};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { link1_mac => $link1_mac }});
				
				my $say_network      = "";
				my $say_interface    = "";
				my $interface_prefix = "";
				if ($network_type eq "bcn")
				{
					$say_network      = "Back-Channel Network ".$network_count;
					$say_interface    = "bcn".$network_count;
					$interface_prefix = "BCN";
				}
				elsif ($network_type eq "sn")
				{
					$say_network      = "Storage Network ".$network_count; 
					$say_interface    = "sn".$network_count;
					$interface_prefix = "SN";
				}
				elsif ($network_type eq "ifn")
				{
					$say_network      = "Internet-Facing Network ".$network_count; 
					$say_interface    = "ifn".$network_count;
					$interface_prefix = "IFN";
				}
				my $say_defroute    = $is_gateway ? "yes" : "no";
				my $cidr            = $anvil->Convert->cidr({subnet => $subnet});
				my $new_link1_file  = $anvil->data->{path}{directories}{ifcfg}."/ifcfg-".$interface_prefix."_".$network_count."_-_Link_1";
				my $old_link1_file  = $new_link1_file;
				my $new_link1_iface = $say_interface."_link1";
				if ((exists $anvil->data->{sys}{mac}{$link1_mac}{iface}) && ($anvil->data->{sys}{mac}{$link1_mac}{iface}))
				{
					$old_link1_file = $anvil->data->{path}{directories}{ifcfg}."/ifcfg-".$anvil->data->{sys}{mac}{$link1_mac}{iface};
				}
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
					say_defroute    => $say_defroute, 
					cidr            => $cidr, 
					new_link1_file  => $new_link1_file,
					old_link1_file  => $old_link1_file, 
					new_link1_iface => $new_link1_iface, 
				}});
				
				# Gather (or create) UUIDs
				my $link1_uuid = get_uuid_from_interface_file($anvil, $old_link1_file);
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { link1_uuid => $link1_uuid }});
				
				my $link1_config =  "# $say_network - Link 1\n";
				   $link1_config .= "HWADDR=\"".uc($link1_mac)."\"\n";
				   $link1_config .= "UUID=\"".$link1_uuid."\"\n";
				   $link1_config .= "NAME=\"".$interface_prefix." ".$network_count." - Link 1\"\n";
				   $link1_config .= "DEVICE=\"".$new_link1_iface."\"\n";
				   $link1_config .= "TYPE=\"Ethernet\"\n";
				   $link1_config .= "BOOTPROTO=\"none\"\n";
				   $link1_config .= "IPV6INIT=\"no\"\n";
				   $link1_config .= "ONBOOT=\"yes\"\n";
				   $link1_config .= "IPADDR=\"".$ip."\"\n";
				   $link1_config .= $cidr ? "PREFIX=\"".$cidr."\"\n" : "NETMASK=\"".$subnet."\"\n";
				if ($is_gateway)
				{
					$link1_config .= "GATEWAY=\"".$gateway."\"\n";
					for (my $i = 0; $i < @{$dns}; $i++)
					{
						$link1_config .= "DNS".($i+1)."=\"".$dns->[$i]."\"\n";
					}
				}
				   $link1_config .= "DEFROUTE=\"".$say_defroute."\"\n";
				   $link1_config .= "USERCTL=\"no\"\n";
				   $link1_config .= "MTU=\"1500\"\n";	# TODO: Make the MTU user-adjustable
				   $link1_config .= "NM_CONTROLLED=\"yes\"\n";
				   $link1_config .= "ZONE=\"".uc($say_interface)."\"";
				   
				# Backup the existing link1 file, if it exists.
				if (-e $old_link1_file)
				{
					$anvil->Storage->backup({debug => 2, file => $old_link1_file});
				}
				
				# Write out the link1 config file.
				$anvil->Storage->write_file({file => $new_link1_file, body => $link1_config, user => "root", group => "root", mode => "0644", overwrite => 1});
				
				if ($old_link1_iface)
 				{
					# Take down and rename the old link 1 interface
					#print "Downing: ..... [$old_link1_iface]\n";
					#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ifdown}." ".$old_link1_iface});
					#print "Disconnecting: [$old_link1_iface]\n";
					#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ip}." link set ".$old_link1_iface." down"});
				}
				if ($old_link1_iface ne $new_link1_iface)
				{
					# Rename it
					#print "Renaming: .... [$old_link1_iface] -> [$new_link1_iface]\n";
					#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ip}." link set ".$old_link1_iface." name ".$new_link1_iface});
					
					# Unllnk the old one, if it exists.
					if (-e $old_link1_file)
					{
						#print "Deleting: .... [$old_link1_file]\n";
						unlink $old_link1_file;
					}
				}
				
				# Drop the new link, too, in case it still has the old config
				#print "Downing: ..... [$new_link1_iface]\n";
				#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{ifdown}." ".$new_link1_iface});
			}
			else
			{
				# Doesn't exist, skip.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "log_0149", variables => { network => $this_network }});
				next;
			}
		}
	}
	
	# We're half-way there.
	$anvil->Job->update_progress({
		progress => 50, 
		job_uuid => $anvil->data->{job}{uuid},
	});
	
	# If any virtio bridges exist, remove it/them.
	my $start                   = 0;
	my ($bridges, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." net-list"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { bridges => $bridges, return_code => $return_code }});
	foreach my $line (split/\n/, $bridges)
	{
		$line = $anvil->Words->clean_spaces({string => $line});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
		if ($line =~ /^----------/)
		{
			$start = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { start => $start }});
			next;
		}
		next if not $start;
		my $bridge = ($line =~ /(.*?)\s/)[0];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { bridge => $bridge }});
		
		$anvil->data->{virsh}{bridge}{$bridge} = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "virsh::bridge::$bridge" => $anvil->data->{virsh}{bridge}{$bridge} }});
	}
	
	foreach my $bridge (sort {$a cmp $b} keys %{$anvil->data->{virsh}{bridge}})
	{
		# Destroy (stop) it.
		my ($destroy, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." net-destroy ".$bridge});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { destroy => $destroy, return_code => $return_code }});
		
		# Disable it from auto-start.
		(my $disable, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." net-autostart ".$bridge." --disable"});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { disable => $disable, return_code => $return_code }});
		
		# Undefine (delete)
		(my $undefine, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." net-undefine ".$bridge});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { undefine => $undefine, return_code => $return_code }});
	}
	
	### TODO: This isn't working... The route table won't set the IFN as the default route properly and 
	###       the IFN links seem to drop out and not return when trying to fix it. For now, we'll do a 
	###       closing reboot which seems to always comes up OK.
	# Reload the network
	#print "reloading nmcli\n";
	#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{nmcli}." connection reload"});
	
	$anvil->Job->update_progress({
		progress => 75, 
		job_uuid => $anvil->data->{job}{uuid},
	});

	return(0);
}

# This will read a network interface file and return the UUID="x" value. If the file doesn't exist or the 
# UUID was not found, a new UUID is generated and returned.
sub get_uuid_from_interface_file
{
	my ($anvil, $file) = @_;
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0131", variables => { function => "get_uuid_from_interface_file" }});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file => $file }});
	
	my $uuid = "";
	if (-e $file)
	{
		my $body = $anvil->Storage->read_file({file => $file});
		foreach my $line (split/\n/, $body)
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
			$line =~ s/#.*//;
			if ($line =~ /UUID=\"(.*?)\"/)
			{
				my $test_uuid = $1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { test_uuid => $test_uuid }});
				if ($anvil->Validate->is_uuid({uuid => $test_uuid}))
				{
					$uuid = $test_uuid;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { uuid => $uuid }});
				}
				last;
			}
		}
	}
	if (not $uuid)
	{
		$uuid = $anvil->Get->uuid();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { uuid => $uuid }});
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { uuid => $uuid }});
	return($uuid);
}

# This will pick up the job, or exit.
sub pickup_job_details
{
	my ($anvil) = @_;
	
	# If this returns '1', the job-uuid was bad. If it returns '2', another process is running.
	my $return = $anvil->Job->get_job_details({check => 1, job_uuid => $anvil->data->{switches}{'job-uuid'}});
	if ($return == 1)
	{
		# It's not a valid UUID.
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
		$anvil->nice_exit({code => 6});
	}
	if ($return == 2)
	{
		# This job is being handled by another process that is still alive.
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
		$anvil->nice_exit({code => 3});
	}
	
	# Still alive? Good.
	my $job_picked_up_by = $anvil->data->{jobs}{job_picked_up_by};
	my $job_progress     = $anvil->data->{jobs}{job_progress};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		job_picked_up_by => $job_picked_up_by,
		job_progress     => $job_progress,
	}});
	
	# See if the job was picked up by another running instance.
	if ($job_picked_up_by)
	{
		# The previous job is gone if we're still alive, we'll take this over.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0147", variables => { 
			pid     => $job_picked_up_by,
			percent => $job_progress, 
		}});
	}
	
	# This will store the variables from the database
	$anvil->data->{variables} = {};
	
	# If we're still alive, pick up the details.
	my $query   = "
SELECT 
    variable_name, 
    variable_value 
FROM 
    variables 
WHERE 
    variable_name 
LIKE 
    'form::config_step%' 
AND 
    variable_source_table = 'hosts' 
AND 
    variable_source_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
;";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
	
	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
	my $count   = @{$results};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		results => $results, 
		count   => $count,
	}});
	foreach my $row (@{$results})
	{
		my $this_variable = $row->[0];
		my $this_value    = $row->[1];
		my $secure        = $this_variable =~ /passw/ ? 1 : 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
			this_variable => $this_variable, 
			this_value    => (($anvil->Log->secure) or (not $secure)) ? $this_value : $anvil->Words->string({key => "log_0186"}), 
		}});
		
		$anvil->_make_hash_reference($anvil->data->{variables}, $this_variable, $this_value);
	}
	
	# Record that we've picked up this job.
	$anvil->Job->update_progress({
		progress => 1, 
		message  => "message_0015", 
		job_uuid => $anvil->data->{job}{uuid},
	});
	
	return(0);
}