anvil/tools/striker-initialize-host

#!/usr/bin/perl
# 
# This program is setuid 'admin' and calls a (new) peer to read its host name and system UUID. It takes the 
# target's password in via a file.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = No database connection.
# 2 = Job UUID not passed and no unclaimed jobs found
# 3 = Job detauls not found in the database.
# 4 = Unable to connect to the target.
# 5 = Failed to install the anvil package.
# 6 = Failed to add our database to the target's anvil.conf file.
# 

use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

# Read switches (target ([user@]host[:port]) and the file with the target's password.
$anvil->data->{switches}{'job-uuid'} = "";
$anvil->Get->switches;
### TODO: Remove this before final release
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
##########################################
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
	'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'},
}});

# Connect to the database(s).
$anvil->Database->connect({check_for_resync => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0003"});
	$anvil->nice_exit({exit_code => 1});
}

$anvil->data->{job}{progress} = 0;
get_job_details($anvil);
wait_for_access($anvil);
set_host_name($anvil);
add_repos($anvil);
configure_lvm($anvil);
add_databases($anvil);
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0047"});
update_progress($anvil, 100, "job_0047");

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This configures filtering on LVM
sub configure_lvm
{
	my ($anvil) = @_;
	
	my $problem = $anvil->Storage->manage_lvm_conf({
		debug       => 2,
		password    => $anvil->data->{data}{password}, 
		port        => $anvil->data->{data}{ssh_port}, 
		target      => $anvil->data->{data}{host_ip_address},
		remote_user => "root",
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	if ($problem)
	{
		$anvil->data->{job}{progress} += 5;
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0274"});
		update_progress($anvil, $anvil->data->{job}{progress}, "job_0274");
	}
	else
	{
		$anvil->data->{job}{progress} += 5;
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0273"});
		update_progress($anvil, $anvil->data->{job}{progress}, "job_0273");
	}
	
	return(0);
}

# Set the host name, if needed.
sub set_host_name
{
	my ($anvil) = @_;
	
	if ((not defined $anvil->data->{data}{host_name}) or (not $anvil->data->{data}{host_name}))
	{
		return(0);
	}
	$anvil->data->{job}{progress} += 5;
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0061", variables => { host_name => $anvil->data->{data}{host_name} }});
	update_progress($anvil, $anvil->data->{job}{progress}, "job_0061,!!host_name!".$anvil->data->{data}{host_name}."!!");
	
	if (not $anvil->Validate->domain_name({name => $anvil->data->{data}{host_name}}))
	{
		# Bad host_name.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0062", variables => { host_name => $anvil->data->{data}{host_name} }});
		$anvil->data->{job}{progress} += 5;
		update_progress($anvil, $anvil->data->{job}{progress}, "job_0062,!!host_name!".$anvil->data->{data}{host_name}."!!");
		return(1);
	}
	
	# Try to set the host name now.
	my ($host_name, $descriptive_host_name) = $anvil->System->host_name({
		debug       => 3, 
		set         => $anvil->data->{data}{host_name},
		password    => $anvil->data->{data}{password}, 
		port        => $anvil->data->{data}{ssh_port}, 
		target      => $anvil->data->{data}{host_ip_address},
		remote_user => "root",
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		host_name             => $host_name,
		descriptive_host_name => $descriptive_host_name, 
	}});
	if ($host_name eq $anvil->data->{data}{host_name})
	{
		# Success!
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0034"});
		$anvil->data->{job}{progress} += 5;
		update_progress($anvil, $anvil->data->{job}{progress}, "job_0034");
	}
	else
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0063", variables => { 
			host_name         => $anvil->data->{data}{host_name},
			current_host_name => $host_name, 
		}});
		$anvil->data->{job}{progress} += 5;
		update_progress($anvil, $anvil->data->{job}{progress}, "job_0063,!!host_name!".$anvil->data->{data}{host_name}."!!,!!current_host_name!".$host_name."!!");
		return(1);
	}
	
	return(0);
}

# Add any databases we're using to the initialized host.
sub add_databases
{
	my ($anvil) = @_;
	
	# Normally, we'd use the BCN IPs to tell the peer how to reach our database, but it's entirely 
	# possible at this stage that the target doesn't have a BCN IP. So we need to figure out what IP it
	# has and we have that are on the same subnet. 
	# Scan our network and the target's network, then compare them.
	update_progress($anvil, ($anvil->data->{job}{progress} += 5), "job_0046");
	
	my $target = $anvil->data->{data}{host_ip_address};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }});
	
	$anvil->Network->get_ips({debug => 3});
	$anvil->Network->get_ips({
		debug       => 3,
		target      => $target,
		remote_user => "root",
		password    => $anvil->data->{data}{password}, 
		port        => $anvil->data->{data}{ssh_port}, 
	});
	my $local_host = $anvil->Get->short_host_name();
	my ($match)    = $anvil->Network->find_matches({
		debug  => 3,
		first  => $local_host,
		second => $target, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { match => $match }});
	
	# We'll sort in reverse order, so if there is a BCN address, we'll use it.
	my $host_uuid   = $anvil->data->{sys}{host_uuid};
	my $db_host     = "";
	my $db_ping     = 1,
	my $db_port     = "";
	my $db_password = "";
	foreach my $interface (sort {$a cmp $b} keys %{$match->{$local_host}})
	{
		my $local_ip = $anvil->data->{network}{$local_host}{interface}{$interface}{ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			interface => $interface, 
			local_ip  => $local_ip,
		}});
		if (($local_ip) && ($anvil->Validate->ipv4({ip => $local_ip})))
		{
			$db_host     = $local_ip;
			$db_port     = $anvil->data->{database}{$host_uuid}{port};
			$db_password = $anvil->data->{database}{$host_uuid}{password};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				db_host     => $db_host,
				db_port     => $db_port, 
				db_password => $anvil->Log->is_secure($db_password),
			}});
		}
		last if $db_host;
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { db_host => $db_host }});
	if ($db_host)
	{
		my $failed = $anvil->Database->manage_anvil_conf({
			debug        => 2, 
			target       => $target,
			remote_user  => "root",
			password     => $anvil->data->{data}{password}, 
			port         => $anvil->data->{data}{ssh_port}, 
			db_host_uuid => $anvil->data->{sys}{host_uuid}, 
			db_host      => $db_host,
			db_ping      => $db_ping, 
			db_port      => $db_port,
			db_password  => $db_password, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
		if ($failed)
		{
			# Something went wrong
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0076"});
			update_progress($anvil, 100, "error_0076");
			$anvil->nice_exit({exit_code => 6});
		}
		else
		{
			# Success! We're done!
			update_progress($anvil, $anvil->data->{job}{progress} += 5, "job_0047");
		}
	}
	else
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0084"});
		update_progress($anvil, 100, "error_0084");
		$anvil->nice_exit({exit_code => 6});
	}
	
	# Now add any peers we know about.
	foreach my $uuid (keys %{$anvil->data->{database}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:uuid'           => $uuid,
			's2:sys::host_uuid' => $anvil->data->{sys}{host_uuid},
		}});
		next if $uuid eq $anvil->data->{sys}{host_uuid};
		
		my $target_host = $anvil->Database->get_host_from_uuid({host_uuid => $uuid, short => 1});
		   $target_host = $target if not $target_host;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host => $target_host }});
		
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0068,!!host_name!".$target_host."!!");
		
		# This is a peer.
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"database::${uuid}::host"     => $anvil->data->{database}{$uuid}{host},
			"database::${uuid}::port"     => $anvil->data->{database}{$uuid}{port},
			"database::${uuid}::password" => $anvil->Log->is_secure($anvil->data->{database}{$uuid}{password}),
			"database::${uuid}::ping"     => $anvil->data->{database}{$uuid}{ping},
		}});
		
		# The host being initialized may not be able to reach the 'host' as set in our config. So get the IPs
		# we know about for this host and see if we can find a match.
		my $peer_host     = "";
		my $peer_port     = $anvil->data->{database}{$uuid}{port};
		my $peer_ping     = $anvil->data->{database}{$uuid}{ping};
		my $peer_password = $anvil->data->{database}{$uuid}{password};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			peer_host     => $peer_host,
			peer_port     => $peer_port,
			peer_password => $anvil->Log->is_secure($peer_password),
			peer_ping     => $peer_ping,
		}});
		
		# Get the IP(s) for the Striker peer.
		$anvil->Network->load_ips({host_uuid => $uuid});
		
		# Find a match between the target and the peer.
		my ($match) = $anvil->Network->find_matches({
			first  => $target,
			second => $uuid,
		});
		my $keys = keys %{$match};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			match  => $match,
			'keys' => $keys,
		}});
		
		# Did we find a match?
		if ($keys)
		{
			# Yup!
			my $match_found = 0;
			foreach my $interface (sort {$a cmp $b} keys %{$match->{$uuid}})
			{
				# Get the IP
				$peer_host = $match->{$uuid}{$interface}{ip};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					interface => $interface, 
					peer_host => $peer_host,
				}});
				if ($peer_host)
				{
					my $failed = $anvil->Database->manage_anvil_conf({
						debug        => 2, 
						target       => $target,
						remote_user  => "root",
						password     => $anvil->data->{data}{password}, 
						port         => $anvil->data->{data}{ssh_port}, 
						db_host_uuid => $uuid, 
						db_host      => $peer_host,
						db_ping      => $peer_ping,
						db_port      => $peer_port,
						db_password  => $peer_password, 
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
					$anvil->data->{job}{progress} += 3;
					if ($failed)
					{
						# Something went wrong
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0076"});
						update_progress($anvil, $anvil->data->{job}{progress}, "job_0070");
					}
					else
					{
						# Success! We're done@
						$match_found = 1;
						update_progress($anvil, $anvil->data->{job}{progress}, "job_0034");
					}
					
					
					last;
				}
			}
			if (not $match_found)
			{
				# No (workable) match, not sure how this could happen though... Bug in 
				# 'Network->find_matches()'?
				$anvil->data->{job}{progress} += 3;
				update_progress($anvil, $anvil->data->{job}{progress}, "job_0069");
			}
		}
		else
		{
			# No match
			$anvil->data->{job}{progress} += 3;
			update_progress($anvil, $anvil->data->{job}{progress}, "job_0069");
		}
	}
	
	return(0);
}

# This adds this machine's repo, and if the internet is up on the target, we'll add the Anvil! repo as well.
sub add_repos
{
	my ($anvil) = @_;
	
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0028"});
	$anvil->data->{job}{progress} += 5;
	update_progress($anvil, $anvil->data->{job}{progress}, "job_0028");
	
	# Add the local repo.
	my $repo = $anvil->Striker->get_local_repo({debug => 2});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { repo => $repo }});
	
	# If this striker is an enterprise user, get the key for when we initialize targets.
	my $enterprise_key = "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"path::configs::alteeve-release.repo" => $anvil->data->{path}{configs}{'alteeve-release.repo'},
	}});
	if (-e $anvil->data->{path}{configs}{'alteeve-release.repo'})
	{
		# Read the file
		my $alteeve_repo = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'alteeve-release.repo'}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { alteeve_repo => $alteeve_repo }});
		foreach my $line (split/\n/, $alteeve_repo)
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
			if ($line =~ /baseurl=.*?\/enterprise\/.*?\/(.*?)\//)
			{
				$enterprise_key = $1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { enterprise_key => $enterprise_key }});
				last;
			}
		}
	}
	
	if ($repo)
	{
		# NOTE: We can't use Storage->write_file() because the target may not have 'rsync' installed
		#       yet.
		my $shell_call = "
".$anvil->data->{path}{exe}{cat}." > /etc/yum.repos.d/".$anvil->Get->short_host_name.".repo << EOF
$repo
EOF
";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			debug       => 3, 
			shell_call  => $shell_call, 
			password    => $anvil->data->{data}{password}, 
			port        => $anvil->data->{data}{ssh_port}, 
			target      => $anvil->data->{data}{host_ip_address},
			remote_user => "root",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0029"});
		$anvil->data->{job}{progress} += 5;
		update_progress($anvil, $anvil->data->{job}{progress}, "job_0029");
	}
	
	# If I have a Red Hat user and password, try to subscribe this syste,.
	if (($anvil->data->{data}{rh_user}) && ($anvil->data->{data}{rh_password}))
	{
		# If there's no internet, this will fail. If the network is up, we can see if the 
		# registration is still needed.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0030"});
		$anvil->data->{job}{progress} += 5;
		update_progress($anvil, $anvil->data->{job}{progress}, "job_0030");
		
		# We'll attach subscriptions if this is set
		my $subscribe  = 1;
		my $shell_call = $anvil->data->{path}{exe}{'subscription-manager'}." identity";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			debug       => 3,
			shell_call  => $shell_call, 
			password    => $anvil->data->{data}{password}, 
			port        => $anvil->data->{data}{ssh_port}, 
			target      => $anvil->data->{data}{host_ip_address},
			remote_user => "root", 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output, 
			error       => $error, 
			return_code => $return_code,
		}});
		if ($return_code eq "0")
		{
			# Already registered.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0031"});
			$anvil->data->{job}{progress} += 5; 
			update_progress($anvil, $anvil->data->{job}{progress}, "job_0031");
		}
		elsif ($return_code eq "1")
		{
			# Registration is needed. This can take a while, so we give it a generous timeout.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0033"});
			$anvil->data->{job}{progress} += 5;
			update_progress($anvil, $anvil->data->{job}{progress}, "job_0033");
			my $bash_password =  $anvil->data->{data}{rh_password};
			   $bash_password =~ s/'/\\\'/g;
			my $shell_call = $anvil->data->{path}{exe}{'subscription-manager'}." register --username ".$anvil->data->{data}{rh_user}." --password '".$bash_password."' --auto-attach --force";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			my ($output, $error, $return_code) = $anvil->Remote->call({
				debug       => 3,
				shell_call  => $shell_call,
				password    => $anvil->data->{data}{password}, 
				port        => $anvil->data->{data}{ssh_port}, 
				target      => $anvil->data->{data}{host_ip_address},
				remote_user => "root", 
				timeout     => 300,
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output, 
				error       => $error, 
				return_code => $return_code,
			}});
			if ($return_code eq "70")
			{
				# No Internet (or can't reach the subscriion servers)
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0032"});
				$anvil->data->{job}{progress} += 5;
				update_progress($anvil, $anvil->data->{job}{progress}, "job_0032");
				$subscribe = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { subscribe => $subscribe }});
			}
			elsif ($return_code)
			{
				# Something went wrong
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "job_0035", variables => { 
					output      => $output, 
					error       => $error, 
					return_code => $return_code,
				}});
				$anvil->data->{job}{progress} += 5; 
				update_progress($anvil, $anvil->data->{job}{progress}, "job_0035,!!return_code!".$return_code."!!,!!output!".$output."!!,!!error!".$error."!!");
				$subscribe = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { subscribe => $subscribe }});
			}
			else
			{
				# Success!
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0034"});
				$anvil->data->{job}{progress} += 5;
				update_progress($anvil, $anvil->data->{job}{progress}, "job_0034");
			}
		}
		elsif ($return_code eq "70")
		{
			# No Internet (or can't reach the subscriion servers)
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0032"});
			$anvil->data->{job}{progress} += 5;
			update_progress($anvil, $anvil->data->{job}{progress}, "job_0032");
			$subscribe = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { subscribe => $subscribe }});
		}
		
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { subscribe => $subscribe }});
		if ($subscribe)
		{
			# We're going to need these repos when it's all said and done. The first two are 
			# enabled by default.
			$anvil->data->{repos}{'rhel-8-for-x86_64-baseos-rpms'}            = 0;
			$anvil->data->{repos}{'rhel-8-for-x86_64-appstream-rpms'}         = 0;
			$anvil->data->{repos}{'codeready-builder-for-rhel-8-x86_64-rpms'} = 0;
			
			# Both target types need 'codeready-builder-for-rhel-8-x86_64-rpms', but only nodes 
			# need 'rhel-8-for-x86_64-highavailability-rpms'
			# We blindly subscribe, then we'll check that they're actually subscribed.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0036", variables => { repo => 'codeready-builder-for-rhel-8-x86_64-rpms' }});
			$anvil->data->{job}{progress} += 5;
			update_progress($anvil, $anvil->data->{job}{progress}, "job_0036,!!repo!codeready-builder-for-rhel-8-x86_64-rpms!!");
			
			my $shell_call = $anvil->data->{path}{exe}{'subscription-manager'}." repos --enable codeready-builder-for-rhel-8-x86_64-rpms";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			my ($output, $error, $return_code) = $anvil->Remote->call({
				debug       => 3,
				shell_call  => $shell_call,
				password    => $anvil->data->{data}{password}, 
				port        => $anvil->data->{data}{ssh_port}, 
				target      => $anvil->data->{data}{host_ip_address},
				remote_user => "root", 
				timeout     => 300,
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output, 
				error       => $error, 
				return_code => $return_code,
			}});
			
			# If this 
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'data::type' => $anvil->data->{data}{type} }});
			if ($anvil->data->{data}{type} eq "node")
			{
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0036", variables => { repo => 'rhel-8-for-x86_64-highavailability-rpms' }});
				$anvil->data->{job}{progress} += 5;
				update_progress($anvil, $anvil->data->{job}{progress}, "job_0036,!!repo!rhel-8-for-x86_64-highavailability-rpms!!");
				
				my $shell_call = $anvil->data->{path}{exe}{'subscription-manager'}." repos --enable rhel-8-for-x86_64-highavailability-rpms";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				my ($output, $error, $return_code) = $anvil->Remote->call({
					debug       => 3,
					shell_call  => $shell_call,
					password    => $anvil->data->{data}{password}, 
					port        => $anvil->data->{data}{ssh_port}, 
					target      => $anvil->data->{data}{host_ip_address},
					remote_user => "root", 
					timeout     => 300,
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output, 
					error       => $error, 
					return_code => $return_code,
				}});
				
				# Add this to the needed list.
				$anvil->data->{repos}{'rhel-8-for-x86_64-highavailability-rpms'} = 0;
			}
			
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0037"});
			$anvil->data->{job}{progress} += 5;
			update_progress($anvil, $anvil->data->{job}{progress}, "job_0037");
			undef $output;
			undef $error;
			undef $return_code;
			undef $shell_call;
			$shell_call = $anvil->data->{path}{exe}{'subscription-manager'}." repos --list-enabled";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			($output, $error, $return_code) = $anvil->Remote->call({
				debug       => 3,
				shell_call  => $shell_call,
				password    => $anvil->data->{data}{password}, 
				port        => $anvil->data->{data}{ssh_port}, 
				target      => $anvil->data->{data}{host_ip_address},
				remote_user => "root", 
				timeout     => 300,
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output, 
				error       => $error, 
				return_code => $return_code,
			}});
			foreach my $line (split/\n/, $output)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
				if ($line =~ /Repo ID:\s+(.*)$/i)
				{
					my $repo = $1;
					$anvil->data->{repos}{$repo} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "repos::${repo}" => $anvil->data->{repos}{$repo} }});
				}
			}
			
			# Are any still missing?
			foreach my $repo (sort {$a cmp $b} keys %{$anvil->data->{repos}})
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "repos::${repo}" => $anvil->data->{repos}{$repo} }});
				if (not $anvil->data->{repos}{$repo})
				{
					# Well this is a problem...
					$anvil->data->{job}{progress} += 5;
					update_progress($anvil, $anvil->data->{job}{progress}, "job_0038,!!repo!".$repo."!!");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "job_0038", variables => { repo => $repo }});
				}
			}
		}
	}
	
	# If this is a CentOS Stream OS, enable the HA repo.
	my ($os_type, $os_arch) = $anvil->Get->os_type({
		password => $anvil->data->{data}{password}, 
		port     => $anvil->data->{data}{ssh_port}, 
		target   => $anvil->data->{data}{host_ip_address},
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		os_type => $os_type, 
		os_arch => $os_arch, 
	}});
	if ($os_type =~ /centos-stream/)
	{
		# Enable the HA repo
		my $shell_call = $anvil->data->{path}{exe}{yum}." config-manager --set-enabled ha";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			debug       => 3, 
			shell_call  => $shell_call, 
			password    => $anvil->data->{data}{password}, 
			port        => $anvil->data->{data}{ssh_port}, 
			target      => $anvil->data->{data}{host_ip_address},
			remote_user => "root",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0224"});
		$anvil->data->{job}{progress} += 2;
		update_progress($anvil, $anvil->data->{job}{progress}, "job_0224");
	}
	
	### NOTE: This seems dumb without internet, but it's worth calling anyway in case the repo on the 
	###       Striker has newer packages. If we go right to installing packages and one of them is newer
	###       than an installed dependency, the install could fail. This prevents that.
	# Call an OS update.
	$anvil->data->{job}{progress} += 5;
	update_progress($anvil, $anvil->data->{job}{progress}, "job_0039");
	$anvil->data->{job}{progress} += 5;
	update_progress($anvil, $anvil->data->{job}{progress}, "job_0040");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "job_0039"});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "job_0040"});
	
	my $shell_call = $anvil->data->{path}{exe}{'dnf'}." -y update";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	my ($output, $error, $return_code) = $anvil->Remote->call({
		debug       => 3,
		shell_call  => $shell_call,
		password    => $anvil->data->{data}{password}, 
		port        => $anvil->data->{data}{ssh_port}, 
		target      => $anvil->data->{data}{host_ip_address},
		remote_user => "root", 
		timeout     => 3600,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		error       => $error, 
		return_code => $return_code,
	}});
	
	# Now remove biosdevname
	$anvil->data->{job}{progress} += 5;
	update_progress($anvil, $anvil->data->{job}{progress}, "job_0041");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "job_0041"});
	undef $output;
	undef $error;
	undef $return_code;
	undef $shell_call;
	$shell_call = $anvil->data->{path}{exe}{'dnf'}." -y remove biosdevname";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	($output, $error, $return_code) = $anvil->Remote->call({
		debug       => 3,
		shell_call  => $shell_call,
		password    => $anvil->data->{data}{password}, 
		port        => $anvil->data->{data}{ssh_port}, 
		target      => $anvil->data->{data}{host_ip_address},
		remote_user => "root", 
		timeout     => 300,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		error       => $error, 
		return_code => $return_code,
	}});
	
	# In the CI, we'll have custom repos installed. So here we're looking to see if 'anvil-X' is already 
	# installed. If so, we won't add our repo.
	my $anvil_role_rpm = "";
	my $alteeve_repo   = 0;
	undef $output;
	undef $error;
	undef $return_code;
	undef $shell_call;
	$shell_call = $anvil->data->{path}{exe}{'dnf'}." list installed";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	($output, $error, $return_code) = $anvil->Remote->call({
		debug       => 3,
		shell_call  => $shell_call,
		password    => $anvil->data->{data}{password}, 
		port        => $anvil->data->{data}{ssh_port}, 
		target      => $anvil->data->{data}{host_ip_address},
		remote_user => "root", 
		timeout     => 300,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		output      => $output, 
		error       => $error, 
		return_code => $return_code,
	}});
	foreach my $line (split/\n/, $output)
	{
		next if $line =~ /Installed Packages/i;
		$line =~ s/\s.*$//;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
		
		next if $line =~ /anvil-core/;
		if ($line =~ /alteeve-release.noarch/)
		{
			$alteeve_repo = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { alteeve_repo => $alteeve_repo }});
		}
		if ($line =~ /anvil-(.*).noarch/)
		{
			$anvil_role_rpm = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_role_rpm => $anvil_role_rpm }});
		}
		last if $anvil_role_rpm && $alteeve_repo;
	}
	
	# Install the Alteeve repo, if possible. There may be no Internet access, so it's OK if this fails.
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		alteeve_repo   => $alteeve_repo,
		anvil_role_rpm => $anvil_role_rpm,
	}});
	if (not $alteeve_repo)
	{
		if ($anvil_role_rpm)
		{
			# There's already an anvil RPM installed, so we're going to skip installing the repo.
			# Warn the user though.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "job_0042", variables => { 
				anvil_role_rpm => $anvil_role_rpm,
				alteeve_repo   => $anvil->data->{path}{urls}{alteeve_repo},
			}});
		}
		else
		{
			my ($alteeve_access) = $anvil->Network->check_internet({
				debug       => 2,
				domains     => ["alteeve.com"],
				password    => $anvil->data->{data}{password}, 
				port        => $anvil->data->{data}{ssh_port}, 
				target      => $anvil->data->{data}{host_ip_address},
				remote_user => "root", 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { alteeve_access => $alteeve_access }});
			if ($alteeve_access)
			{
				$shell_call = $anvil->data->{path}{exe}{'dnf'}." -y install ".$anvil->data->{path}{urls}{alteeve_repo};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				($output, $error, $return_code) = $anvil->Remote->call({
					debug       => 3,
					shell_call  => $shell_call,
					password    => $anvil->data->{data}{password}, 
					port        => $anvil->data->{data}{ssh_port}, 
					target      => $anvil->data->{data}{host_ip_address},
					remote_user => "root", 
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output, 
					error       => $error, 
					return_code => $return_code,
				}});
				
				# If successful, we now need to configure the repo.
				$shell_call = $anvil->data->{path}{exe}{'alteeve-repo-setup'}." --yes";
				if ($enterprise_key)
				{
					$shell_call = $anvil->data->{path}{exe}{'alteeve-repo-setup'}." --key ".$enterprise_key." --yes";
				}
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				($output, $error, $return_code) = $anvil->Remote->call({
					debug       => 2,
					shell_call  => $shell_call,
					password    => $anvil->data->{data}{password}, 
					port        => $anvil->data->{data}{ssh_port}, 
					target      => $anvil->data->{data}{host_ip_address},
					remote_user => "root", 
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output, 
					error       => $error, 
					return_code => $return_code,
				}});
			}
		}
	}
	
	# Install the anvil package now.
	my $package = $anvil->data->{data}{type} eq "dr" ? "anvil-dr" : "anvil-node";
	$anvil->data->{job}{progress} += 5;
	update_progress($anvil, $anvil->data->{job}{progress}, "job_0042,!!package!".$package."!!");
	$anvil->data->{job}{progress} += 5;
	update_progress($anvil, $anvil->data->{job}{progress}, "job_0040");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "job_0042", variables => { 'package' => $package }});
	undef $output;
	undef $error;
	undef $return_code;
	undef $shell_call;
	$shell_call = $anvil->data->{path}{exe}{'dnf'}." -y install ".$package;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	($output, $error, $return_code) = $anvil->Remote->call({
		debug       => 2,
		shell_call  => $shell_call,
		password    => $anvil->data->{data}{password}, 
		port        => $anvil->data->{data}{ssh_port}, 
		target      => $anvil->data->{data}{host_ip_address},
		remote_user => "root", 
		timeout     => 3600,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		error       => $error, 
		return_code => $return_code,
	}});
	
	$anvil->data->{job}{progress} += 5;
	update_progress($anvil, $anvil->data->{job}{progress}, "job_0043");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "job_0043"});
	undef $output;
	undef $error;
	undef $return_code;
	undef $shell_call;
	$shell_call = $anvil->data->{path}{exe}{'dnf'}." list installed ".$package;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	($output, $error, $return_code) = $anvil->Remote->call({
		debug       => 3,
		shell_call  => $shell_call,
		password    => $anvil->data->{data}{password}, 
		port        => $anvil->data->{data}{ssh_port}, 
		target      => $anvil->data->{data}{host_ip_address},
		remote_user => "root", 
		timeout     => 3600,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		error       => $error, 
		return_code => $return_code,
	}});
	# The return code of '0' means found, '1' means not found
	if ($return_code)
	{
		# Failed...
		update_progress($anvil, 100, "job_0044");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "job_0044"});
		$anvil->nice_exit({exit_code => 6});
	}
	else
	{
		# Found it!
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "job_0045"});
		$anvil->data->{job}{progress} += 5;
		update_progress($anvil, $anvil->data->{job}{progress}, "job_0045");
	}
	
	return(0);
}

# This goes into a loop until the target is accessible.
sub wait_for_access
{
	my ($anvil) = @_;
	
	### TODO: If the keys changed, the user may have been prompted to fix the keys for the admin user, 
	###       but not the 'root' user. This, we may fail to access the node as this runs as 'root'. Tell
	###       the user to check for bad keys and clear them if/when access fails.
	# Test access
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0023", variables => { target => $anvil->data->{data}{say_target} }});
	$anvil->data->{job}{progress} += 5;
	update_progress($anvil, $anvil->data->{job}{progress}, "job_0023,!!target!".$anvil->data->{data}{say_target}."!!");
	$anvil->data->{job}{progress} += 5;
	my $waiting = 1;
	my $access  = 0;
	my $timeout = time + 600;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { timeout => $timeout, 'time' => time }});
	while ($waiting)
	{
		$access = $anvil->Remote->test_access({
			password => $anvil->data->{data}{password}, 
			port     => $anvil->data->{data}{ssh_port}, 
			target   => $anvil->data->{data}{host_ip_address},
			user     => "root",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});

		if ($access)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0024"});
			$anvil->data->{job}{progress} += 5;
			update_progress($anvil, $anvil->data->{job}{progress}, "job_0024");
			$waiting = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
		}
		else
		{
			my $time_left = $timeout - time;
			if ($time_left < 0)
			{
				$waiting = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
			}
			else
			{
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "job_0025", variables => {
					target  => $anvil->data->{data}{say_target},
					timeout => $time_left, 
				}});
				update_progress($anvil, $anvil->data->{job}{progress}, "job_0025,!!target!".$anvil->data->{data}{say_target}."!!,!!timeout!".$time_left."!!");
				sleep 5;
			}
		}
	}
	
	if (not $access)
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "job_0026"});
		update_progress($anvil, 100, "job_0026");
		$anvil->nice_exit({exit_code => 3});
	}
	
	return(0);
}

# This picks up the details for the job, so we know who to connect to.
sub get_job_details
{
	my ($anvil) = @_;
	
	# If we don't have a job-uuid, try to find one.
	if (not $anvil->data->{switches}{'job-uuid'})
	{
		my $return = $anvil->Database->get_jobs();
		foreach my $hash_ref (@{$return})
		{
			my $job_command  = $hash_ref->{job_command};
			my $job_progress = $hash_ref->{job_progress};
			my $job_uuid     = $hash_ref->{job_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				job_command  => $job_command,
				job_progress => $job_progress,
				job_uuid     => $job_uuid, 
			}});
			next if $job_command ne $anvil->data->{path}{exe}{$THIS_FILE};
			next if $job_progress;
			
			# Still alive? found it!
			$anvil->data->{switches}{'job-uuid'} = $job_uuid;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'} }});
			last;
		}
	}
	
	if (not $anvil->data->{switches}{'job-uuid'})
	{
		# No job UUID.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "error_0073"});
		$anvil->nice_exit({exit_code => 2});
	}
	
	my $return = $anvil->Database->get_job_details({debug => 3, job_uuid => $anvil->data->{switches}{'job-uuid'}});
	if (not $return)
	{
		# No job details
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "error_0034", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }});
		$anvil->nice_exit({exit_code => 3});
	}
	
	$anvil->data->{jobs}{job_uuid}    = $anvil->data->{switches}{'job-uuid'};
	$anvil->data->{jobs}{job_command} = $return->{job_command};
	$anvil->data->{jobs}{job_data}    = $return->{job_data};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		'jobs::job_uuid'    => $anvil->data->{jobs}{job_uuid},
		'jobs::job_command' => $anvil->data->{jobs}{job_command},
		'jobs::job_data'    => $anvil->Log->is_secure($anvil->data->{jobs}{job_data}),
	}});
	
	foreach my $line (split/\n/, $anvil->data->{jobs}{job_data})
	{
		my $secure = $line =~ /passw/ ? 1 : 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => $secure, list => { line => $line }});
		
		my ($variable, $value) = ($line =~ /^(.*?)=(.*)$/);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
			variable => $variable,
			value    => $secure ? $anvil->Log->is_secure($value) : $value,
		}});
		
		$anvil->data->{data}{$variable} = $value;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"data::${variable}" => $secure ? $anvil->Log->is_secure($anvil->data->{data}{$variable}) : $anvil->data->{data}{$variable},
		}});
	}
	
	$anvil->data->{data}{say_target} = "root\@".$anvil->data->{data}{host_ip_address}.":".$anvil->data->{data}{ssh_port};
	
	# Update that we've picked the job up.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0027", variables => { 'job-uuid' => $anvil->data->{switches}{'job-uuid'} }});
	update_progress($anvil, 0, "clear");
	$anvil->data->{job}{progress} += 5;
	update_progress($anvil, $anvil->data->{job}{progress}, "job_0027,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!");
	
	return(0);
}

# If this is being called as a job, this will allow the progress to be updated.
sub update_progress
{
	my ($anvil, $progress, $message) = @_;

	$progress = 95 if $progress > 100;
	if (not $anvil->data->{switches}{'job-uuid'})
	{
		return(0);
	}
	
	$anvil->Job->update_progress({
		debug    => 3,
		progress => $progress, 
		message  => $message,
		job_uuid => $anvil->data->{switches}{'job-uuid'},
	});
	
	return(0);
}