anvil/tools/anvil-update-system

#!/usr/bin/perl
# 
# This updates the host system. 
# NOTE: This doesn't update the Anvil! software stack yet, just the OS.
# 
# - On Striker; This will take the system offline and then run 'dnf -y update'. 
# - On Nodes; This will do nothing until all servers are off the node. Then the node will be withdrawn, 
#   updated and then rejoin the cluster.
# - On DR; This will do nothing until no servers are running, then it will update the system.
#  
# In all cases, the system will be rebooted if the kernel is updated.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = No database connections available.
# 2 = The job UUID was passed, but it wasn't valid.
# 3 = It looks like the update failed, reset progress to '0'.
# 

use strict;
use warnings;
use Anvil::Tools;

# Disable buffering
$| = 1;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});

$anvil->Storage->read_config({file => "/etc/anvil/anvil.conf"});

# Read switches
$anvil->data->{switches}{'job-uuid'}  = "";
$anvil->Get->switches;

# Log that we've started.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});

# Connect to DBs.
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, exit.
	print $anvil->Words->string({key => "error_0003"})."\n";
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0003"});
	$anvil->nice_exit({exit_code => 1});
}

# Did we get called with a job UUID? If not, try to find a pending job and take it.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }});
if (not $anvil->data->{switches}{'job-uuid'})
{
	# See if a job is waiting to run.
	my $query = "
SELECT 
    job_uuid 
FROM 
    jobs 
WHERE 
    job_command LIKE '".$THIS_FILE."%'
AND 
    job_progress != '100'
AND 
    job_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->Get->host_uuid)." 
;";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
	my $count   = @{$results};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		results => $results, 
		count   => $count, 
	}});
	if ($count == 1)
	{
		# Found it
		$anvil->data->{jobs}{job_uuid} = defined $results->[0]->[0] ? $results->[0]->[0] : "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }});
	}
}
if ($anvil->data->{switches}{'job-uuid'})
{
	# Is it set and valid?
	if (not $anvil->Validate->is_uuid({uuid => $anvil->data->{switches}{'job-uuid'}}))
	{
		# It's not a UUID.
		print $anvil->Words->string({key => "error_0033", variables => { uuid => $anvil->data->{switches}{'job-uuid'} }})."\n";
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0033", variables => { uuid => $anvil->data->{switches}{'job-uuid'} } });
		$anvil->nice_exit({code => 2});
	}
	
	# If I'm here, see if we can read the job details.
	my $query = "
SELECT 
    job_host_uuid, 
    job_data, 
    job_updated, 
    job_name,
    job_status 
FROM 
    jobs 
WHERE 
    job_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{switches}{'job-uuid'})." 
;";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
	
	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
	my $count   = @{$results};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		results => $results, 
		count   => $count, 
	}});
	if ($count < 1)
	{
		print $anvil->Words->string({key => "error_0034", variables => { uuid => $anvil->data->{switches}{'job-uuid'} }})."\n";
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0034", variables => { uuid => $anvil->data->{switches}{'job-uuid'} } });
		$anvil->nice_exit({code => 2});
	}
	
	# If we're here, we're good. Load the details
	$anvil->data->{jobs}{job_uuid}      = $anvil->data->{switches}{'job-uuid'};
	$anvil->data->{jobs}{job_host_uuid} = $results->[0]->[0];
	$anvil->data->{jobs}{job_data}      = $results->[0]->[1];
	$anvil->data->{jobs}{job_updated}   = $results->[0]->[2];
	$anvil->data->{jobs}{job_name}      = $results->[0]->[3];
	$anvil->data->{jobs}{job_status}    = $results->[0]->[4];
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"jobs::job_uuid"      => $anvil->data->{jobs}{job_uuid}, 
		"jobs::job_host_uuid" => $anvil->data->{jobs}{job_host_uuid},
		"jobs::job_data"      => $anvil->data->{jobs}{job_data}, 
		"jobs::job_updated"   => $anvil->data->{jobs}{job_updated}, 
		"jobs::job_name"      => $anvil->data->{jobs}{job_name}, 
		"jobs::job_status"    => $anvil->data->{jobs}{job_status}, 
	}});
}

# Clea any old runs.
$anvil->data->{sys}{last_update} = time;
update_progress($anvil, 0, "clear");

# We'll keep a count of lines and packages to show the user.
$anvil->data->{counts}{downloaded} = 0;
$anvil->data->{counts}{installed}  = 0;
$anvil->data->{counts}{verified}   = 0;
$anvil->data->{counts}{lines}      = 0;

# Mark that we're starting
update_progress($anvil, 1, "message_0058,!!downloaded!0!!,!!installed!0!!,!!verified!0!!,!!lines!0!!");
update_progress($anvil, 2, "message_0033");

# Make sure maintenance mode is enabled.
$anvil->System->maintenance_mode({debug => 3, set => 1});

# Run the update
run_os_update($anvil);

# We're done updating
my $reboot_needed = $anvil->System->reboot_needed({debug => 2, set => 1});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot_needed => $reboot_needed }});
if ($reboot_needed)
{
	update_progress($anvil, 100, "message_0039");
}
else
{
	update_progress($anvil, 100, "message_0040");
}

# Clear maintenance mode.
$anvil->System->maintenance_mode({debug => 3, set => 0});

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This updates the progress if we were called with a job UUID.
sub update_progress
{
	my ($anvil, $progress, $message) = @_;
	
	# Log the progress percentage.
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		progress         => $progress,
		message          => $message, 
		"jobs::job_uuid" => $anvil->data->{jobs}{job_uuid}, 
	}});
	
	if ($anvil->data->{jobs}{job_uuid})
	{
		# Get the current job_status and append this new one.
		my $job_picked_up_by = $$;
		my $job_picked_up_at = 0;
		my $job_status       = "";
		if ($message eq "clear")
		{
			$job_picked_up_by = 0;
		}
		else
		{
			my $query = "
SELECT 
    job_status, 
    job_picked_up_at
FROM 
    jobs 
WHERE 
    job_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{jobs}{job_uuid})."
;";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
			
			$job_status       = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
			$job_picked_up_at = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[1];
			$job_status       = "" if not defined $job_status;
			$job_picked_up_at = 0  if not defined $job_picked_up_at;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				job_status       => $job_status,
				job_picked_up_at => $job_picked_up_at,
			}});
			
			# Set that the job is now picked up if the progress is '1' or it 'job_picked_up_at' 
			# is not set yet.
			if ((not $job_picked_up_at) or ($progress eq "1"))
			{
				$job_picked_up_at = time;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_picked_up_at => $job_picked_up_at }});
			}
			
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { message => $message }});
			if (($message) && ($job_status))
			{
				$job_status .= "\n";
			}
			if ($message)
			{
				$job_status .= $message;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_status => $job_status }});
			}
		}
		
		# Insert counts
		if ($job_status =~ /message_0058/gs)
		{
			my $downloaded = $anvil->data->{counts}{downloaded} ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{downloaded}}) : 0;
			my $installed  = $anvil->data->{counts}{installed}  ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{installed}})  : 0;
			my $verified   = $anvil->data->{counts}{verified}   ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{verified}})   : 0;
			my $lines      = $anvil->data->{counts}{lines}      ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{lines}})      : 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"s1:counts::downloaded" => $anvil->data->{counts}{downloaded},
				"s2:downloaded"         => $downloaded, 
				"s3:counts::installed"  => $anvil->data->{counts}{installed},
				"s4:installed"          => $installed, 
				"s5:counts::verified"   => $anvil->data->{counts}{verified},
				"s6:verified"           => $verified, 
				"s7:counts::lines"      => $anvil->data->{counts}{lines},
				"s8:lines"              => $lines, 
			}});
			
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ">> job_status" => $job_status }});
			$job_status =~ s/message_0058,!!downloaded!.*?!!,!!installed!.*?!!,!!verified!.*?!!,!!lines!.*?!!/message_0058,!!downloaded!$downloaded!!,!!installed!$installed!!,!!verified!$verified!!,!!lines!$lines!!/sm;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "<< job_status" => $job_status }});
		}
		
		my $query = "
UPDATE 
    jobs 
SET 
    job_picked_up_by = ".$anvil->data->{sys}{database}{use_handle}->quote($job_picked_up_by).", 
    job_picked_up_at = ".$anvil->data->{sys}{database}{use_handle}->quote($job_picked_up_at).",
    job_updated      = ".$anvil->data->{sys}{database}{use_handle}->quote(time).",
    job_progress     = ".$anvil->data->{sys}{database}{use_handle}->quote($progress).", 
    job_status       = ".$anvil->data->{sys}{database}{use_handle}->quote($job_status).", 
    modified_date    = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})." 
WHERE 
    job_uuid         = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{jobs}{job_uuid})." 
";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
		$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
		
		# Note this update time
		$anvil->data->{sys}{last_update} = time;
	}
	
	return(0);
}

# This updates the OS.
sub run_os_update
{
	my ($anvil) = @_;
	
	# NOTE: We run this directly to better monitor progress and update the progress.
	my $transaction_shown = 0;
	my $success           = 0;
	my $to_update         = 0;
	my $percent_step      = 0;
	my $progress          = 5;
	my $counted_lines     = 0;
	my $next_step         = 0;
	my $verifying         = 0;
	my $output            = "";
	my $shell_call = $anvil->data->{path}{exe}{dnf}." clean expire-cache && ".$anvil->data->{path}{exe}{dnf}." -y update; ".$anvil->data->{path}{exe}{echo}." return_code:\$?";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	open (my $file_handle, $shell_call." 2>&1 |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }});
	while(<$file_handle>)
	{
		chomp;
		my $line   =  $_;
		   $output .= $line."\n";
		   $line   =  $anvil->Words->clean_spaces({string => $line});
		   $anvil->data->{counts}{lines}++;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "counts::lines" => $anvil->data->{counts}{lines}, line => $line }});
		
		if ($line =~ /^kernel /)
		{
			# Reboot will be needed.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0199"});
			
			my $reboot_needed = $anvil->System->reboot_needed({debug => 2, set => 1});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot_needed => $reboot_needed }});
		}
		
		# If there were no updates, let the user know.
		if ($line =~ /^Nothing to do/i)
		{
			update_progress($anvil, 95, "message_0057");
		}
		
		if ((not $verifying) && ($line =~ /^Verifying /i))
		{
			# Update done, verifying now.
			$verifying = 1;
			update_progress($anvil, $progress, "message_0038");
		}
		
		if ($line =~ /Running transaction/i)
		{
			# Done downloading
			if (not $transaction_shown)
			{
				update_progress($anvil, $progress, "message_0037");
				$transaction_shown = 1;
			}
		}
		
		if ($line =~ /return_code:(\d+)$/)
		{
			my $return_code = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
			if ($return_code == 0)
			{
				$success = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { success => $success }});
			}
		}
		
		if ($line =~ / (\d+) Packages$/i)
		{
			my $counted_lines =  $1;
			   $to_update     += $counted_lines;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				counted_lines => $counted_lines,
				to_update     => $to_update, 
			}});
		}
		
		if ($line =~ /Total download size: (.*)$/i)
		{
			my $update_size = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update_size => $update_size }});
			
			# Ready to install, update to 5%. The next step will count up to 95%.
			update_progress($anvil, $progress, "message_0035,!!size!$update_size!!");
			
			# The total (reliable) count of events is (to_update * 3), counting '(x/y): ' 
			# (download), 'Upgrading '/'Installing ' and 'Verifying '. We ignore the scriplet
			# and other lines as it's hard to predict how many there will be, and they pass fast
			# enough to not really matter for a progress bar.
			$to_update    *= 4;
			$percent_step =  $anvil->Convert->round({number => ($to_update / 90)});
			$next_step    =  $percent_step;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				to_update    => $to_update, 
				percent_step => $percent_step, 
				next_step    => $next_step, 
			}});
		}
		
		# If 'percent_step' is set, we're ready to start counting lines.
		if (($percent_step) && (($line =~ /\(\d+\/\d+\): /) or ($line =~ /^Upgrading /i) or ($line =~ /^Installing /) or ($line =~ /^Cleanup /i) or ($line =~ /^Verifying /i)))
		{
			$counted_lines++;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { counted_lines => $counted_lines }});
			
			if ($line =~ /\(\d+\/\d+\): /)
			{
				$anvil->data->{counts}{downloaded}++;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "counts::downloaded" => $anvil->data->{counts}{downloaded} }});
			}
			if (($line =~ /^Upgrading /i) or ($line =~ /^Installing /))
			{
				$anvil->data->{counts}{installed}++;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "counts::installed" => $anvil->data->{counts}{installed} }});
			}
			if ($line =~ /^Verifying /i)
			{
				$anvil->data->{counts}{verified}++;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "counts::verified" => $anvil->data->{counts}{verified} }});
			}
			
			if ($counted_lines > $next_step)
			{
				# Step up the progress.
				$next_step += $percent_step;
				$progress++;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
					next_step => $next_step,
					progress  => $progress, 
				}});
				next if $progress > 95;
				
				update_progress($anvil, $progress, "");
			}
		}
		
		# Update the progress if it's been more than a second since the last update.
		if (time > $anvil->data->{sys}{last_update})
		{
			update_progress($anvil, $progress, "");
		}
	}
	close $file_handle;
	
	# Reload daemons to pick up any changed systemctl daemons.
	my $systemctl_output = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{systemctl}." daemon-reload", source => $THIS_FILE, line => __LINE__});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { systemctl_output => $systemctl_output }});
	
	# Did it work?
	if (not $success)
	{
		# Nope.
		update_progress($anvil, 0, "message_0036");
		
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0035", variables => { output => $output } });
		$anvil->nice_exit({code => 3});
	}
	
	return(0);
};