#!/usr/bin/perl # # This updates the host system. # NOTE: This doesn't update the Anvil! software stack yet, just the OS. # # - On Striker; This will take the system offline and then run 'dnf -y update'. # - On Nodes; This will do nothing until all servers are off the node. Then the node will be withdrawn, # updated and then rejoin the cluster. # - On DR; This will do nothing until no servers are running, then it will update the system. # # In all cases, the system will be rebooted if the kernel is updated. # # Exit codes; # 0 = Normal exit. # 1 = No database connections available. # 2 = The job UUID was passed, but it wasn't valid. # 3 = It looks like the update failed, reset progress to '0'. # 4 = Failed to withdraw the node from the cluster. # # TODO: # - Rebuild this to be 'striker-update-system' and have it update local strikers and all nodes. # use strict; use warnings; use Anvil::Tools; # Disable buffering $| = 1; my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; if (($running_directory =~ /^\./) && ($ENV{PWD})) { $running_directory =~ s/^\./$ENV{PWD}/; } my $anvil = Anvil::Tools->new(); ### TODO: Remove this before final release $anvil->Log->level({set => 2}); $anvil->Log->secure({set => 1}); ########################################## # Read switches (target ([user@]host[:port]) and the file with the target's password. $anvil->Get->switches({list => [ "clear-cache", "no-db", "no-reboot", "reboot"], man => $THIS_FILE}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }}); # Connect to DBs. if ($anvil->data->{switches}{'no-db'}) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, secure => 0, key => "log_0743"}); # If there was a job-uuid, clear it. $anvil->data->{sys}{database}{connections} = 0; $anvil->data->{switches}{'job-uuid'} = ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::database::connections' => $anvil->data->{sys}{database}{connections}, 'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'}, }}); } else { $anvil->Database->connect; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); if (not $anvil->data->{sys}{database}{connections}) { # No databases, exit. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, secure => 0, key => "error_0003"}); $anvil->nice_exit({exit_code => 1}); } } if ($anvil->data->{switches}{'job-uuid'}) { # See if another instance is running. If so, sleep for 10 seconds and then exit. The other instance # could be the '--no-db' run we're about to clobber. my $pids = $anvil->System->pids({ ignore_me => 1, program_name => $THIS_FILE, }); my $other_instances = @{$pids}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { other_instances => $other_instances }}); if ($other_instances) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0233", variables => { program => $THIS_FILE }}); sleep 10; $anvil->nice_exit({exit_code => 0}); } # Load the job details. If anything is returned, there was a problem. my $return = $anvil->Job->get_job_details({job_uuid => $anvil->data->{switches}{'job-uuid'}}); if ($return) { # It's not a UUID. $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }}); $anvil->nice_exit({exit_code => 2}); } } # Clear any old runs. update_progress($anvil, 0, "clear"); # We'll keep a count of lines and packages to show the user. $anvil->data->{counts}{downloaded} = 0; $anvil->data->{counts}{installed} = 0; $anvil->data->{counts}{verified} = 0; $anvil->data->{counts}{lines} = 0; # Mark that we're starting update_progress($anvil, 1, "message_0058,!!downloaded!0!!,!!installed!0!!,!!verified!0!!,!!lines!0!!"); update_progress($anvil, 2, "message_0033"); # Make sure maintenance mode is enabled. $anvil->System->maintenance_mode({set => 1}) if $anvil->data->{sys}{database}{connections}; # Run the update run_os_update($anvil, 3); # If we had no database, try to reconnect now tha if (not $anvil->data->{sys}{database}{connections}) { # Start the anvil-daemon, the caller likely called without a DB because we're being updated by # striker-update-cluster, and so there will be a job waiting for us. $anvil->System->enable_daemon({now => 1, daemon => "anvil-daemon"}); $anvil->System->enable_daemon({now => 1, daemon => "scancore"}); $anvil->Database->connect; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); if ($anvil->data->{sys}{database}{connections}) { # If there's a job for us waiting, mark it as almost done. my $query = " SELECT job_uuid FROM jobs WHERE job_command LIKE '%".$THIS_FILE."%' AND job_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)." AND job_progress = 0 ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); my $job_uuid = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); if ($job_uuid) { $anvil->data->{jobs}{job_uuid} = $job_uuid; $anvil->data->{switches}{'job-uuid'} = $job_uuid; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'jobs::job_uuid' => $anvil->data->{jobs}{job_uuid}, 'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'}, }}); update_progress($anvil, 0, "clear"); update_progress($anvil, 90, "message_0324"); } } } # We're done updating my $reboot_needed = $anvil->System->reboot_needed({debug => 2}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot_needed => $reboot_needed }}); if ($reboot_needed) { if (not $anvil->data->{switches}{'no-reboot'}) { # Clear maintenance mode. $anvil->System->maintenance_mode({set => 0}) if $anvil->data->{sys}{database}{connections}; # Record that we're rebooting so that 'striker-update-cluster' knows to wait for a reboot. if ($anvil->data->{switches}{'job-uuid'}) { my $query = " UPDATE jobs SET job_data = 'rebooted', modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." WHERE job_uuid = ".$anvil->Database->quote($anvil->data->{switches}{'job-uuid'})." ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); } # Register a job to reboot. update_progress($anvil, 98, "message_0318"); # Record that we're going to reboot now. update_progress($anvil, 100, "message_0317"); if ($anvil->data->{sys}{database}{connections}) { my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ file => $THIS_FILE, line => __LINE__, job_command => $anvil->data->{path}{exe}{'anvil-manage-power'}." --reboot -y".$anvil->Log->switches, job_data => "", job_name => "reboot::system", job_title => "job_0009", job_description => "job_0006", job_progress => 0, job_status => "anvil_startup", }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); # If we got a job UUID, unlink the cache file. if (-e $anvil->data->{path}{data}{reboot_cache}) { unlink $anvil->data->{path}{data}{reboot_cache}; } } } else { # Record that a reboot is needed. update_progress($anvil, 100, "message_0039"); } } else { update_progress($anvil, 100, "message_0040"); } # Clear maintenance mode. $anvil->System->maintenance_mode({set => 0}) if $anvil->data->{sys}{database}{connections}; $anvil->nice_exit({exit_code => 0}); ############################################################################################################# # Functions # ############################################################################################################# # This updates the progress if we were called with a job UUID. sub update_progress { my ($anvil, $progress, $message) = @_; # Log the progress percentage. $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { progress => $progress, message => $message, "jobs::job_uuid" => $anvil->data->{jobs}{job_uuid}, }}); if ($progress > 100) { $progress = 100; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { progress => $progress }}); } if ($anvil->data->{jobs}{job_uuid}) { $anvil->Job->update_progress({ debug => 2, 'print' => 1, progress => $progress, message => $message, job_uuid => $anvil->data->{jobs}{job_uuid}, }); } return(0); } # This updates the OS. sub run_os_update { my ($anvil, $progress) = @_; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { progress => $progress }}); # This needs to be set to avoid warnings when called without a job-uuid. $anvil->data->{sys}{last_update} = 0; # Make sure that, if we're a node, we're out of the cluster. my $host_type = $anvil->Get->host_type(); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }}); if ($host_type eq "node") { # Call anvil-safe-stop update_progress($anvil, $progress++, "message_0314"); my $problem = $anvil->Cluster->parse_cib(); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); if (not $problem) { # Call anvil-safe-stop update_progress($anvil, $progress++, "message_0315"); my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code, }}); if ($return_code) { # Something went wrong, abort. update_progress($anvil, 100, "error_0420,!!return_code!".$return_code."!!,!!output!".$output."!!"); # Set the job_data to 'failed' so that striker-update-cluster' knows to abort. if ($anvil->data->{switches}{'job-uuid'}) { my $query = " UPDATE jobs SET job_data = 'failed', modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." WHERE job_uuid = ".$anvil->Database->quote($anvil->data->{switches}{'job-uuid'})." ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); } $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0035", variables => { output => $output } }); $anvil->nice_exit({exit_code => 4}); } } } # Should we clear the cache? if ($anvil->data->{switches}{'clear-cache'}) { # Yes. my $shell_call = $anvil->data->{path}{exe}{dnf}." clean all"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code, }}); update_progress($anvil, 5, "message_0316"); } # Before we start, do we need to remove our locally build DRBD kernel modules? my $package_changes = 0; my $remove_drbd_kmod = 0; my $shell_call = $anvil->data->{path}{exe}{dnf}." check-update"; open (my $file_handle, $shell_call." 2>&1 |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }}); while(<$file_handle>) { chomp; my $line = $anvil->Words->clean_spaces({string => $_}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); if (($line =~ /kmod-drbd/) or ($line =~ /kernel/)) { # Looks like it. $remove_drbd_kmod = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { remove_drbd_kmod => $remove_drbd_kmod }}); last; } } close $file_handle; # So, shall we? if ($remove_drbd_kmod) { # Yes, remove update_progress($anvil, $progress++, "message_0320"); my $versions_to_remove = ""; my $shell_call = $anvil->data->{path}{exe}{dnf}." list installed"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code, }}); foreach my $line (split/\n/, $output) { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); if ($line =~ /(kmod-drbd-\d+.*?)\s/) { $versions_to_remove .= $1." "; $package_changes++; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { versions_to_remove => $versions_to_remove, package_changes => $package_changes, }}); } } # Now remove those packages. update_progress($anvil, $progress++, "message_0321"); $shell_call = $anvil->data->{path}{exe}{dnf}." -y remove ".$versions_to_remove; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); open ($file_handle, $shell_call." 2>&1 |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }}); while(<$file_handle>) { chomp; my $line = $anvil->Words->clean_spaces({string => $_}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); } close $file_handle; } # NOTE: We run this directly to better monitor progress and update the progress. my $transaction_shown = 0; my $success = 0; my $to_update = 0; my $percent_step = 0; my $counted_lines = 0; my $next_step = 0; my $verifying = 0; my $output = ""; $shell_call = $anvil->data->{path}{exe}{dnf}." -y update; ".$anvil->data->{path}{exe}{echo}." return_code:\$?"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }}); open ($file_handle, $shell_call." 2>&1 |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }}); while(<$file_handle>) { chomp; my $line = $_; $output .= $line."\n"; $line = $anvil->Words->clean_spaces({string => $line}); $anvil->data->{counts}{lines}++; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "counts::lines" => $anvil->data->{counts}{lines}, line => $line }}); # If there were no updates, let the user know. if ($line =~ /^Nothing to do/i) { update_progress($anvil, 95, "message_0057"); } if ((not $verifying) && ($line =~ /^Verifying /i)) { # Update done, verifying now. $verifying = 1; update_progress($anvil, $progress++, "message_0038"); } if ($line =~ /Running transaction/i) { # Done downloading if (not $transaction_shown) { update_progress($anvil, $progress++, "message_0037"); $transaction_shown = 1; } } if ($line =~ /return_code:(\d+)$/) { my $return_code = $1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }}); if ($return_code == 0) { $success = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { success => $success }}); } } if (($line =~ / (\d+) Packages$/i) or ($line =~ / (\d+) Package$/i)) { my $counted_lines = $1; $package_changes = $counted_lines; $to_update += $counted_lines; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { package_changes => $package_changes, counted_lines => $counted_lines, to_update => $to_update, }}); } if ($line =~ /Total download size: (.*)$/i) { my $update_size = $1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update_size => $update_size }}); # Ready to install, update to 5%. The next step will count up to 95%. update_progress($anvil, $progress++, "message_0035,!!size!$update_size!!"); # The total (reliable) count of events is (to_update * 3), counting '(x/y): ' # (download), 'Upgrading '/'Installing ' and 'Verifying '. We ignore the scriplet # and other lines as it's hard to predict how many there will be, and they pass fast # enough to not really matter for a progress bar. $to_update *= 4; $percent_step = $anvil->Convert->round({number => ($to_update / 90)}); $next_step = $percent_step; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { to_update => $to_update, percent_step => $percent_step, next_step => $next_step, }}); } # If 'percent_step' is set, we're ready to start counting lines. if (($percent_step) && (($line =~ /\(\d+\/\d+\): /) or ($line =~ /^Upgrading /i) or ($line =~ /^Installing /) or ($line =~ /^Cleanup /i) or ($line =~ /^Verifying /i))) { $counted_lines++; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { counted_lines => $counted_lines }}); if ($line =~ /\(\d+\/\d+\): /) { $anvil->data->{counts}{downloaded}++; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "counts::downloaded" => $anvil->data->{counts}{downloaded} }}); } if (($line =~ /^Upgrading /i) or ($line =~ /^Installing /)) { $anvil->data->{counts}{installed}++; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "counts::installed" => $anvil->data->{counts}{installed} }}); } if ($line =~ /^Verifying /i) { $anvil->data->{counts}{verified}++; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "counts::verified" => $anvil->data->{counts}{verified} }}); } if ($counted_lines > $next_step) { # Step up the progress. $next_step += $percent_step; $progress++; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { next_step => $next_step, progress => $progress, }}); $progress = 95 if $progress > 95; update_progress($anvil, $progress, ""); } } # Update the progress if it's been more than a second since the last update. if (time > $anvil->data->{sys}{last_update}) { $progress = 95 if $progress > 95; update_progress($anvil, $progress, ""); } } close $file_handle; # Reload daemons to pick up any changed systemctl daemons. $shell_call = $anvil->data->{path}{exe}{systemctl}." daemon-reload"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); my ($systemctl_output, $return_code) = $anvil->System->call({shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { systemctl_output => $systemctl_output, return_code => $return_code }}); ### See if the kernel has been updated. # Get the newest installed kernel $shell_call = $anvil->data->{path}{exe}{rpm}." -q kernel | ".$anvil->data->{path}{exe}{'sort'}." | ".$anvil->data->{path}{exe}{tail}." -n 1"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); (my $installed_kernel, $return_code) = $anvil->System->call({shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { installed_kernel => $installed_kernel, return_code => $return_code, }}); $installed_kernel =~ s/^kernel-(\d+.\d+\.\d+-\d+)\..*$/$1/; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { installed_kernel => $installed_kernel }}); # Get the running kernel $shell_call = $anvil->data->{path}{exe}{uname}." -r"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); (my $active_kernel, $return_code) = $anvil->System->call({shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { active_kernel => $active_kernel, return_code => $return_code, }}); $active_kernel =~ s/(\d+.\d+\.\d+-\d+)\..*$/$1/; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { active_kernel => $active_kernel }}); if ($installed_kernel ne $active_kernel) { # Reboot needed $anvil->data->{sys}{reboot} = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::reboot' => $anvil->data->{sys}{reboot} }}); } # If we installed and packages, and '--reboot' was given, reboot anyway. if (($package_changes) && ($anvil->data->{switches}{reboot})) { # Reboot needed $anvil->data->{sys}{reboot} = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::reboot' => $anvil->data->{sys}{reboot} }}); my $reboot_needed = $anvil->System->reboot_needed({set => 1}); } # Did it work? if (not $success) { # Nope. update_progress($anvil, 0, "message_0036"); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0035", variables => { output => $output } }); sleep 5; $anvil->nice_exit({exit_code => 3}); } # Run anvil-version-changes. $shell_call = $anvil->data->{path}{exe}{'anvil-version-changes'}.$anvil->Log->switches; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code, }}); return(0); };