* Reworded striker-update-cluster to use anvil-update-system for on-system OS updates.

* Updated DRBD->get_status() to take the new 'host' paramter to allow the caller to define the hash key string used in the stored data.
* Updated Get->anvil_version() (and a few other places) to use the new 'striker-ui-api' shell user, replacing the 'apache' user.
* Updated Remote->test_access() to take the new 'close' parameter to close the SSH session used when testing access to the target.
* Fixed a logging bug in anvil-manage-power.
* Updated anvil-update-system to take the '--no-reboot' and 'clear-cache' command line switches.

Signed-off-by: digimer <mkelly@alteeve.ca>

Anvil/Tools.pm

@@ -851,9 +851,6 @@ sub _set_defaults
 		},
 	};
 	$anvil->data->{sys} = {
-		apache				=>	{
-			user				=>	"admin",
-		},
 		daemon				=>	{
 			dhcpd				=>	"dhcpd.service",
 			firewalld			=>	"firewalld.service",

Anvil/Tools/DRBD.pm

@@ -2146,6 +2146,10 @@ If any data for the host was stored in a previous call, it will be deleted befor
 
 Parameters;
 
+=head3 host (optional)
+
+By default, the hash key C<< host_name >> listed above is either the local system's short host name, or the C<< target >>. If you'd like to use a specific host name in the hash key, you can use this parameter to set it.
+
 =head3 password (optional)
 
 This is the password to use when connecting to a remote machine. If not set, but C<< target >> is, an attempt to connect without a password will be made.
@@ -2172,22 +2176,42 @@ sub get_status
 	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "DRBD->get_status()" }});
 	
+	my $host        = defined $parameter->{host}        ? $parameter->{host}        : "";
 	my $password    = defined $parameter->{password}    ? $parameter->{password}    : "";
 	my $port        = defined $parameter->{port}        ? $parameter->{port}        : "";
 	my $remote_user = defined $parameter->{remote_user} ? $parameter->{remote_user} : "root";
 	my $target      = defined $parameter->{target}      ? $parameter->{target}      : "";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		host        => $host, 
 		password    => $anvil->Log->is_secure($password),
 		port        => $port, 
 		remote_user => $remote_user, 
 		target      => $target, 
 	}});
 	
+	# If we weren't passed a host, use this machine's short host name.
+	my $is_local = $anvil->Network->is_local({host => $target});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { is_local => $is_local }});
+	if (not $host)
+	{
+		# Host not set, set one.
+		if ($is_local)
+		{
+			$host = $anvil->Get->short_host_name();
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host => $host }});
+		}
+		else
+		{
+			# Remote, using the target as the host.
+			$host = $target;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host => $host }});
+		}
+	}
+	
 	# Is this a local call or a remote call?
 	my $shell_call = $anvil->data->{path}{exe}{drbdsetup}." status --json";
 	my $output     = "";
-	my $host       = $anvil->Get->short_host_name();
-	my $is_local   = $anvil->Network->is_local({host => $target});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
 	if ($is_local)
 	{
 		# Local.
@@ -2200,7 +2224,6 @@ sub get_status
 	else
 	{
 		# Remote call.
-		$host = $target;
 		($output, my $error, $anvil->data->{drbd}{status}{$host}{return_code}) = $anvil->Remote->call({
 			debug       => $debug, 
 			shell_call  => $shell_call, 

Anvil/Tools/Get.pm

@@ -326,7 +326,7 @@ sub anvil_version
 			schema_cache_file => $schema_cache_file, 
 			user              => $user,
 		}});
-		if ($user eq "apache")
+		if (($user eq "apache") or ($user eq "striker-ui-api"))
 		{
 			# Try to read the local cached version.
 			if (-e $anvil_cache_file)
@@ -1867,8 +1867,8 @@ sub host_uuid
 					debug     => $debug, 
 					file      => $anvil->data->{path}{data}{host_uuid}, 
 					body      => $uuid,
-					user      => "apache", 
-					group     => "apache",
+					user      => "striker-ui-api", 
+					group     => "striker-ui-api",
 					mode      => "0666",
 					overwrite => 0,
 				});
@@ -2529,7 +2529,7 @@ sub switches
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { found => $found }});
 			if (not $found)
 			{
-				print "Switch '--".$set_switch." not recognized.\n";
+				print "Switch '--".$set_switch."' is not recognized.\n";
 				$problem = 1;
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }});
 			}

Anvil/Tools/Job.pm

@@ -756,29 +756,6 @@ WHERE
 		$job_status =~ s/message_0058,!!downloaded!.*?!!,!!installed!.*?!!,!!verified!.*?!!,!!lines!.*?!!/message_0058,!!downloaded!$downloaded!!,!!installed!$installed!!,!!verified!$verified!!,!!lines!$lines!!/sm;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "<< job_status" => $job_status }});
 	}
-	# This is used by 'anvil-download-file'
-	if ($job_status =~ /message_0142/gs)
-	{
-		### NOTE: Is this needed anymore?
-# 		my $downloaded = $anvil->data->{counts}{downloaded} ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{downloaded}}) : 0;
-# 		my $installed  = $anvil->data->{counts}{installed}  ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{installed}})  : 0;
-# 		my $verified   = $anvil->data->{counts}{verified}   ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{verified}})   : 0;
-# 		my $lines      = $anvil->data->{counts}{lines}      ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{lines}})      : 0;
-# 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-# 			"s1:counts::downloaded" => $anvil->data->{counts}{downloaded},
-# 			"s2:downloaded"         => $downloaded, 
-# 			"s3:counts::installed"  => $anvil->data->{counts}{installed},
-# 			"s4:installed"          => $installed, 
-# 			"s5:counts::verified"   => $anvil->data->{counts}{verified},
-# 			"s6:verified"           => $verified, 
-# 			"s7:counts::lines"      => $anvil->data->{counts}{lines},
-# 			"s8:lines"              => $lines, 
-# 		}});
-# 		
-# 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { ">> job_status" => $job_status }});
-# 		$job_status =~ s/message_0142,!!downloaded!.*?!!,!!installed!.*?!!,!!verified!.*?!!,!!lines!.*?!!/message_0058,!!downloaded!$downloaded!!,!!installed!$installed!!,!!verified!$verified!!,!!lines!$lines!!/sm;
-# 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "<< job_status" => $job_status }});
-	}
 	
 	$job_uuid = $anvil->Database->insert_or_update_jobs({
 		file                 => $THIS_FILE, 

Anvil/Tools/Network.pm

@@ -1212,7 +1212,7 @@ Paramters;
 
 =head3 target (required)
 
-This is the host we're looking for connection options with.
+This is the host (name or UUID) we're looking for connection options with.
 
 =cut
 sub find_access

Anvil/Tools/Remote.pm

@@ -311,6 +311,7 @@ sub call
 	# NOTE: The shell call might contain sensitive data, so we show '--' if 'secure' is set and $anvil->Log->secure is not.
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		'close'    => $close, 
+		no_cache   => $no_cache,
 		password   => $anvil->Log->is_secure($password), 
 		secure     => $secure, 
 		shell_call => (not $secure) ? $shell_call : $anvil->Log->is_secure($shell_call),
@@ -634,6 +635,19 @@ sub call
 		{
 			$error = $anvil->Words->string({key => $message_key, variables => $variables});
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => $message_key, variables => $variables});
+			
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 
+				'close'    => $close, 
+				password   => $anvil->Log->is_secure($password), 
+				secure     => $secure, 
+				shell_call => (not $secure) ? $shell_call : $anvil->Log->is_secure($shell_call),
+				ssh_fh     => $ssh_fh,
+				start_time => $start_time, 
+				timeout    => $timeout, 
+				port       => $port, 
+				target     => $target,
+				ssh_fh_key => $ssh_fh_key, 
+			}});
 		}
 	
 	}
@@ -676,6 +690,10 @@ sub call
 				error      => $ssh_fh->error,
 			}});
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { error => $error }});
+			
+			# Close the connection.
+			$close = 1;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { 'close' => $close }});
 		}
 		
 		# Take the last new line off.
@@ -923,6 +941,10 @@ This attempts to log into the target to verify that the target is up and reachab
 
 Parameters;
 
+=head3 close (optional, default '1')
+
+If set, the SSH connection used to test the access to the remote host wil be closed. This can be useful it there might be a delay between when the connecton is tested and when it is used again.
+
 =head3 password (optional)
 
 This is the password used to connect to the remote target as the given user.
@@ -950,12 +972,14 @@ sub test_access
 	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Remote->test_access()" }});
 	
+	my $close    = defined $parameter->{'close'}  ? $parameter->{'close'}  : 1;
 	my $password = defined $parameter->{password} ? $parameter->{password} : "";
 	my $port     = defined $parameter->{port}     ? $parameter->{port}     : 22;
 	my $target   = defined $parameter->{target}   ? $parameter->{target}   : "";
 	my $user     = defined $parameter->{user}     ? $parameter->{user}     : getpwuid($<); 
 	my $access   = 0;
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => 0, list => { 
+		'close'  => $close,
 		password => $anvil->Log->is_secure($password), 
 		port     => $port, 
 		target   => $target,
@@ -969,7 +993,7 @@ sub test_access
 		shell_call  => $anvil->data->{path}{exe}{echo}." 1", 
 		target      => $target,
 		remote_user => $user, 
-		'close'     => 1,
+		'close'     => $close,
 		no_cache    => 1,
 	});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 

Anvil/Tools/Storage.pm

@@ -452,7 +452,7 @@ sub change_mode
 
 This changes the owner and/or group of a file or directory.
 
- $anvil->Storage->change_owner({path => "/tmp/foo", user => "apache", group => "apache" });
+ $anvil->Storage->change_owner({path => "/tmp/foo", user => "striker-ui-api", group => "striker-ui-api" });
 
 If it fails to write the file, an alert will be logged and 'C<< 1 >>' will be returned. Otherwise, 'C<< 0 >>' will be returned.
 
@@ -4972,11 +4972,11 @@ sub update_config
 			body        => $new_file,
 			debug       => $debug,
 			file        => $anvil->data->{path}{configs}{'anvil.conf'},
-			group       => "apache", 
+			group       => "striker-ui-api", 
 			mode        => "0640",
 			overwrite   => 1,
 			secure      => 1,
-			user        => "apache",
+			user        => "striker-ui-api",
 			password    => $password, 
 			port        => $port, 
 			target      => $target, 

man/Makefile.am

@@ -18,6 +18,7 @@ dist_man8_MANS		= \
 			  anvil-manage-dr.8 \
 			  anvil-manage-files.8 \
 			  anvil-manage-keys.1 \
+			  anvil-manage-power.8 \
 			  anvil-manage-server.8 \
 			  anvil-manage-server-storage.8 \
 			  anvil-manage-storage-groups.8 \
@@ -27,4 +28,4 @@ dist_man8_MANS		= \
 			  striker-check-machines.8 \
 			  striker-collect-debug.8 \
 			  striker-initialize-host.8 \
-			  striker-update-cluster
+			  striker-update-cluster.8 

man/anvil-boot-server.8

@@ -40,7 +40,7 @@ This is the server UUID of the server to boot. Generally this isn't needed, exce
 \fB\-\-wait\fR
 When using '\fB\-\-server\fR all', the request to boot each server will normally not wait for the server to boot. When this is set, this behaviour is changed and the boot will wait before moving on to boot the next server.
 .TP
-Be away that when this is used, if a server fails to boot, no further servers will be started.
+Be aware that when this is used, if a server fails to boot, no further servers will be started.
 .IP
 .SH AUTHOR
 Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.

man/anvil-manage-power.8

@@ -0,0 +1,45 @@
+.\" Manpage for the Anvil! power management tool
+.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
+.TH anvil-manage-power "8" "July 11 2023" "Anvil! Intelligent Availability™ Platform"
+.SH NAME
+anvil-manage-power \- This program can power off, reboot, or set a flag indicating one of these actions are required.
+.SH SYNOPSIS
+.B anvil-manage-power 
+\fI\,<command> \/\fR[\fI\,options\/\fR]
+.SH DESCRIPTION
+This program can mark a machine as needing to be powered off or rebooted, or perform those actions directly or as a job. 
+.TP
+\-?, \-h, \fB\-\-help\fR
+Show this man page.
+.TP
+\fB\-\-log-secure\fR
+When logging, record sensitive data, like passwords.
+.TP
+\-v, \-vv, \-vvv
+Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
+.SS "Commands:"
+.TP
+\fB\-\-no-wait\fR
+.TP
+Normally, this program will not reboot a machine until the uptime is over five minutes. This is done to provide a chance for someone to log in and disable anvil-daemon in the case of a reboot loop. This switch prevents waiting for that 5 minute delay.
+.TP
+\fB\-\-poweroff\fR, \fB\-\-power\-off\fR
+.TP
+This powers off the host.
+.TP
+\fB\-\-reboot\fR
+.TP
+This reboots the host. 
+.TP
+\fB\-\-reboot\-needed\fR [0,1]
+.TP
+This sets (1) or clears (0) the 'reboot needed' flag for the host system. 
+.TP
+\fB\-\-y\fR, \fB\-\-yes\fR
+.TP
+If passed, requests to reboot or power off won't ask for confirmation.
+.IP
+.SH AUTHOR
+Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
+.SH "REPORTING BUGS"
+Report bugs to users@clusterlabs.org

man/anvil-update-system.8

@@ -0,0 +1,39 @@
+.\" Manpage for the Anvil! cluster update tool.
+.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
+.TH anvil-update-system "8" "July 14 2023" "Anvil! Intelligent Availability™ Platform"
+.SH NAME
+anvil-update-system \- This program updates the local operting system
+.SH SYNOPSIS
+.B anvil-update-system 
+\fI\,<command> \/\fR[\fI\,options\/\fR]
+.SH DESCRIPTION
+This program updates the local operating system. If the kernel is updated, a reboot will be performed.
+.TP
+.B Note:
+.TP
+If the host is an Anvil! subnode, the subnode will be removed from the Anvil! node (and servers migrated off, or, shut down if the peer subnode is offline).
+.TP
+.SH OPTIONS
+.TP
+\-?, \-h, \fB\-\-help\fR
+Show this man page.
+.TP
+\fB\-\-log-secure\fR
+When logging, record sensitive data, like passwords.
+.TP
+\-v, \-vv, \-vvv
+Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
+.SS "Commands:"
+.TP
+\fB\-\-clear\-cache\fR
+.TP
+This will force the dnf cache to be cleared before the OS update is started. This slows the update down a bit, but ensures the latest updates are installed.
+.TP
+\fB\-\-no\-reboot\fR
+.TP
+If the kernel is updated, the system will normally be rebooted. This switch prevents the reboot from occuring.
+.IP
+.SH AUTHOR
+Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
+.SH "REPORTING BUGS"
+Report bugs to users@clusterlabs.org

man/striker-update-cluster.8

@@ -0,0 +1,53 @@
+.\" Manpage for the Anvil! cluster update tool.
+.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
+.TH striker-update-cluster "8" "July 11 2023" "Anvil! Intelligent Availability™ Platform"
+.SH NAME
+striker-update-cluster \- This program updates all physical machines in an Anvil! cluster
+.SH SYNOPSIS
+.B striker-update-cluster 
+\fI\,<command> \/\fR[\fI\,options\/\fR]
+.SH DESCRIPTION
+This program sequentially updates Striker dashboards, DR hosts and Anvil! nodes (the paired sub-nodes). It does this without needing to take hosted servers offline.
+.TP
+.B Note:
+.TP
+This program requires all machines be online, and Anvil! nodes being paired and sync'ed. When nodes are updated, the inactive subnode will be removed from the node, updated, rebooted if necessary, and then rejoined to the node. Then hosted servers will migrate to the now-updated subnode, and the process repeated for the other subnode. Anvil! nodes are updated sequentially, so the process can take some time to complete, but should not require a maintenance window.
+.TP
+The upgrade process will live-migrate all hosted servers! If any hosted server is either under heavy load, or the replication link (the BCN or MN) is relatively lower bandwidth, this could cause performance concerns. As such, it's ideal to run the upgrades at a time less sensitive to performance impacts.
+.TP
+.SH OPTIONS
+.TP
+\-?, \-h, \fB\-\-help\fR
+Show this man page.
+.TP
+\fB\-\-log-secure\fR
+When logging, record sensitive data, like passwords.
+.TP
+\-v, \-vv, \-vvv
+Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
+.SS "Commands:"
+.TP
+\fB\-\-clear\-cache\fR
+.TP
+This will force the dnf cache to be cleared before the OS update is started. This slows the update down a bit, but ensures the latest updates are installed.
+.TP
+\fB\-\-force\fR
+.TP
+If any Striker dashboards or DR hosts are unavailable, or if an entire node (paired subnodes) is offline, this switch will allow you to force the upgrade attempt.
+.TP
+\fB\-y\fR, \fB\-\-yes\fR
+.TP
+Automatically continue with the upgrade without prompting for confirmation.
+.TP
+\fB\-\-no\-reboot\fR
+.TP
+If the kernel is updated on a remote system, the system will normally be rebooted. This switch prevents the reboot from occuring.
+.TP
+\fB\-\-reboot\-self\fR
+.TP
+By default, if the local system needs to be updated, a message is printed but the local system is NOT rebooted. This switch will instead cause this host to reboot at the end of the cluster update.
+.IP
+.SH AUTHOR
+Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
+.SH "REPORTING BUGS"
+Report bugs to users@clusterlabs.org

share/words.xml

@@ -1555,6 +1555,8 @@ Note: This is a permanent action! If you protect this server again later, a full
 		<key name="job_0464">Calling select ScanCore scan agents to ensure the database is updated.</key>
 		<key name="job_0465">Reload (adjust) a DRBD resource</key>
 		<key name="job_0466">This job is to reload (adjust) a DRBD resource. It's run as a job as it blocks until the adjust is run on all nodes.</key>
+		<key name="job_0467">Update the base operating system.</key>
+		<key name="job_0468">This uses 'dnf' to do an OS update on the host. If this is run on a node, 'anvil-safe-stop' will be called to withdraw the subnode from the node's cluster. If the peer subnode is also offline, hosted servers will be shut down.</key>
 
 		<!-- Log entries -->
 		<key name="log_0001">Starting: [#!variable!program!#].</key>
@@ -2408,7 +2410,10 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
 		<key name="message_0005">There was an unknown error while connecting as: [#!variable!user!#] to: [#!variable!remote_user!#@#!variable!target!#]. The error was: [#!variable!error!#]</key>
 		<key name="message_0006">We were unable to log in to: [#!variable!connection!#]. Please check that the password is correct or that passwordless SSH is configured properly.</key>
 		<key name="message_0007">An SSH session was successfully opened to: [#!variable!target!#].</key>
-		<key name="message_0008">The remote shell call: [#!variable!shell_call!#] to: [#!variable!connection!#] failed with the error: [#!variable!error!#].</key>
+		<key name="message_0008">The remote shell call: [#!variable!shell_call!#] to: [#!variable!connection!#] failed with the error:
+====
+#!variable!error!#
+====</key>
 		<key name="message_0009">The SSH session to: [#!variable!target!#] was successfully closed.</key>
 		<key name="message_0010">The SSH session to: [#!variable!target!#] was closed because 'no_cache' was set and there was an open SSH connection.</key>
 		<key name="message_0011">Wrote the system UUID to the file: [#!variable!file!#] to enable the web based tools to read this system's UUID.</key>
@@ -2899,6 +2904,10 @@ Proceed? [y/N]</key>
 		<key name="message_0311">Picked up the special operation job.</key>
 		<key name="message_0312">Reloading (adjusting) the DRBD resource: [#!variable!resource!#]. This will not complete until all peers have also reloaded this resource.</key>
 		<key name="message_0313">DRBD resource: [#!variable!resource!#] has been reloaded.</key>
+		<key name="message_0314">Checking if the subnode is out of the node's cluster before updating the OS.</key>
+		<key name="message_0315">The subnode is in the node's cluster, asking it to withdraw. This could take some time if servers need to be migrated.</key>
+		<key name="message_0316">Cleared 'dnf' cache.</key>
+		<key name="message_0317">The kernel was updated, so a reboot is required. Rebooting now.</key>
 
 		<!-- Translate names (protocols, etc) -->
 		<key name="name_0001">Normal Password</key>	<!-- none in mail-server -->

tools/anvil-manage-power

@@ -34,17 +34,15 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 
 my $anvil = Anvil::Tools->new();
 
-# Read switches
-$anvil->data->{switches}{'poweroff'}      = "";
-$anvil->data->{switches}{'power-off'}     = "";
-$anvil->data->{switches}{'reboot'}        = "";
-$anvil->data->{switches}{'y'}             = "";
-$anvil->data->{switches}{'yes'}           = "";
-$anvil->data->{switches}{'reboot-needed'} = "";
-$anvil->data->{switches}{'job-uuid'}      = "";
-$anvil->data->{switches}{'no-delay'}      = "";
-$anvil->Get->switches;
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
+# Read switches (target ([user@]host[:port]) and the file with the target's password.
+$anvil->Get->switches({list => [
+	"no-wait", 
+	"power-off",
+	"poweroff", 
+	"reboot",
+	"reboot-needed"], man => $THIS_FILE});
+$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 
 if ($anvil->data->{switches}{'power-off'})
 {
@@ -191,7 +189,7 @@ sub do_poweroff
 	# We'll wait until the system has at least 5 minutes of uptime, unless '--no-wait' was given.
 	my $uptime = $anvil->data->{switches}{'no-wait'} ? 0 : $anvil->Get->uptime;
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-		"switches::no-wait" => $anvil->data->{switches}{'no-delay'}, 
+		"switches::no-wait" => $anvil->data->{switches}{'no-wait'}, 
 		uptime              => $uptime,
 	}});
 	

tools/anvil-update-system

@@ -36,10 +36,12 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 
 my $anvil = Anvil::Tools->new();
 
-# Read switches
-$anvil->data->{switches}{'job-uuid'} = "";
-$anvil->Get->switches;
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
+# Read switches (target ([user@]host[:port]) and the file with the target's password.
+$anvil->Get->switches({list => [
+	"clear-cache",
+	"no-reboot"], man => $THIS_FILE});
+$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 
 # Log that we've started.
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
@@ -90,7 +92,32 @@ my $reboot_needed = $anvil->System->reboot_needed({debug => 2});
 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot_needed => $reboot_needed }});
 if ($reboot_needed)
 {
-	update_progress($anvil, 100, "message_0039");
+	if (not $anvil->data->{switches}{'no-reboot'})
+	{
+		# Record that we're going to reboot now.
+		update_progress($anvil, 100, "message_0317");
+
+		# Clear maintenance mode.
+		$anvil->System->maintenance_mode({set => 0});
+		
+		# Record that we're rebooting so that 'striker-update-cluster' knows to wait for a reboot.
+		my $query = "UPDATE jobs SET job_status = 'rebooted', modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." WHERE job_uuid = ".$anvil->Database->quote($job_uuid).";";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
+		$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
+		
+		sleep 2;
+		my $shell_call = $anvil->data->{path}{exe}{systemctl}." reboot";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+		
+		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
+		$anvil->nice_exit({exit_code => 0});
+	}
+	else
+	{
+		# Record that a reboot is needed.
+		update_progress($anvil, 100, "message_0039");
+	}
 }
 else
 {
@@ -140,17 +167,57 @@ sub run_os_update
 	# This needs to be set to avoid warnings when called without a job-uuid.
 	$anvil->data->{sys}{last_update} = 0;
 	
+	# Make sure that, if we're a node, we're out of the cluster.
+	my $host_type = $anvil->Get->host_type();
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }});
+	if ($host_type eq "node")
+	{
+		# Call anvil-safe-stop
+		update_progress($anvil, 3, "message_0314");
+		
+		my $problem = $anvil->Cluster->parse_cib({debug => 3});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
+		if (not $problem)
+		{
+			# Call anvil-safe-stop
+			update_progress($anvil, 4, "message_0315");
+			
+			my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+			
+			my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				output      => $output, 
+				return_code => $return_code, 
+			}});
+		}
+	}
+	
+	# Should we clear the cache?
+	if ($anvil->data->{switches}{'clear-cache'})
+	{
+		# Yes.
+		my $shell_call  = $anvil->data->{path}{exe}{dnf}." clean all";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			output      => $output, 
+			return_code => $return_code, 
+		}});
+		update_progress($anvil, 5, "message_0316");
+	}
+	
 	# NOTE: We run this directly to better monitor progress and update the progress.
 	my $transaction_shown = 0;
 	my $success           = 0;
 	my $to_update         = 0;
 	my $percent_step      = 0;
-	my $progress          = 5;
+	my $progress          = 6;
 	my $counted_lines     = 0;
 	my $next_step         = 0;
 	my $verifying         = 0;
 	my $output            = "";
-	my $shell_call        = $anvil->data->{path}{exe}{dnf}." clean expire-cache && ".$anvil->data->{path}{exe}{dnf}." -y update --best --allowerasing; ".$anvil->data->{path}{exe}{echo}." return_code:\$?";
+	my $shell_call        = $anvil->data->{path}{exe}{dnf}." -y update --best --allowerasing; ".$anvil->data->{path}{exe}{echo}." return_code:\$?";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
 	open (my $file_handle, $shell_call." 2>&1 |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }});
 	while(<$file_handle>)
@@ -162,14 +229,6 @@ sub run_os_update
 		   $anvil->data->{counts}{lines}++;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "counts::lines" => $anvil->data->{counts}{lines}, line => $line }});
 		
-		if ($line =~ /^kernel /)
-		{
-			# Reboot will be needed.
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0687", variables => { reason => "#!string!log_0690!#" }});
-			my $reboot_needed = $anvil->System->reboot_needed({set => 1});
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { reboot_needed => $reboot_needed }});
-		}
-		
 		# If there were no updates, let the user know.
 		if ($line =~ /^Nothing to do/i)
 		{
@@ -286,6 +345,37 @@ sub run_os_update
 	my ($systemctl_output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{systemctl}." daemon-reload", source => $THIS_FILE, line => __LINE__});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { systemctl_output => $systemctl_output, return_code => $return_code }});
 	
+	### See if the kernel has been updated.
+	# Get the newest installed kernel
+	my $shell_call = $anvil->data->{path}{exe}{rpm}." -q kernel | ".$anvil->data->{path}{exe}{'sort'}." | ".$anvil->data->{path}{exe}{tail}." -n 1";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+	my ($installed_kernel, $return_code) = $anvil->System->call({shell_call => $shell_call});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		installed_kernel => $installed_kernel, 
+		return_code      => $return_code, 
+	}});
+	$installed_kernel =~ s/^kernel-(\d+.\d+\.\d+-\d+)\..*$/$1/;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { installed_kernel => $installed_kernel }});
+	
+	# Get the running kernel
+	$shell_call  = $anvil->data->{path}{exe}{uname}." -r";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+	(my $active_kernel, $return_code) = $anvil->System->call({shell_call => $shell_call});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		active_kernel => $active_kernel, 
+		return_code   => $return_code, 
+	}});
+	$active_kernel =~ s/(\d+.\d+\.\d+-\d+)\..*$/$1/;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { active_kernel => $active_kernel }});
+	
+	if ($installed_kernel ne $active_kernel)
+	{
+		# Reboot needed
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0687", variables => { reason => "#!string!log_0690!#" }});
+		my $reboot_needed = $anvil->System->reboot_needed({set => 1});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot_needed => $reboot_needed }});
+	}
+	
 	# Did it work?
 	if (not $success)
 	{