* Updated anvil-update-system to detect kmod-drbd upgrade problems and fix them.

* Updated striker-update-cluster and anvil-update-system to take '--reboot' to request a reboot if any packages are updated.

Signed-off-by: digimer <mkelly@alteeve.ca>

man/anvil-update-system.8

@@ -32,6 +32,10 @@ This will force the dnf cache to be cleared before the OS update is started. Thi
 \fB\-\-no\-reboot\fR
 .TP
 If the kernel is updated, the system will normally be rebooted. This switch prevents the reboot from occuring.
+.TP
+\fB\-\-reboot\fR
+.TP
+Normally, the system will only reboot if the kernel is updated. If this is used, and if any packages are updated, then a reboot will be performed. This is recommended in most cases.
 .IP
 .SH AUTHOR
 Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.

man/striker-update-cluster.8

@@ -45,7 +45,15 @@ If the kernel is updated on a remote system, the system will normally be reboote
 .TP
 \fB\-\-reboot\-self\fR
 .TP
-By default, if the local system needs to be updated, a message is printed but the local system is NOT rebooted. This switch will instead cause this host to reboot at the end of the cluster update.
+By default, if the kenrel on the local system is updated, and so a reboot is needed, a message is printed but the local system is NOT rebooted. This switch will instead cause this host to reboot at the end of the cluster update.
+.TP
+See \fB\-\-reboot\fR for rebooting if anything is updated.
+.TP
+\fB\-\-reboot\fR
+.TP
+Normally, the system will only reboot if the kernel is updated. If this is used, and if any packages are updated, then a reboot will be performed. This is recommended in most cases. 
+.TP
+Must be used with \fB\-\-reboot\-self\fR to reboot the local system. Otherwise, it is passed along to target machines via their anvil-update-system calls.
 .IP
 .SH AUTHOR
 Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.

share/words.xml

@@ -2916,6 +2916,10 @@ Proceed? [y/N]</key>
 		<key name="message_0317">The kernel was updated, so a reboot is required. Rebooting now.</key>
 		<key name="message_0318">Registering a job to reboot this host.</key>
 		<key name="message_0319">Preparing to update the entire Anvil! cluster.</key>
+		<key name="message_0320">It looks like there's a kmod-drbd package conflict. Removing old packages and trying to reinstall.</key>
+		<key name="message_0321">Removing the old drbd-kmod RPMs now.</key>
+		<key name="message_0322">Installing the latest DRBD kmod RPM now.</key>
+		<key name="message_0323">Retrying the OS update now.</key>
 
 		<!-- Translate names (protocols, etc) -->
 		<key name="name_0001">Normal Password</key>	<!-- none in mail-server -->

tools/anvil-update-system

@@ -40,7 +40,8 @@ my $anvil = Anvil::Tools->new();
 # Read switches (target ([user@]host[:port]) and the file with the target's password.
 $anvil->Get->switches({list => [
 	"clear-cache",
-	"no-reboot"], man => $THIS_FILE});
+	"no-reboot",
+	"reboot"], man => $THIS_FILE});
 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 
@@ -86,7 +87,7 @@ update_progress($anvil, 2, "message_0033");
 $anvil->System->maintenance_mode({set => 1});
 
 # Run the update
-run_os_update($anvil);
+run_os_update($anvil, 1, 3);
 
 # We're done updating
 my $reboot_needed = $anvil->System->reboot_needed({debug => 2});
@@ -168,6 +169,7 @@ sub update_progress
 	{
 		$anvil->Job->update_progress({
 			debug    => 3, 
+			'print'  => 1,
 			progress => $progress, 
 			message  => $message, 
 			job_uuid => $anvil->data->{jobs}{job_uuid},
@@ -180,7 +182,11 @@ sub update_progress
 # This updates the OS.
 sub run_os_update
 {
-	my ($anvil) = @_;
+	my ($anvil, $try, $progress) = @_;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+		try      => $try,
+		progress => $progress, 
+	}});
 	
 	# This needs to be set to avoid warnings when called without a job-uuid.
 	$anvil->data->{sys}{last_update} = 0;
@@ -191,14 +197,14 @@ sub run_os_update
 	if ($host_type eq "node")
 	{
 		# Call anvil-safe-stop
-		update_progress($anvil, 3, "message_0314");
+		update_progress($anvil, $progress++, "message_0314");
 		
 		my $problem = $anvil->Cluster->parse_cib({debug => 3});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
 		if (not $problem)
 		{
 			# Call anvil-safe-stop
-			update_progress($anvil, 4, "message_0315");
+			update_progress($anvil, $progress++, "message_0315");
 			
 			my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'};
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
@@ -251,11 +257,11 @@ WHERE
 	}
 	
 	# NOTE: We run this directly to better monitor progress and update the progress.
+	my $package_changes   = 0;
 	my $transaction_shown = 0;
 	my $success           = 0;
 	my $to_update         = 0;
 	my $percent_step      = 0;
-	my $progress          = 6;
 	my $counted_lines     = 0;
 	my $next_step         = 0;
 	my $verifying         = 0;
@@ -308,11 +314,13 @@ WHERE
 		
 		if ($line =~ / (\d+) Packages$/i)
 		{
-			my $counted_lines =  $1;
+			my $counted_lines   =  $1;
-			   $to_update     += $counted_lines;
+			   $package_changes =  $counted_lines;
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+			   $to_update       += $counted_lines;
-				counted_lines => $counted_lines,
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-				to_update     => $to_update, 
+				package_changes => $package_changes, 
+				counted_lines   => $counted_lines,
+				to_update       => $to_update, 
 			}});
 		}
 		
@@ -384,6 +392,76 @@ WHERE
 	}
 	close $file_handle;
 	
+	# If this is the first try and it failed, see if it's a DRBD issue.
+	if ((not $success) && ($try == 1))
+	{
+		# Is this the DRBD kmod issue?
+		my $remove_drbd_kmod = 0;
+		foreach my $line (split/\n/, $output)
+		{
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
+			if ($line =~ /kmod-drbd/)
+			{
+				# Looks like it.
+				$remove_drbd_kmod = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { remove_drbd_kmod => $remove_drbd_kmod }});
+				last;
+			}
+		}
+		
+		# Clear the old kmod and try the update again.
+		if ($remove_drbd_kmod)
+		{
+			update_progress($anvil, $progress++, "message_0320");
+			
+			my $versions_to_remove = "";
+			my $shell_call         = $anvil->data->{path}{exe}{dnf}." list installed";
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+			
+			my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				output      => $output, 
+				return_code => $return_code, 
+			}});
+			foreach my $line (split/\n/, $output)
+			{
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
+				if ($line =~ /(kmod-drbd-\d+.*?)\s/)
+				{
+					$versions_to_remove .= $1." ";
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { versions_to_remove	 => $versions_to_remove }});
+				}
+			}
+			
+			# Now remove those packages.
+			update_progress($anvil, $progress++, "message_0321");
+			$shell_call = $anvil->data->{path}{exe}{dnf}." -y remove ".$versions_to_remove;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+			
+			($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				output      => $output, 
+				return_code => $return_code, 
+			}});
+			
+			# Now install the new packages.
+			update_progress($anvil, $progress++, "message_0322");
+			$shell_call = $anvil->data->{path}{exe}{dnf}." -y install kmod-drbd";
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+			
+			($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				output      => $output, 
+				return_code => $return_code, 
+			}});
+			
+			# Now try again.
+			update_progress($anvil, $progress++, "message_0323");
+			run_os_update($anvil, 2, $progress);
+			return(0);
+		}
+	}
+	
 	# Reload daemons to pick up any changed systemctl daemons.
 	my ($systemctl_output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{systemctl}." daemon-reload", source => $THIS_FILE, line => __LINE__});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { systemctl_output => $systemctl_output, return_code => $return_code }});
@@ -419,6 +497,15 @@ WHERE
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot_needed => $reboot_needed }});
 	}
 	
+	# If we installed and packages, and '--reboot' was given, reboot anyway.
+	if (($package_changes) && ($anvil->data->{switches}{reboot}))
+	{
+		# Reboot needed
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0687", variables => { reason => "#!string!log_0690!#" }});
+		my $reboot_needed = $anvil->System->reboot_needed({set => 1});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot_needed => $reboot_needed }});
+	}
+	
 	# Did it work?
 	if (not $success)
 	{
@@ -426,6 +513,7 @@ WHERE
 		update_progress($anvil, 0, "message_0036");
 		
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0035", variables => { output => $output } });
+		sleep 5;
 		$anvil->nice_exit({exit_code => 3});
 	}
 	

tools/striker-update-cluster

@@ -44,6 +44,7 @@ $anvil->Get->switches({list => [
 	"clear-cache", 
 	"force", 
 	"no-reboot", 
+	"reboot", 
 	"reboot-self", 
 	"y", 
 	"yes"], man => $THIS_FILE});
@@ -407,6 +408,10 @@ sub update_nodes
 			{
 				$shell_call .= " --clear-cache";
 			}
+			if ($anvil->data->{switches}{reboot})
+			{
+				$shell_call .= " --reboot";
+			}
 			$shell_call .= $anvil->Log->switches();
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 			$job_uuid = "";
@@ -757,36 +762,16 @@ sub update_strikers_and_dr
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot_time => $reboot_time }});
 			
 			print "- Beginning OS update of: [".$short_host_name."]\n";
-			my $rebooted    = 0;
+			my $rebooted = 0;
-			my $output      = "";
+			if (($anvil->data->{switches}{'clear-cache'}) && ($host_uuid eq $anvil->Get->host_uuid))
-			my $error       = "";
-			my $return_code = "";
-			if ($anvil->data->{switches}{'clear-cache'})
 			{
-				my $shell_call  = $anvil->data->{path}{exe}{dnf}." clean all";
+				my $shell_call = $anvil->data->{path}{exe}{dnf}." clean all";
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
-				if ($host_uuid eq $anvil->Get->host_uuid)
+				my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
-				{
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-					($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
+					output      => $output, 
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					return_code => $return_code, 
-						output      => $output, 
+				}});
-						return_code => $return_code, 
-					}});
-				}
-				else
-				{
-					($output, $error, $return_code) = $anvil->Remote->call({
-						timeout    => 0,
-						no_cache   => 1,
-						shell_call => $shell_call, 
-						target     => $anvil->data->{peer}{$short_host_name}{access}{ip},
-					});
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-						output      => $output, 
-						error       => $error,
-						return_code => $return_code,
-					}});
-				}
 				print "- Cache cleared.\n";
 			}
 			print "- Calling update now.\n";
@@ -794,7 +779,7 @@ sub update_strikers_and_dr
 			print "-       watch the progress via the system logs. You can also check wiht 'ps aux | grep dnf'.\n";
 			if ($host_uuid eq $anvil->Get->host_uuid)
 			{
-				my $shell_call  = $anvil->data->{path}{exe}{dnf}." -y update";
+				my $shell_call = $anvil->data->{path}{exe}{dnf}." -y update";
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 				my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
@@ -810,6 +795,31 @@ sub update_strikers_and_dr
 					print "==============\n";
 				}
 				
+				# Loop through the output.
+				my $package_changes = 0;
+				foreach my $line (split/\n/, $output)
+				{
+					$line = $anvil->Words->clean_spaces({string => $line});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
+					
+					if ($line =~ / (\d+) Packages$/i)
+					{
+						$package_changes += $1;
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { package_changes => $package_changes }});
+					}
+				}
+				
+				# Did the user want to reboot on any update?
+				if (($package_changes) && ($anvil->data->{switches}{reboot}) && ($anvil->data->{switches}{'reboot-self'}))
+				{
+					# Reboot needed
+					print "- Updated: [".$package_changes."] packages, and '--reboot --reboot-self' used, reboot needed!\n";
+					$anvil->data->{sys}{reboot_needed} = 1;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+						"sys::reboot_needed" => $anvil->data->{sys}{reboot_needed},
+					}});
+				}
+				
 				# Get the newest installed kernel
 				$shell_call = $anvil->data->{path}{exe}{rpm}." -q kernel | ".$anvil->data->{path}{exe}{'sort'}." | ".$anvil->data->{path}{exe}{tail}." -n 1";
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
@@ -858,6 +868,10 @@ sub update_strikers_and_dr
 				{
 					$shell_call .= " --clear-cache";
 				}
+				if ($anvil->data->{switches}{reboot})
+				{
+					$shell_call .= " --reboot";
+				}
 				$shell_call .= $anvil->Log->switches();
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 				my $job_uuid = $anvil->Database->insert_or_update_jobs({
@@ -961,9 +975,9 @@ sub update_strikers_and_dr
 			
 			# Run anvil-version-change
 			print "- Running 'anvil-version-changes' now.\n";
-			   $output      = "";
+			my $output      = "";
-			   $error       = "";
+			my $error       = "";
-			   $return_code = "";
+			my $return_code = "";
 			my $shell_call  = $anvil->data->{path}{exe}{'anvil-version-changes'};
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 			if ($host_uuid eq $anvil->Get->host_uuid)