* Changed the call to reboot after OS update to instead set a reboot required flag.

* Created tools/anvil-clear-reboot to clear the "reboot needed" flag. Also created, but not yet using (and may not use) units/anvil-boot-time.service.
* Started work on having jobs show their data via JSON / jquery.
* Updated anvil-update-system to record messages indicating the progress so far.

Signed-off-by: Digimer <digimer@alteeve.ca>

Anvil/Tools.pm

@@ -788,6 +788,29 @@ sub _set_defaults
 		},
 	};
 	$anvil->data->{defaults} = {
+		database	=>	{
+			locking		=>	{
+				reap_age	=>	300,
+			}
+		},
+		language	=>	{
+			# Default language for all output shown to a user.
+			output		=>	'en_CA',
+		},
+		limits		=>	{
+			# This is the maximum number of times we're allow to loop when injecting variables 
+			# into a string being processed in Anvil::Tools::Words->string();
+			string_loops	=>	1000,
+		},
+		'log'		=>	{
+			db_transactions	=>	0,
+			facility	=>	"local0",
+			language	=>	"en_CA",
+			level		=>	1,
+			secure		=>	0,
+			server		=>	"",
+			tag		=>	"anvil",
+		},
 		## Network stuff... The second octet auto-increments to handle N-number of netowrks. As such,
 		##                  we need to use a wider spread between the BCNs, SNs and IFNs than we had
 		##                  in v2.
@@ -814,29 +837,6 @@ sub _set_defaults
 				netmask		=>	"255.255.0.0",
 			},
 		},
-		database	=>	{
-			locking		=>	{
-				reap_age	=>	300,
-			}
-		},
-		language	=>	{
-			# Default language for all output shown to a user.
-			output		=>	'en_CA',
-		},
-		limits		=>	{
-			# This is the maximum number of times we're allow to loop when injecting variables 
-			# into a string being processed in Anvil::Tools::Words->string();
-			string_loops	=>	1000,
-		},
-		'log'		=>	{
-			db_transactions	=>	0,
-			facility	=>	"local0",
-			language	=>	"en_CA",
-			level		=>	1,
-			secure		=>	0,
-			server		=>	"",
-			tag		=>	"anvil",
-		},
 		template	=>	{
 			html		=>	"alteeve",
 		},
@@ -942,6 +942,9 @@ sub _set_paths
 			'lock'			=>	{
 				database			=>	"/tmp/anvil-tools.database.lock",
 			},
+			proc			=>	{
+				uptime				=>	"/proc/uptime",
+			},
 			secure			=>	{
 				postgres_pgpass			=>	"/var/lib/pgsql/.pgpass",
 			},

Anvil/Tools/System.pm

@@ -901,6 +901,7 @@ sub maintenance_mode
 	
 	if ($set)
 	{
+		### TODO: stop other systems from using this database.
 		# Am I enabling or disabling?
 		if ($set eq "1")
 		{

cgi-bin/striker

@@ -278,7 +278,7 @@ sub process_update
 			debug           => 2,
 			file            => $THIS_FILE, 
 			line            => __LINE__, 
-			job_command     => "anvil-update-system --uuid ".$anvil->Get->host_uuid, 
+			job_command     => "anvil-update-system --host-uuid ".$anvil->Get->host_uuid, 
 			job_data        => "", 
 			job_name        => "update::system", 
 			job_title       => "job_0003", 
@@ -772,10 +772,10 @@ sub check_availability
 {
 	my ($anvil) = @_;
 	
-	my $debug     = 3;
+	my $debug     = 2;
 	my $available = 1;
 	
-	# Set maintenance mode.
+	# Check maintenance mode.
 	my $maintenance_mode = $anvil->System->maintenance_mode({debug => $debug});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { maintenance_mode => $maintenance_mode }});
 	
@@ -783,6 +783,14 @@ sub check_availability
 	{
 		$available = 0;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { available => $available }});
+		
+		$anvil->data->{say}{maintenance} = $anvil->Template->get({file => "striker.html", name => "striker-offline", variables => {
+			title_id    => "",
+			message_id  => "",
+			title       => "#!string!striker_0046!#",
+			description => $anvil->Words->string({key => "striker_0090", variables => {  }}),
+		}});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'say::maintenance' => $anvil->data->{say}{maintenance} }});
 	}
 	
 	# TODO: Phase this out

html/skins/alteeve/config.js

@@ -133,4 +133,11 @@ $( window ).on( "load", function()
 			}
 		});
 	});
+	
+	$.getJSON('/status/jobs.json', { get_param: 'value' }, function(data) {
+		$.each(data.ips, function(index, element) {
+			//console.log('- entry: ['+index+'], on: ['+element.on+'], address: ['+element.address+'], subnet: ['+element.subnet+'].');
+			//console.log('- gateway: ['+element.gateway+'], dns: ['+element.dns+'], default gateway: ['+element.default_gateway+'].');
+		});
+	});
 })

share/words.xml

@@ -58,12 +58,14 @@ NOTE: You must update the password of any other system using this host's
 		<key name="message_0030">Failed to write the new password to the temporary file: [#!variable!file!#]. Please check the logs for details.</key>
 		<key name="message_0031">Beginning configuration of local system.</key>
 		<key name="message_0032">Peer: [#!variable!peer!#], database: [#!variable!name!#], UUID: [#!variable!uuid!#]</key>
-		<key name="message_0033"></key>
+		<key name="message_0033">Clearing update cache and checking for available updates.</key>
 		<key name="message_0034"><![CDATA[Logged in as: [<span class="fixed_width">#!data!sys::users::user_name!#</span>]]]></key>
-		<key name="message_0035"></key>
+		<key name="message_0035">Downloading approximately: [#!variable!size!#] worth of updates.</key>
-		<key name="message_0036"></key>
+		<key name="message_0036">ERROR: There was a problem with the OS update process. Please check the system logs for more details.</key>
-		<key name="message_0037"></key>
+		<key name="message_0037">Downloading complete. Installation of updates now underway.</key>
-		<key name="message_0038"></key>
+		<key name="message_0038">Updates finished. Verifying now.</key>
+		<key name="message_0039">System update complete! The kernel was updated, so a reboot is required.</key>
+		<key name="message_0040">System update complete! A reboot is not required.</key>
 		
 		<!-- Log entries -->
 		<key name="log_0001">Starting: [#!variable!program!#].</key>
@@ -297,6 +299,8 @@ The database connection error was:
 		<key name="log_0196">Failed to reconnect to the database, and now no connections remail. Exiting.</key>
 		<key name="log_0197"><![CDATA[System->maintenance_mode() was passed an invalid 'set' value: [#!variable!set!#]. No action taken.]]></key>
 		<key name="log_0198">The user: [#!variable!user!#] logged out successfully.</key>
+		<key name="log_0199">A system reboot is required, setting the database flag.</key>
+		<key name="log_0200">A system reboot is required, setting the database flag.</key>
 		
 		<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
 		<key name="t_0000">Test</key>
@@ -424,6 +428,8 @@ Here we will inject 't_0006', which injects 't_0001' which has a variable: [#!st
 		<key name="striker_0086">Would you like to update the operating system on this machine? This Striker will be placed into maintenance mode until the update completes.</key>
 		<key name="striker_0087">When enabled on a Striker dashboard, the web interface will be disabled and ScanCore will not record to the local database. When enabled on a node, no servers will be allowed to run on it, and any already running on it will be migrated. When run on a DR node, that node will be disconnected from storage and no servers will be allowed to run on it. When disabled, all normal functions are available</key>
 		<key name="striker_0088">The system will be updated momentarily. This system will now be in maintenance mode until the update is complete.</key>
+		<key name="striker_0089">This indicates whether this system needs to be rebooted or not.</key>
+		<key name="striker_0090">This system is in maintenance mode and is not currently available.</key>
 		
 		<!-- Strings used by jobs -->
 		<key name="job_0001">Configure Network</key>

tools/anvil-clear-reboot

@@ -0,0 +1,55 @@
+#!/usr/bin/perl
+# 
+# Exit codes;
+# 0 = Normal exit.
+# 1 = No database connections available.
+
+use strict;
+use warnings;
+use Anvil::Tools;
+
+# Disable buffering
+$| = 1;
+
+my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
+my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
+if (($running_directory =~ /^\./) && ($ENV{PWD}))
+{
+	$running_directory =~ s/^\./$ENV{PWD}/;
+}
+
+my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
+
+$anvil->Storage->read_config({file => "/etc/anvil/anvil.conf"});
+
+# Read switches
+$anvil->Get->switches;
+
+# Connect to DBs.
+$anvil->Database->connect;
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
+if (not $anvil->data->{sys}{database}{connections})
+{
+	# No databases, exit.
+	print $anvil->Words->string({key => "error_0003"})."\n";
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0003"});
+	$anvil->nice_exit({exit_code => 1});
+}
+
+# If the uptime is less than ten minutes, clear the reboot flag.
+my $uptime = $anvil->Storage->read_file({file => $anvil->data->{path}{proc}{uptime}});
+
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0200"});
+$anvil->Database->insert_or_update_variables({
+	variable_name         => "reboot::needed", 
+	variable_value        => "0", 
+	variable_default      => "0", 
+	variable_description  => "striker_0089", 
+	variable_section      => "system", 
+	variable_source_uuid  => $anvil->Get->host_uuid, 
+	variable_source_table => "hosts", 
+	update_value_only     => 1, 
+});
+
+# We're done
+$anvil->nice_exit({exit_code => 0});

tools/anvil-daemon

@@ -86,6 +86,32 @@ sub run_once
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_output => $database_output }});
 	}
 	
+	# If the uptime is less than ten minutes, clear the reboot flag.
+	my $uptime = $anvil->Storage->read_file({
+		debug      => 2,
+		force_read => 1,
+		cache      => 0,
+		file       => $anvil->data->{path}{proc}{uptime},
+	});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uptime => $uptime }});
+	
+	# Clean it up. We'll have gotten two numbers, the uptime in seconds (to two decimal places) and the 
+	# total idle time. We only care about the int number.
+	$uptime =~ s/^(\d+)\..*$/$1/;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uptime => $uptime }});
+	
+	if ($uptime < 600)
+	{
+		# Clear the reboot request.
+		my $output = $anvil->System->call({
+			debug      => 2,
+			shell_call => $anvil->data->{path}{exe}{'anvil-clear-reboot'}, 
+			source     => $THIS_FILE, 
+			line       => __LINE__,
+		});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output }});
+	}
+	
 	return(0);
 }
 
@@ -156,6 +182,9 @@ sub run_jobs
 {
 	my ($anvil) = @_;
 	
+	# We'll also update the jobs.json file.
+	my $jobs_file = "{\"jobs\":[\n";
+	
 	# Get a list of pending or incomplete jobs.
 	my $query = "
 SELECT 
@@ -164,15 +193,15 @@ SELECT
     job_data, 
     job_picked_up_by, 
     job_picked_up_at, 
+    extract(epoch from job_picked_up_at), 
     job_updated, 
+    extract(epoch from modified_date) 
     job_progress 
 FROM 
     jobs 
 WHERE 
-    job_progress != 100 
-AND 
     job_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->Get->host_uuid)."
-LIMIT 1;";
+;";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
 	
 	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
@@ -188,18 +217,40 @@ LIMIT 1;";
 		my $job_data         = defined $row->[2] ? $row->[2] : "";
 		my $job_picked_up_by =         $row->[3];
 		my $job_picked_up_at =         $row->[4]; 
+		my $unix_picked_up   =         $row->[3];
 		my $job_updated      =         $row->[5];
-		my $job_progress     =         $row->[6];
+		my $unix_updated     =         $row->[6];
+		my $job_progress     =         $row->[7];
+		my $started_seconds_ago = time - $unix_picked_up;
+		my $updated_seconds_ago = time - $unix_updated;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-			job_uuid         => $job_uuid,
+			job_uuid            => $job_uuid,
-			job_command      => $job_command,
+			job_command         => $job_command,
-			job_data         => $job_data,
+			job_data            => $job_data,
-			job_picked_up_by => $job_picked_up_by,
+			job_picked_up_by    => $job_picked_up_by,
-			job_picked_up_at => $job_picked_up_at,
+			job_picked_up_at    => $job_picked_up_at,
-			job_updated      => $job_updated,
+			unix_picked_up      => $unix_picked_up, 
-			job_progress     => $job_progress,
+			job_updated         => $job_updated,
+			unix_updated        => $unix_updated, 
+			job_progress        => $job_progress,
+			started_seconds_ago => $started_seconds_ago, 
+			updated_seconds_ago => $updated_seconds_ago, 
 		}});
 		
+		# If the job is done, see if it was recently enough to record in the jobs.json file.
+		if ($job_progress eq "100") 
+		{
+			# Record in JSON if it wass last updated less than 5 minutes ago.
+			if ($updated_seconds_ago < 300)
+			{
+				$jobs_file .= "{ \"job_uuid\":\"".$job_uuid."\", \"job_command\":\"".$job_command."\", \"job_data\":\"".$job_data."\", \"job_picked_up_at\":\"".$job_picked_up_at."\", \"job_updated\":\"".$job_updated."\", \"job_progress\":\"".$job_progress."\", \"job_progress\":\"".$job_progress."\", \"started_seconds_ago\":\"".$started_seconds_ago."\", \"updated_seconds_ago\":\"".$updated_seconds_ago."\" }, \n";
+			}
+			next;
+		}
+		
+		# If we're here, the job isn't done. So first, record it.
+		$jobs_file .= "{ \"job_uuid\":\"".$job_uuid."\", \"job_command\":\"".$job_command."\", \"job_data\":\"".$job_data."\", \"job_picked_up_at\":\"".$job_picked_up_at."\", \"job_updated\":\"".$job_updated."\", \"job_progress\":\"".$job_progress."\", \"job_progress\":\"".$job_progress."\", \"started_seconds_ago\":\"".$started_seconds_ago."\", \"updated_seconds_ago\":\"".$updated_seconds_ago."\" }, \n";
+		
 		# See if the job was picked up by another running instance.
 		if ($job_picked_up_by)
 		{
@@ -227,7 +278,8 @@ LIMIT 1;";
 			stderr_file => "/tmp/anvil.job.".$job_uuid.".stderr", 
 			shell_call  => $job_command." --job-uuid ".$job_uuid, 
 			source      => $THIS_FILE, 
-			line        => __LINE__});
+			line        => __LINE__,
+		});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "jobs::handles::${job_uuid}" => $anvil->data->{jobs}{handles}{$job_uuid} }});
 		
 		# Record the PID
@@ -246,6 +298,9 @@ WHERE
 		$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
 	}
 	
+	# Close the jobs file.
+	$jobs_file .= "]}\n";
+	
 	return(0);
 }
 

tools/anvil-update-system

@@ -108,23 +108,34 @@ WHERE
 }
 
 # Mark that we're starting
-update_progress($anvil, 1);
+update_progress($anvil, 1, "message_0033");
 
 my ($reboot) = run_os_update($anvil);
 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }});
 
 # We're done updating
-update_progress($anvil, 100);
+my ($reboot_needed, $variable_uuid, $modified_date) = $anvil->Database->read_variable({
-
+	variable_name         => "reboot::needed",
-# Reboot if needed.
+	variable_source_uuid  => $anvil->Get->host_uuid, 
-if (($reboot) && (not $anvil->data->{switches}{'no-reboot'}))
+	variable_source_table => "hosts", 
+});
+$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+	reboot_needed => $reboot_needed, 
+	variable_uuid => $variable_uuid, 
+	modified_date => $modified_date, 
+}});
+if ($reboot_needed)
+{
+	update_progress($anvil, 100, "message_0039");
+}
+else
 {
-	# Reboot.
+	update_progress($anvil, 100, "message_0040");
-	$anvil->System->call({shell_call => $anvil->data->{path}{exe}{'shutdown'}." --reboot now"});
 }
 
 $anvil->nice_exit({exit_code => 0});
 
+
 #############################################################################################################
 # Functions                                                                                                 #
 #############################################################################################################
@@ -132,13 +143,35 @@ $anvil->nice_exit({exit_code => 0});
 # This updates the progress if we were called with a job UUID.
 sub update_progress
 {
-	my ($anvil, $progress) = @_;
+	my ($anvil, $progress, $message) = @_;
 	
 	# Log the progress percentage.
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { progress => $progress }});
 	
 	if ($anvil->data->{jobs}{job_uuid})
 	{
+		# Get the current job_status and append this new one.
+		my $query = "
+SELECT 
+    job_status 
+FROM 
+    jobs 
+WHERE 
+    job_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{jobs}{job_uuid})."
+;";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
+		
+		my $job_status = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
+		   $job_status = "" if not defined $old_job_status;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { job_status => $job_status }});
+			
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { message => $message }});
+		if ($message)
+		{
+			$job_status .= $message."\n";
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { job_status => $job_status }});
+		}
+		
 		my $query = "
 UPDATE 
     jobs 
@@ -146,6 +179,7 @@ SET
     job_picked_up_by = ".$anvil->data->{sys}{database}{use_handle}->quote($$).", 
     job_updated      = now(),
     job_progress     = ".$progress.", 
+    job_status       = ".$anvil->data->{sys}{database}{use_handle}->quote($job_status).", 
     modified_date    = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})." 
 WHERE 
     job_uuid         = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{jobs}{job_uuid})." 
@@ -169,7 +203,7 @@ sub run_os_update
 	my $progress      = 5;
 	my $counted_lines = 0;
 	my $next_step     = 0;
-	my $reboot        = 0;
+	my $verifying     = 0;
 	my $output        = "";
 	my $shell_call = $anvil->data->{path}{exe}{dnf}." clean expire-cache && ".$anvil->data->{path}{exe}{dnf}." -y update; ".$anvil->data->{path}{exe}{echo}." return_code:\$?";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
@@ -185,8 +219,31 @@ sub run_os_update
 		if ($line =~ /^kernel /)
 		{
 			# Reboot will be needed.
-			$reboot = 1;
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0199"});
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }});
+			$anvil->Database->insert_or_update_variables({
+				variable_name         => "reboot::needed", 
+				variable_value        => "1", 
+				variable_default      => "0", 
+				variable_description  => "striker_0089", 
+				variable_section      => "system", 
+				variable_source_uuid  => $anvil->Get->host_uuid, 
+				variable_source_table => "hosts", 
+				update_value_only     => 1, 
+			});
+			
+		}
+		
+		if ((not $verifying) && ($line =~ /^Verifying /i))
+		{
+			# Update done, verifying now.
+			$verifying = 1;
+			update_progress($anvil, $progress, "message_0038");
+		}
+		
+		if ($line =~ /Running transaction/i)
+		{
+			# Done downloading
+			update_progress($anvil, $progress, "message_0037");
 		}
 		
 		if ($line =~ /return_code:(\d+)$/)
@@ -210,10 +267,13 @@ sub run_os_update
 			}});
 		}
 		
-		if ($line =~ /Total download size/i)
+		if ($line =~ /Total download size: (.*)$/i)
 		{
+			my $update_size = $1;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update_size => $update_size }});
+			
 			# Ready to install, update to 5%. The next step will count up to 95%.
-			update_progress($anvil, $progress);
+			update_progress($anvil, $progress, "message_0035,!!size!$update_size!!");
 			
 			# The total (reliable) count of events is (to_update * 3), counting '(x/y): ' 
 			# (download), 'Upgrading '/'Installing ' and 'Verifying '. We ignore the scriplet
@@ -246,7 +306,7 @@ sub run_os_update
 				}});
 				next if $progress > 95;
 				
-				update_progress($anvil, $progress);
+				update_progress($anvil, $progress, "");
 			}
 		}
 	}
@@ -256,13 +316,11 @@ sub run_os_update
 	if (not $success)
 	{
 		# Nope.
-		update_progress($anvil, 0);
+		update_progress($anvil, 0, "message_0036");
 		
-		print $anvil->Words->string({key => "error_0035", variables => { output => $output }})."\n";
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0035", variables => { output => $output } });
 		$anvil->nice_exit({code => 3});
 	}
 	
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }});
+	return(0);
-	return($reboot);
 };

units/anvil-boot-time.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Anvil! Intelligent Availability Platform - Boot-time tasks
+# Don't run until we can talk to our and our peer's databases.
+Wants=network.target
+Wants=postgresql.service
+
+[Service]
+Type=simple
+ExecStart=/usr/sbin/anvil-clear-reboot
+ExecStop=/bin/kill -WINCH ${MAINPID}
+
+[Install]
+WantedBy=default.target