* Finished the initial version of tools/anvil-update-system.

* Updated the RPM to .13 to disable postun's disabling of postgres, which breaks Anvil! software using the database during RPM updates.
* Fixed a logging bug where the number of DB connections was not inserting the number properly.
* Fixed exits in tools/anvil-prep-database.

Signed-off-by: Digimer <digimer@alteeve.ca>

Anvil/Tools.pm

@@ -890,6 +890,7 @@ sub _set_paths
 				createdb			=>	"/usr/bin/createdb",
 				createuser			=>	"/usr/bin/createuser",
 				dmidecode			=>	"/usr/sbin/dmidecode",
+				dnf				=>	"/usr/bin/dnf",
 				echo				=>	"/usr/bin/echo",
 				ethtool				=>	"/usr/sbin/ethtool",
 				expect				=>	"/usr/bin/expect", 

cgi-bin/striker

@@ -52,7 +52,7 @@ if (not -e $anvil->data->{path}{data}{host_uuid})
 }
 
 $anvil->Database->connect();
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
 if (not $anvil->data->{sys}{db_connections})
 {
 	# No databases, exit.

rpm/SPECS/anvil.spec

@@ -3,7 +3,7 @@
 %define anvilgroup    admin
 Name:           anvil
 Version:        3.0
-Release:        12%{?dist}
+Release:        13%{?dist}
 Summary:        Alteeve Anvil! complete package.
 
 License:        GPLv2+
@@ -343,18 +343,20 @@ sed -i.anvil 's/SELINUX=permissive/SELINUX=enforcing/' /etc/selinux/config
 setenforce 1
 
 %postun striker
+### TODO: Stopping postgres breaks the Anvil! during OS updates. Need to find a
+###       way to run this only during uninstalls, and not during updates.
 ### TODO: This breaks the repos
-rm -rf /usr/share/anvil
-echo "Closing the postgresql ports."
+# rm -rf /usr/share/anvil
+# echo "Closing the postgresql ports."
 #firewall-cmd --zone=public --remove-service=http
 #firewall-cmd --zone=public --remove-service=http --permanent
-firewall-cmd --zone=public --remove-service=postgresql
-firewall-cmd --zone=public --remove-service=postgresql --permanent
-echo "Disabling and stopping postgresql-9.6."
+# firewall-cmd --zone=public --remove-service=postgresql
+# firewall-cmd --zone=public --remove-service=postgresql --permanent
+# echo "Disabling and stopping postgresql-9.6."
 # systemctl disable httpd.service
 # systemctl stop httpd.service
-systemctl disable postgresql.service
-systemctl stop postgresql.service
+# systemctl disable postgresql.service
+# systemctl stop postgresql.service
 
 
 %files core
@@ -379,6 +381,9 @@ systemctl stop postgresql.service
 
 
 %changelog
+* Tue Aug 14 2018 Madison Kelly <mkelly@alteeve.ca> 3.0-13
+- Disabled the postun as it breaks connections to the DB during updates.
+
 * Thu Aug 02 2018 Madison Kelly <mkelly@alteeve.ca> 3.0-12
 - Added perl-Proc-Simple to core dependencies.
 

share/words.xml

@@ -230,7 +230,7 @@ The database connection error was:
 		<key name="log_0129">all</key>	<!-- Used when logging DB writes to all DBs -->
 		<key name="log_0130"><![CDATA[[ Error ] - The method: [#!variable!method!#] was called and the parameter: [#!variable!parameter!#] was passed an invalid UUID: [#!variable!uuid!#].]]></key>
 		<key name="log_0131">Entering function: [#!variable!function!#]</key>
-		<key name="log_0132">Connected to: [#!variable!connections!#] database(s).</key>
+		<key name="log_0132">Connected to: [#!data!sys::db_connections!#] database(s).</key>
 		<key name="log_0133">Failed to read the system UUID. Received a non-UUID string: [#!variable!uuid!#]. Is the user: [#!variable!user!#] in the 'kmem' group?</key>
 		<key name="log_0134">The read host UUID: [#!variable!uuid!#] does not appear to be a valid UUID.</key>
 		<key name="log_0135">Runtime was approximately: [#!variable!runtime!#] seconds.</key>
@@ -460,6 +460,14 @@ Here we will inject 't_0006', which injects 't_0001' which has a variable: [#!st
 		<key name="error_0030"><![CDATA[Failed to process the template: [<span class="code">#!variable!template!#</span>] in the template file: [<span class="code">#!variable!file!#</span>]. Details of the problem should be in: [<span class="code">#!data!sys::log_file!#</span>].]]></key>
 		<key name="error_0031">The 'host-uuid': [#!variable!host_uuid!#] is not valid.</key>
 		<key name="error_0032">The '#!variable!switch!#' switch is missing.</key>
+		<key name="error_0033">The job UUID was passed via '--job-uuid' but the passed in value: [#!variable!uuid!#] is not a valid UUID.</key>
+		<key name="error_0034">The job UUID was passed via '--job-uuid': [#!variable!uuid!#] doesn't match a job in the database.</key>
+		<key name="error_0035">
+The update appears to have not completed successfully. The output was:
+====
+#!variable!output!#
+====
+		</key>
 		
 		<!-- These are units, words and so on used when displaying information. -->
 		<key name="unit_0001">Yes</key>

tools/anvil-change-password

@@ -48,7 +48,7 @@ if (($< != 0) && ($> != 0))
 
 # Connect
 $anvil->Database->connect();
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
 if (not $anvil->data->{sys}{db_connections})
 {
 	# No databases, exit.

tools/anvil-configure-striker

@@ -47,7 +47,7 @@ if (($< != 0) && ($> != 0))
 # Connect
 $anvil->Database->connect;
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0031"});
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
 if (not $anvil->data->{sys}{db_connections})
 {
 	# No databases, exit.

tools/anvil-daemon

@@ -30,7 +30,7 @@ my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
 # Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks
 # is to setup the database server.
 $anvil->Database->connect();
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
 
 # There are some things we only want to run on (re)start and don't need to always run.
 run_once($anvil);
@@ -40,7 +40,7 @@ $anvil->Storage->record_md5sums;
 
 # Disconnect. We'll reconnect inside the loop
 $anvil->Database->disconnect;
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
 
 # These are the things we always want running.
 while(1)

tools/anvil-prep-database

@@ -56,7 +56,7 @@ if ($local_uuid)
 	{
 		# Not installed.
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0144"});
-		exit(5);
+		$anvil->nice_exit({code => 5});
 	}
 	elsif (not $running)
 	{
@@ -91,7 +91,7 @@ if ($local_uuid)
 				{
 					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0050"});
 				}
-				exit(1);
+				$anvil->nice_exit({code => 1});
 			}
 			else
 			{
@@ -203,7 +203,7 @@ if ($local_uuid)
 		{
 			# Failed to start
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0094"});
-			exit(2);
+			$anvil->nice_exit({code => 2});
 		}
 	}
 	
@@ -241,7 +241,7 @@ if ($local_uuid)
 	{
 		# No database user defined
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0099", variables => { uuid => $local_uuid }});
-		exit(3);
+		$anvil->nice_exit({code => 3});
 	}
 	my $user_list = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT usename, usesysid FROM pg_catalog.pg_user;'\"", source => $THIS_FILE, line => __LINE__});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { user_list => $user_list }});
@@ -278,7 +278,7 @@ if ($local_uuid)
 		if (not $user_exists)
 		{
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0096", variables => { user => $database_user }});
-			exit(4);
+			$anvil->nice_exit({code => 4});
 		}
 		
 		# Update/set the passwords.
@@ -340,7 +340,7 @@ if ($local_uuid)
 		if (not $database_exists)
 		{
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0109", variables => { database => $database_name }});
-			exit(5);
+			$anvil->nice_exit({code => 5});
 		}
 	}
 	

tools/anvil-update-states

@@ -23,7 +23,7 @@ my $anvil = Anvil::Tools->new({log_level => 1, log_secure => 1});
 $anvil->Storage->read_config({file => "/etc/anvil/anvil.conf"});
 $anvil->Database->connect;
 
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
 if (not $anvil->data->{sys}{db_connections})
 {
 	# No databases, exit.

tools/anvil-update-system

@@ -8,8 +8,14 @@
 #   updated and then rejoin the cluster.
 # - On DR; This will do nothing until no servers are running, then it will update the system.
 #  
-# In all cases, the system will be rebooted  
+# In all cases, the system will be rebooted if the kernel is updated.
 # 
+# Exit codes;
+# 0 = Normal exit.
+# 1 = No database connections available.
+# 2 = The job UUID was passed, but it wasn't valid.
+# 3 = It looks like the update failed, reset progress to '0'.
+
 use strict;
 use warnings;
 use Anvil::Tools;
@@ -24,24 +30,98 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 	$running_directory =~ s/^\./$ENV{PWD}/;
 }
 
-my $anvil = Anvil::Tools->new({log_level => 1, log_secure => 1});
+my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
 
 $anvil->Storage->read_config({file => "/etc/anvil/anvil.conf"});
-$anvil->Database->connect;
 
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
+# Read switches
+$anvil->data->{switches}{'no-reboot'} = 0;
+$anvil->data->{switches}{'job-uuid'}  = "";
+$anvil->Get->switches;
+
+# Connect to DBs.
+$anvil->Database->connect;
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
 if (not $anvil->data->{sys}{db_connections})
 {
 	# No databases, exit.
 	print $anvil->Words->string({key => "error_0003"})."\n";
-	$anvil->nice_exit({exit_code => 2});
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0003"});
+	$anvil->nice_exit({exit_code => 1});
 }
 
 # Did we get called with a job UUID?
-$anvil->data->{jobs}{job_uuid} = defined $anvil->data->{switches}{'job-uuid'} ? $anvil->data->{switches}{'job-uuid'} : "";
+if ($anvil->data->{switches}{'job-uuid'})
+{
+	# Is it set and valid?
+	if (not $anvil->Validate->is_uuid({uuid => $anvil->data->{switches}{'job-uuid'}}))
+	{
+		# It's not a UUID.
+		print $anvil->Words->string({key => "error_0033", variables => { uuid => $anvil->data->{switches}{'job-uuid'} }})."\n";
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0033", variables => { uuid => $anvil->data->{switches}{'job-uuid'} } });
+		$anvil->nice_exit({code => 2});
+	}
+	
+	# If I'm here, see if we can read the job details.
+	my $query = "
+SELECT 
+    job_host_uuid, 
+    job_data, 
+    job_updated, 
+    job_name,
+    job_status 
+FROM 
+    jobs 
+WHERE 
+    job_uuid = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{switches}{'job-uuid'})." 
+;";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+	
+	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
+	my $count   = @{$results};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		results => $results, 
+		count   => $count, 
+	}});
+	if ($count < 1)
+	{
+		print $anvil->Words->string({key => "error_0034", variables => { uuid => $anvil->data->{switches}{'job-uuid'} }})."\n";
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0034", variables => { uuid => $anvil->data->{switches}{'job-uuid'} } });
+		$anvil->nice_exit({code => 2});
+	}
+	
+	# If we're here, we're good. Load the details
+	$anvil->data->{jobs}{job_uuid}      = $anvil->data->{switches}{'job-uuid'};
+	$anvil->data->{jobs}{job_host_uuid} = $results->[0]->[0];
+	$anvil->data->{jobs}{job_data}      = $results->[0]->[1];
+	$anvil->data->{jobs}{job_updated}   = $results->[0]->[2];
+	$anvil->data->{jobs}{job_name}      = $results->[0]->[3];
+	$anvil->data->{jobs}{job_status}    = $results->[0]->[4];
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		"jobs::job_uuid"      => $anvil->data->{jobs}{job_uuid}, 
+		"jobs::job_host_uuid" => $anvil->data->{jobs}{job_host_uuid},
+		"jobs::job_data"      => $anvil->data->{jobs}{job_data}, 
+		"jobs::job_updated"   => $anvil->data->{jobs}{job_updated}, 
+		"jobs::job_name"      => $anvil->data->{jobs}{job_name}, 
+		"jobs::job_status"    => $anvil->data->{jobs}{job_status}, 
+	}});
+}
+
+# Mark that we're starting
 update_progress($anvil, 1);
 
-run_os_update($anvil);
+my ($reboot) = run_os_update($anvil);
+$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }});
+
+# We're done updating
+update_progress($anvil, 100);
+
+# Reboot if needed.
+if (($reboot) && (not $anvil->data->{switches}{'no-reboot'}))
+{
+	# Reboot.
+	$anvil->System->call({shell_call => $anvil->data->{path}{exe}{'shutdown'}." --reboot now"});
+}
 
 $anvil->nice_exit({exit_code => 0});
 
@@ -54,6 +134,9 @@ sub update_progress
 {
 	my ($anvil, $progress) = @_;
 	
+	# Log the progress percentage.
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { progress => $progress }});
+	
 	if ($anvil->data->{jobs}{job_uuid})
 	{
 		my $query = "
@@ -61,14 +144,14 @@ UPDATE
     jobs 
 SET 
     job_picked_up_by = ".$anvil->data->{sys}{use_db_fh}->quote($$).", 
+    job_updated      = now(),
     job_progress     = ".$progress.", 
     modified_date    = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{sys}{db_timestamp})." 
 WHERE 
-    job_uuid         = ".$anvil->data->{sys}{use_db_fh}->quote($job_uuid)." 
+    job_uuid         = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{jobs}{job_uuid})." 
 ";
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
-	$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
-		
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+		$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
 	}
 	
 	return(0);
@@ -79,36 +162,115 @@ sub run_os_update
 {
 	my ($anvil) = @_;
 	
-	my $stdout_file = "/tmp/os_update.stdout";
-	my $stderr_file = "/tmp/os_update.stderr";
-	my $shell_call  = $anvil->data->{path}{exe}{dnf}." clean expire-cache && ".$anvil->data->{path}{exe}{dnf}." -y update";
-	
-	my $process = $anvil->System->call({
-		debug       => 2,
-		background  => 1, 
-		stdout_file => $stdout_file,
-		stderr_file => $stderr_file, 
-		shell_call  => $job_command." --job-uuid ".$job_uuid, 
-		source      => $THIS_FILE, 
-		line        => __LINE__,
-	});
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { process => $process }});
+	# NOTE: We run this directly to better monitor progress and update the progress.
+	my $success       = 0;
+	my $to_update     = 0;
+	my $percent_step  = 0;
+	my $progress      = 5;
+	my $counted_lines = 0;
+	my $next_step     = 0;
+	my $reboot        = 0;
+	my $output        = "";
+	my $shell_call = $anvil->data->{path}{exe}{dnf}." clean expire-cache && ".$anvil->data->{path}{exe}{dnf}." -y update; ".$anvil->data->{path}{exe}{echo}." return_code:\$?";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+	open (my $file_handle, $shell_call." 2>&1 |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }});
+	while(<$file_handle>)
+	{
+		chomp;
+		my $line   =  $_;
+		   $output .= $line."\n";
+		   $line   =  $anvil->Words->clean_spaces({string => $line});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
 		
-	# Record our PID
-	my $pid = $anvil->data->{jobs}{handles}{$job_uuid}->pid();
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { pid => $pid }});
-	my $query = "
-UPDATE 
-    jobs 
-SET 
-    job_picked_up_by = ".$anvil->data->{sys}{use_db_fh}->quote($pid).", 
-    modified_date    = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{sys}{db_timestamp})." 
-WHERE 
-    job_uuid         = ".$anvil->data->{sys}{use_db_fh}->quote($job_uuid)." 
-";
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
-	$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
+		if ($line =~ /^kernel /)
+		{
+			# Reboot will be needed.
+			$reboot = 1;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }});
+		}
+		
+		if ($line =~ /return_code:(\d+)$/)
+		{
+			my $return_code = $1;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
+			if ($return_code == 0)
+			{
+				$success = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { success => $success }});
+			}
+		}
+		
+		if ($line =~ / (\d+) Packages$/i)
+		{
+			my $counted_lines =  $1;
+			   $to_update     += $counted_lines;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				counted_lines => $counted_lines,
+				to_update     => $to_update, 
+			}});
+		}
+		
+		if ($line =~ /Total download size/i)
+		{
+			# Ready to install, update to 5%. The next step will count up to 95%.
+			update_progress($anvil, $progress);
+			
+			# The total (reliable) count of events is (to_update * 3), counting '(x/y): ' 
+			# (download), 'Upgrading ', 'Installing ' and 'Verifying '. We ignore the scriplet
+			# and other lines as it's hard to predict how many there will be, and they pass fast
+			# enough to not really matter for a progress bar.
+			$to_update    *= 3;
+			$percent_step =  $to_update / 90;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				to_update    => $to_update, 
+				percent_step => $percent_step, 
+			}});
+			
+			if ($percent_step =~ /\d+\.\d+/)
+			{
+				# Round up.
+				$percent_step += 1;
+				$percent_step =  int($percent_step);
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { percent_step => $percent_step }});
+			}
+			
+			$next_step = $percent_step;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { next_step => $next_step }});
+		}
+		
+		# If 'percent_step' is set, we're ready to start counting lines.
+		if (($percent_step) && (($line =~ /\(\d+\/\d+\): /) or ($line =~ /Upgrading /i) or ($line =~ /Installing /) or ($line =~ /Cleanup /)))
+		{
+			$counted_lines++;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { counted_lines => $counted_lines }});
+			
+			if ($counted_lines > $next_step)
+			{
+				# Step up the progress.
+				$next_step += $percent_step;
+				$progress++;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					next_step => $next_step,
+					progress  => $progress, 
+				}});
+				
+				update_progress($anvil, $progress);
+			}
+		}
+	}
+	close $file_handle;
 	
+	# Did it work?
+	if (not $success)
+	{
+		# Nope.
+		update_progress($anvil, 0);
+		
+		print $anvil->Words->string({key => "error_0035", variables => { output => $output }})."\n";
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0035", variables => { output => $output } });
+		$anvil->nice_exit({code => 3});
+	}
 	
-	return(0);
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }});
+	return($reboot);
 };

tools/anvil.sql

@@ -357,7 +357,7 @@ CREATE TABLE jobs (
     job_uuid           uuid                        not null    primary key,    -- 
     job_host_uuid      uuid                        not null,                   -- This is the host that requested the job
     job_command        text                        not null,                   -- This is the command to run (usually a shell call).
-    job_data           text                        not null,                   -- This is optional data to be used by anvil-data 
+    job_data           text                        not null,                   -- 
     job_picked_up_by   numeric                     not null    default 0,      -- This is the PID of the 'anvil-jobs' script that picked up the job.
     job_picked_up_at   numeric                     not null    default 0,      -- This is unix timestamp of when the job was picked up.
     job_updated        numeric                     not null    default 0,      -- This is unix timestamp that is perdiodically updated for jobs that take a long time. It is used to help determine when a job is hung.