From dd88051d9b1de09b9f001dd6b3bc83f54ce4ce1d Mon Sep 17 00:00:00 2001 From: Digimer Date: Tue, 14 Aug 2018 03:45:36 -0400 Subject: [PATCH] * Finished the initial version of tools/anvil-update-system. * Updated the RPM to .13 to disable postun's disabling of postgres, which breaks Anvil! software using the database during RPM updates. * Fixed a logging bug where the number of DB connections was not inserting the number properly. * Fixed exits in tools/anvil-prep-database. Signed-off-by: Digimer --- Anvil/Tools.pm | 1 + cgi-bin/striker | 2 +- rpm/SPECS/anvil.spec | 21 +-- share/words.xml | 10 +- tools/anvil-change-password | 2 +- tools/anvil-configure-striker | 2 +- tools/anvil-daemon | 4 +- tools/anvil-prep-database | 12 +- tools/anvil-update-states | 2 +- tools/anvil-update-system | 242 ++++++++++++++++++++++++++++------ tools/anvil.sql | 2 +- 11 files changed, 238 insertions(+), 62 deletions(-) diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index a23c1fd4..72cd3120 100755 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -890,6 +890,7 @@ sub _set_paths createdb => "/usr/bin/createdb", createuser => "/usr/bin/createuser", dmidecode => "/usr/sbin/dmidecode", + dnf => "/usr/bin/dnf", echo => "/usr/bin/echo", ethtool => "/usr/sbin/ethtool", expect => "/usr/bin/expect", diff --git a/cgi-bin/striker b/cgi-bin/striker index 59d2a6b3..5872c857 100755 --- a/cgi-bin/striker +++ b/cgi-bin/striker @@ -52,7 +52,7 @@ if (not -e $anvil->data->{path}{data}{host_uuid}) } $anvil->Database->connect(); -$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }}); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); if (not $anvil->data->{sys}{db_connections}) { # No databases, exit. diff --git a/rpm/SPECS/anvil.spec b/rpm/SPECS/anvil.spec index 0141a3f7..17b23c0f 100644 --- a/rpm/SPECS/anvil.spec +++ b/rpm/SPECS/anvil.spec @@ -3,7 +3,7 @@ %define anvilgroup admin Name: anvil Version: 3.0 -Release: 12%{?dist} +Release: 13%{?dist} Summary: Alteeve Anvil! complete package. License: GPLv2+ @@ -343,18 +343,20 @@ sed -i.anvil 's/SELINUX=permissive/SELINUX=enforcing/' /etc/selinux/config setenforce 1 %postun striker +### TODO: Stopping postgres breaks the Anvil! during OS updates. Need to find a +### way to run this only during uninstalls, and not during updates. ### TODO: This breaks the repos -rm -rf /usr/share/anvil -echo "Closing the postgresql ports." +# rm -rf /usr/share/anvil +# echo "Closing the postgresql ports." #firewall-cmd --zone=public --remove-service=http #firewall-cmd --zone=public --remove-service=http --permanent -firewall-cmd --zone=public --remove-service=postgresql -firewall-cmd --zone=public --remove-service=postgresql --permanent -echo "Disabling and stopping postgresql-9.6." +# firewall-cmd --zone=public --remove-service=postgresql +# firewall-cmd --zone=public --remove-service=postgresql --permanent +# echo "Disabling and stopping postgresql-9.6." # systemctl disable httpd.service # systemctl stop httpd.service -systemctl disable postgresql.service -systemctl stop postgresql.service +# systemctl disable postgresql.service +# systemctl stop postgresql.service %files core @@ -379,6 +381,9 @@ systemctl stop postgresql.service %changelog +* Tue Aug 14 2018 Madison Kelly 3.0-13 +- Disabled the postun as it breaks connections to the DB during updates. + * Thu Aug 02 2018 Madison Kelly 3.0-12 - Added perl-Proc-Simple to core dependencies. diff --git a/share/words.xml b/share/words.xml index 89b08654..928401d7 100644 --- a/share/words.xml +++ b/share/words.xml @@ -230,7 +230,7 @@ The database connection error was: all Entering function: [#!variable!function!#] - Connected to: [#!variable!connections!#] database(s). + Connected to: [#!data!sys::db_connections!#] database(s). Failed to read the system UUID. Received a non-UUID string: [#!variable!uuid!#]. Is the user: [#!variable!user!#] in the 'kmem' group? The read host UUID: [#!variable!uuid!#] does not appear to be a valid UUID. Runtime was approximately: [#!variable!runtime!#] seconds. @@ -460,6 +460,14 @@ Here we will inject 't_0006', which injects 't_0001' which has a variable: [#!st #!variable!template!#] in the template file: [#!variable!file!#]. Details of the problem should be in: [#!data!sys::log_file!#].]]> The 'host-uuid': [#!variable!host_uuid!#] is not valid. The '#!variable!switch!#' switch is missing. + The job UUID was passed via '--job-uuid' but the passed in value: [#!variable!uuid!#] is not a valid UUID. + The job UUID was passed via '--job-uuid': [#!variable!uuid!#] doesn't match a job in the database. + +The update appears to have not completed successfully. The output was: +==== +#!variable!output!# +==== + Yes diff --git a/tools/anvil-change-password b/tools/anvil-change-password index 32a3e70b..9fc2d602 100755 --- a/tools/anvil-change-password +++ b/tools/anvil-change-password @@ -48,7 +48,7 @@ if (($< != 0) && ($> != 0)) # Connect $anvil->Database->connect(); -$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }}); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); if (not $anvil->data->{sys}{db_connections}) { # No databases, exit. diff --git a/tools/anvil-configure-striker b/tools/anvil-configure-striker index b81702a0..fedd7bc7 100755 --- a/tools/anvil-configure-striker +++ b/tools/anvil-configure-striker @@ -47,7 +47,7 @@ if (($< != 0) && ($> != 0)) # Connect $anvil->Database->connect; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0031"}); -$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }}); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); if (not $anvil->data->{sys}{db_connections}) { # No databases, exit. diff --git a/tools/anvil-daemon b/tools/anvil-daemon index ce2c1c5e..38bf6c5b 100755 --- a/tools/anvil-daemon +++ b/tools/anvil-daemon @@ -30,7 +30,7 @@ my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1}); # Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks # is to setup the database server. $anvil->Database->connect(); -$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }}); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); # There are some things we only want to run on (re)start and don't need to always run. run_once($anvil); @@ -40,7 +40,7 @@ $anvil->Storage->record_md5sums; # Disconnect. We'll reconnect inside the loop $anvil->Database->disconnect; -$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }}); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); # These are the things we always want running. while(1) diff --git a/tools/anvil-prep-database b/tools/anvil-prep-database index 9b159073..338f3c3b 100755 --- a/tools/anvil-prep-database +++ b/tools/anvil-prep-database @@ -56,7 +56,7 @@ if ($local_uuid) { # Not installed. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0144"}); - exit(5); + $anvil->nice_exit({code => 5}); } elsif (not $running) { @@ -91,7 +91,7 @@ if ($local_uuid) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0050"}); } - exit(1); + $anvil->nice_exit({code => 1}); } else { @@ -203,7 +203,7 @@ if ($local_uuid) { # Failed to start $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0094"}); - exit(2); + $anvil->nice_exit({code => 2}); } } @@ -241,7 +241,7 @@ if ($local_uuid) { # No database user defined $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0099", variables => { uuid => $local_uuid }}); - exit(3); + $anvil->nice_exit({code => 3}); } my $user_list = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT usename, usesysid FROM pg_catalog.pg_user;'\"", source => $THIS_FILE, line => __LINE__}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { user_list => $user_list }}); @@ -278,7 +278,7 @@ if ($local_uuid) if (not $user_exists) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0096", variables => { user => $database_user }}); - exit(4); + $anvil->nice_exit({code => 4}); } # Update/set the passwords. @@ -340,7 +340,7 @@ if ($local_uuid) if (not $database_exists) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0109", variables => { database => $database_name }}); - exit(5); + $anvil->nice_exit({code => 5}); } } diff --git a/tools/anvil-update-states b/tools/anvil-update-states index 63b9547d..1f64ba41 100755 --- a/tools/anvil-update-states +++ b/tools/anvil-update-states @@ -23,7 +23,7 @@ my $anvil = Anvil::Tools->new({log_level => 1, log_secure => 1}); $anvil->Storage->read_config({file => "/etc/anvil/anvil.conf"}); $anvil->Database->connect; -$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }}); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); if (not $anvil->data->{sys}{db_connections}) { # No databases, exit. diff --git a/tools/anvil-update-system b/tools/anvil-update-system index af34bc5f..8e65cfc4 100755 --- a/tools/anvil-update-system +++ b/tools/anvil-update-system @@ -8,8 +8,14 @@ # updated and then rejoin the cluster. # - On DR; This will do nothing until no servers are running, then it will update the system. # -# In all cases, the system will be rebooted +# In all cases, the system will be rebooted if the kernel is updated. # +# Exit codes; +# 0 = Normal exit. +# 1 = No database connections available. +# 2 = The job UUID was passed, but it wasn't valid. +# 3 = It looks like the update failed, reset progress to '0'. + use strict; use warnings; use Anvil::Tools; @@ -24,24 +30,98 @@ if (($running_directory =~ /^\./) && ($ENV{PWD})) $running_directory =~ s/^\./$ENV{PWD}/; } -my $anvil = Anvil::Tools->new({log_level => 1, log_secure => 1}); +my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1}); $anvil->Storage->read_config({file => "/etc/anvil/anvil.conf"}); -$anvil->Database->connect; -$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }}); +# Read switches +$anvil->data->{switches}{'no-reboot'} = 0; +$anvil->data->{switches}{'job-uuid'} = ""; +$anvil->Get->switches; + +# Connect to DBs. +$anvil->Database->connect; +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); if (not $anvil->data->{sys}{db_connections}) { # No databases, exit. print $anvil->Words->string({key => "error_0003"})."\n"; - $anvil->nice_exit({exit_code => 2}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0003"}); + $anvil->nice_exit({exit_code => 1}); } # Did we get called with a job UUID? -$anvil->data->{jobs}{job_uuid} = defined $anvil->data->{switches}{'job-uuid'} ? $anvil->data->{switches}{'job-uuid'} : ""; +if ($anvil->data->{switches}{'job-uuid'}) +{ + # Is it set and valid? + if (not $anvil->Validate->is_uuid({uuid => $anvil->data->{switches}{'job-uuid'}})) + { + # It's not a UUID. + print $anvil->Words->string({key => "error_0033", variables => { uuid => $anvil->data->{switches}{'job-uuid'} }})."\n"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0033", variables => { uuid => $anvil->data->{switches}{'job-uuid'} } }); + $anvil->nice_exit({code => 2}); + } + + # If I'm here, see if we can read the job details. + my $query = " +SELECT + job_host_uuid, + job_data, + job_updated, + job_name, + job_status +FROM + jobs +WHERE + job_uuid = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{switches}{'job-uuid'})." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + if ($count < 1) + { + print $anvil->Words->string({key => "error_0034", variables => { uuid => $anvil->data->{switches}{'job-uuid'} }})."\n"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0034", variables => { uuid => $anvil->data->{switches}{'job-uuid'} } }); + $anvil->nice_exit({code => 2}); + } + + # If we're here, we're good. Load the details + $anvil->data->{jobs}{job_uuid} = $anvil->data->{switches}{'job-uuid'}; + $anvil->data->{jobs}{job_host_uuid} = $results->[0]->[0]; + $anvil->data->{jobs}{job_data} = $results->[0]->[1]; + $anvil->data->{jobs}{job_updated} = $results->[0]->[2]; + $anvil->data->{jobs}{job_name} = $results->[0]->[3]; + $anvil->data->{jobs}{job_status} = $results->[0]->[4]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "jobs::job_uuid" => $anvil->data->{jobs}{job_uuid}, + "jobs::job_host_uuid" => $anvil->data->{jobs}{job_host_uuid}, + "jobs::job_data" => $anvil->data->{jobs}{job_data}, + "jobs::job_updated" => $anvil->data->{jobs}{job_updated}, + "jobs::job_name" => $anvil->data->{jobs}{job_name}, + "jobs::job_status" => $anvil->data->{jobs}{job_status}, + }}); +} + +# Mark that we're starting update_progress($anvil, 1); -run_os_update($anvil); +my ($reboot) = run_os_update($anvil); +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }}); + +# We're done updating +update_progress($anvil, 100); + +# Reboot if needed. +if (($reboot) && (not $anvil->data->{switches}{'no-reboot'})) +{ + # Reboot. + $anvil->System->call({shell_call => $anvil->data->{path}{exe}{'shutdown'}." --reboot now"}); +} $anvil->nice_exit({exit_code => 0}); @@ -54,6 +134,9 @@ sub update_progress { my ($anvil, $progress) = @_; + # Log the progress percentage. + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { progress => $progress }}); + if ($anvil->data->{jobs}{job_uuid}) { my $query = " @@ -61,14 +144,14 @@ UPDATE jobs SET job_picked_up_by = ".$anvil->data->{sys}{use_db_fh}->quote($$).", + job_updated = now(), job_progress = ".$progress.", modified_date = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{sys}{db_timestamp})." WHERE - job_uuid = ".$anvil->data->{sys}{use_db_fh}->quote($job_uuid)." + job_uuid = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{jobs}{job_uuid})." "; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); - $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); - + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); } return(0); @@ -79,36 +162,115 @@ sub run_os_update { my ($anvil) = @_; - my $stdout_file = "/tmp/os_update.stdout"; - my $stderr_file = "/tmp/os_update.stderr"; - my $shell_call = $anvil->data->{path}{exe}{dnf}." clean expire-cache && ".$anvil->data->{path}{exe}{dnf}." -y update"; - - my $process = $anvil->System->call({ - debug => 2, - background => 1, - stdout_file => $stdout_file, - stderr_file => $stderr_file, - shell_call => $job_command." --job-uuid ".$job_uuid, - source => $THIS_FILE, - line => __LINE__, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { process => $process }}); + # NOTE: We run this directly to better monitor progress and update the progress. + my $success = 0; + my $to_update = 0; + my $percent_step = 0; + my $progress = 5; + my $counted_lines = 0; + my $next_step = 0; + my $reboot = 0; + my $output = ""; + my $shell_call = $anvil->data->{path}{exe}{dnf}." clean expire-cache && ".$anvil->data->{path}{exe}{dnf}." -y update; ".$anvil->data->{path}{exe}{echo}." return_code:\$?"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); + open (my $file_handle, $shell_call." 2>&1 |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }}); + while(<$file_handle>) + { + chomp; + my $line = $_; + $output .= $line."\n"; + $line = $anvil->Words->clean_spaces({string => $line}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); - # Record our PID - my $pid = $anvil->data->{jobs}{handles}{$job_uuid}->pid(); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { pid => $pid }}); - my $query = " -UPDATE - jobs -SET - job_picked_up_by = ".$anvil->data->{sys}{use_db_fh}->quote($pid).", - modified_date = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{sys}{db_timestamp})." -WHERE - job_uuid = ".$anvil->data->{sys}{use_db_fh}->quote($job_uuid)." -"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); - $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); + if ($line =~ /^kernel /) + { + # Reboot will be needed. + $reboot = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }}); + } + + if ($line =~ /return_code:(\d+)$/) + { + my $return_code = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }}); + if ($return_code == 0) + { + $success = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { success => $success }}); + } + } + + if ($line =~ / (\d+) Packages$/i) + { + my $counted_lines = $1; + $to_update += $counted_lines; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + counted_lines => $counted_lines, + to_update => $to_update, + }}); + } + + if ($line =~ /Total download size/i) + { + # Ready to install, update to 5%. The next step will count up to 95%. + update_progress($anvil, $progress); + + # The total (reliable) count of events is (to_update * 3), counting '(x/y): ' + # (download), 'Upgrading ', 'Installing ' and 'Verifying '. We ignore the scriplet + # and other lines as it's hard to predict how many there will be, and they pass fast + # enough to not really matter for a progress bar. + $to_update *= 3; + $percent_step = $to_update / 90; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + to_update => $to_update, + percent_step => $percent_step, + }}); + + if ($percent_step =~ /\d+\.\d+/) + { + # Round up. + $percent_step += 1; + $percent_step = int($percent_step); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { percent_step => $percent_step }}); + } + + $next_step = $percent_step; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { next_step => $next_step }}); + } + + # If 'percent_step' is set, we're ready to start counting lines. + if (($percent_step) && (($line =~ /\(\d+\/\d+\): /) or ($line =~ /Upgrading /i) or ($line =~ /Installing /) or ($line =~ /Cleanup /))) + { + $counted_lines++; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { counted_lines => $counted_lines }}); + + if ($counted_lines > $next_step) + { + # Step up the progress. + $next_step += $percent_step; + $progress++; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + next_step => $next_step, + progress => $progress, + }}); + + update_progress($anvil, $progress); + } + } + } + close $file_handle; + # Did it work? + if (not $success) + { + # Nope. + update_progress($anvil, 0); + + print $anvil->Words->string({key => "error_0035", variables => { output => $output }})."\n"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0035", variables => { output => $output } }); + $anvil->nice_exit({code => 3}); + } - return(0); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }}); + return($reboot); }; diff --git a/tools/anvil.sql b/tools/anvil.sql index 931b6e23..e87fb8e0 100644 --- a/tools/anvil.sql +++ b/tools/anvil.sql @@ -357,7 +357,7 @@ CREATE TABLE jobs ( job_uuid uuid not null primary key, -- job_host_uuid uuid not null, -- This is the host that requested the job job_command text not null, -- This is the command to run (usually a shell call). - job_data text not null, -- This is optional data to be used by anvil-data + job_data text not null, -- job_picked_up_by numeric not null default 0, -- This is the PID of the 'anvil-jobs' script that picked up the job. job_picked_up_at numeric not null default 0, -- This is unix timestamp of when the job was picked up. job_updated numeric not null default 0, -- This is unix timestamp that is perdiodically updated for jobs that take a long time. It is used to help determine when a job is hung.