* Changed the call to reboot after OS update to instead set a reboot required flag.

* Created tools/anvil-clear-reboot to clear the "reboot needed" flag. Also created, but not yet using (and may not use) units/anvil-boot-time.service.
* Started work on having jobs show their data via JSON / jquery.
* Updated anvil-update-system to record messages indicating the progress so far.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 6 years ago
parent 51c9d2952d
commit d147e10552
  1. 49
      Anvil/Tools.pm
  2. 1
      Anvil/Tools/System.pm
  3. 14
      cgi-bin/striker
  4. 7
      html/skins/alteeve/config.js
  5. 16
      share/words.xml
  6. 55
      tools/anvil-clear-reboot
  7. 79
      tools/anvil-daemon
  8. 94
      tools/anvil-update-system
  9. 13
      units/anvil-boot-time.service

@ -788,6 +788,29 @@ sub _set_defaults
},
};
$anvil->data->{defaults} = {
database => {
locking => {
reap_age => 300,
}
},
language => {
# Default language for all output shown to a user.
output => 'en_CA',
},
limits => {
# This is the maximum number of times we're allow to loop when injecting variables
# into a string being processed in Anvil::Tools::Words->string();
string_loops => 1000,
},
'log' => {
db_transactions => 0,
facility => "local0",
language => "en_CA",
level => 1,
secure => 0,
server => "",
tag => "anvil",
},
## Network stuff... The second octet auto-increments to handle N-number of netowrks. As such,
## we need to use a wider spread between the BCNs, SNs and IFNs than we had
## in v2.
@ -814,29 +837,6 @@ sub _set_defaults
netmask => "255.255.0.0",
},
},
database => {
locking => {
reap_age => 300,
}
},
language => {
# Default language for all output shown to a user.
output => 'en_CA',
},
limits => {
# This is the maximum number of times we're allow to loop when injecting variables
# into a string being processed in Anvil::Tools::Words->string();
string_loops => 1000,
},
'log' => {
db_transactions => 0,
facility => "local0",
language => "en_CA",
level => 1,
secure => 0,
server => "",
tag => "anvil",
},
template => {
html => "alteeve",
},
@ -942,6 +942,9 @@ sub _set_paths
'lock' => {
database => "/tmp/anvil-tools.database.lock",
},
proc => {
uptime => "/proc/uptime",
},
secure => {
postgres_pgpass => "/var/lib/pgsql/.pgpass",
},

@ -901,6 +901,7 @@ sub maintenance_mode
if ($set)
{
### TODO: stop other systems from using this database.
# Am I enabling or disabling?
if ($set eq "1")
{

@ -278,7 +278,7 @@ sub process_update
debug => 2,
file => $THIS_FILE,
line => __LINE__,
job_command => "anvil-update-system --uuid ".$anvil->Get->host_uuid,
job_command => "anvil-update-system --host-uuid ".$anvil->Get->host_uuid,
job_data => "",
job_name => "update::system",
job_title => "job_0003",
@ -772,10 +772,10 @@ sub check_availability
{
my ($anvil) = @_;
my $debug = 3;
my $debug = 2;
my $available = 1;
# Set maintenance mode.
# Check maintenance mode.
my $maintenance_mode = $anvil->System->maintenance_mode({debug => $debug});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { maintenance_mode => $maintenance_mode }});
@ -783,6 +783,14 @@ sub check_availability
{
$available = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { available => $available }});
$anvil->data->{say}{maintenance} = $anvil->Template->get({file => "striker.html", name => "striker-offline", variables => {
title_id => "",
message_id => "",
title => "#!string!striker_0046!#",
description => $anvil->Words->string({key => "striker_0090", variables => { }}),
}});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'say::maintenance' => $anvil->data->{say}{maintenance} }});
}
# TODO: Phase this out

@ -133,4 +133,11 @@ $( window ).on( "load", function()
}
});
});
$.getJSON('/status/jobs.json', { get_param: 'value' }, function(data) {
$.each(data.ips, function(index, element) {
//console.log('- entry: ['+index+'], on: ['+element.on+'], address: ['+element.address+'], subnet: ['+element.subnet+'].');
//console.log('- gateway: ['+element.gateway+'], dns: ['+element.dns+'], default gateway: ['+element.default_gateway+'].');
});
});
})

@ -58,12 +58,14 @@ NOTE: You must update the password of any other system using this host's
<key name="message_0030">Failed to write the new password to the temporary file: [#!variable!file!#]. Please check the logs for details.</key>
<key name="message_0031">Beginning configuration of local system.</key>
<key name="message_0032">Peer: [#!variable!peer!#], database: [#!variable!name!#], UUID: [#!variable!uuid!#]</key>
<key name="message_0033"></key>
<key name="message_0033">Clearing update cache and checking for available updates.</key>
<key name="message_0034"><![CDATA[Logged in as: [<span class="fixed_width">#!data!sys::users::user_name!#</span>]]]></key>
<key name="message_0035"></key>
<key name="message_0036"></key>
<key name="message_0037"></key>
<key name="message_0038"></key>
<key name="message_0035">Downloading approximately: [#!variable!size!#] worth of updates.</key>
<key name="message_0036">ERROR: There was a problem with the OS update process. Please check the system logs for more details.</key>
<key name="message_0037">Downloading complete. Installation of updates now underway.</key>
<key name="message_0038">Updates finished. Verifying now.</key>
<key name="message_0039">System update complete! The kernel was updated, so a reboot is required.</key>
<key name="message_0040">System update complete! A reboot is not required.</key>
<!-- Log entries -->
<key name="log_0001">Starting: [#!variable!program!#].</key>
@ -297,6 +299,8 @@ The database connection error was:
<key name="log_0196">Failed to reconnect to the database, and now no connections remail. Exiting.</key>
<key name="log_0197"><![CDATA[System->maintenance_mode() was passed an invalid 'set' value: [#!variable!set!#]. No action taken.]]></key>
<key name="log_0198">The user: [#!variable!user!#] logged out successfully.</key>
<key name="log_0199">A system reboot is required, setting the database flag.</key>
<key name="log_0200">A system reboot is required, setting the database flag.</key>
<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
<key name="t_0000">Test</key>
@ -424,6 +428,8 @@ Here we will inject 't_0006', which injects 't_0001' which has a variable: [#!st
<key name="striker_0086">Would you like to update the operating system on this machine? This Striker will be placed into maintenance mode until the update completes.</key>
<key name="striker_0087">When enabled on a Striker dashboard, the web interface will be disabled and ScanCore will not record to the local database. When enabled on a node, no servers will be allowed to run on it, and any already running on it will be migrated. When run on a DR node, that node will be disconnected from storage and no servers will be allowed to run on it. When disabled, all normal functions are available</key>
<key name="striker_0088">The system will be updated momentarily. This system will now be in maintenance mode until the update is complete.</key>
<key name="striker_0089">This indicates whether this system needs to be rebooted or not.</key>
<key name="striker_0090">This system is in maintenance mode and is not currently available.</key>
<!-- Strings used by jobs -->
<key name="job_0001">Configure Network</key>

@ -0,0 +1,55 @@
#!/usr/bin/perl
#
# Exit codes;
# 0 = Normal exit.
# 1 = No database connections available.
use strict;
use warnings;
use Anvil::Tools;
# Disable buffering
$| = 1;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
$anvil->Storage->read_config({file => "/etc/anvil/anvil.conf"});
# Read switches
$anvil->Get->switches;
# Connect to DBs.
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, exit.
print $anvil->Words->string({key => "error_0003"})."\n";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0003"});
$anvil->nice_exit({exit_code => 1});
}
# If the uptime is less than ten minutes, clear the reboot flag.
my $uptime = $anvil->Storage->read_file({file => $anvil->data->{path}{proc}{uptime}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0200"});
$anvil->Database->insert_or_update_variables({
variable_name => "reboot::needed",
variable_value => "0",
variable_default => "0",
variable_description => "striker_0089",
variable_section => "system",
variable_source_uuid => $anvil->Get->host_uuid,
variable_source_table => "hosts",
update_value_only => 1,
});
# We're done
$anvil->nice_exit({exit_code => 0});

@ -86,6 +86,32 @@ sub run_once
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_output => $database_output }});
}
# If the uptime is less than ten minutes, clear the reboot flag.
my $uptime = $anvil->Storage->read_file({
debug => 2,
force_read => 1,
cache => 0,
file => $anvil->data->{path}{proc}{uptime},
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uptime => $uptime }});
# Clean it up. We'll have gotten two numbers, the uptime in seconds (to two decimal places) and the
# total idle time. We only care about the int number.
$uptime =~ s/^(\d+)\..*$/$1/;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uptime => $uptime }});
if ($uptime < 600)
{
# Clear the reboot request.
my $output = $anvil->System->call({
debug => 2,
shell_call => $anvil->data->{path}{exe}{'anvil-clear-reboot'},
source => $THIS_FILE,
line => __LINE__,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output }});
}
return(0);
}
@ -156,6 +182,9 @@ sub run_jobs
{
my ($anvil) = @_;
# We'll also update the jobs.json file.
my $jobs_file = "{\"jobs\":[\n";
# Get a list of pending or incomplete jobs.
my $query = "
SELECT
@ -164,15 +193,15 @@ SELECT
job_data,
job_picked_up_by,
job_picked_up_at,
extract(epoch from job_picked_up_at),
job_updated,
extract(epoch from modified_date)
job_progress
FROM
jobs
WHERE
job_progress != 100
AND
job_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->Get->host_uuid)."
LIMIT 1;";
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
@ -188,18 +217,40 @@ LIMIT 1;";
my $job_data = defined $row->[2] ? $row->[2] : "";
my $job_picked_up_by = $row->[3];
my $job_picked_up_at = $row->[4];
my $unix_picked_up = $row->[3];
my $job_updated = $row->[5];
my $job_progress = $row->[6];
my $unix_updated = $row->[6];
my $job_progress = $row->[7];
my $started_seconds_ago = time - $unix_picked_up;
my $updated_seconds_ago = time - $unix_updated;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
job_uuid => $job_uuid,
job_command => $job_command,
job_data => $job_data,
job_picked_up_by => $job_picked_up_by,
job_picked_up_at => $job_picked_up_at,
job_updated => $job_updated,
job_progress => $job_progress,
job_uuid => $job_uuid,
job_command => $job_command,
job_data => $job_data,
job_picked_up_by => $job_picked_up_by,
job_picked_up_at => $job_picked_up_at,
unix_picked_up => $unix_picked_up,
job_updated => $job_updated,
unix_updated => $unix_updated,
job_progress => $job_progress,
started_seconds_ago => $started_seconds_ago,
updated_seconds_ago => $updated_seconds_ago,
}});
# If the job is done, see if it was recently enough to record in the jobs.json file.
if ($job_progress eq "100")
{
# Record in JSON if it wass last updated less than 5 minutes ago.
if ($updated_seconds_ago < 300)
{
$jobs_file .= "{ \"job_uuid\":\"".$job_uuid."\", \"job_command\":\"".$job_command."\", \"job_data\":\"".$job_data."\", \"job_picked_up_at\":\"".$job_picked_up_at."\", \"job_updated\":\"".$job_updated."\", \"job_progress\":\"".$job_progress."\", \"job_progress\":\"".$job_progress."\", \"started_seconds_ago\":\"".$started_seconds_ago."\", \"updated_seconds_ago\":\"".$updated_seconds_ago."\" }, \n";
}
next;
}
# If we're here, the job isn't done. So first, record it.
$jobs_file .= "{ \"job_uuid\":\"".$job_uuid."\", \"job_command\":\"".$job_command."\", \"job_data\":\"".$job_data."\", \"job_picked_up_at\":\"".$job_picked_up_at."\", \"job_updated\":\"".$job_updated."\", \"job_progress\":\"".$job_progress."\", \"job_progress\":\"".$job_progress."\", \"started_seconds_ago\":\"".$started_seconds_ago."\", \"updated_seconds_ago\":\"".$updated_seconds_ago."\" }, \n";
# See if the job was picked up by another running instance.
if ($job_picked_up_by)
{
@ -227,7 +278,8 @@ LIMIT 1;";
stderr_file => "/tmp/anvil.job.".$job_uuid.".stderr",
shell_call => $job_command." --job-uuid ".$job_uuid,
source => $THIS_FILE,
line => __LINE__});
line => __LINE__,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "jobs::handles::${job_uuid}" => $anvil->data->{jobs}{handles}{$job_uuid} }});
# Record the PID
@ -246,6 +298,9 @@ WHERE
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
}
# Close the jobs file.
$jobs_file .= "]}\n";
return(0);
}

@ -108,23 +108,34 @@ WHERE
}
# Mark that we're starting
update_progress($anvil, 1);
update_progress($anvil, 1, "message_0033");
my ($reboot) = run_os_update($anvil);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }});
# We're done updating
update_progress($anvil, 100);
# Reboot if needed.
if (($reboot) && (not $anvil->data->{switches}{'no-reboot'}))
my ($reboot_needed, $variable_uuid, $modified_date) = $anvil->Database->read_variable({
variable_name => "reboot::needed",
variable_source_uuid => $anvil->Get->host_uuid,
variable_source_table => "hosts",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
reboot_needed => $reboot_needed,
variable_uuid => $variable_uuid,
modified_date => $modified_date,
}});
if ($reboot_needed)
{
update_progress($anvil, 100, "message_0039");
}
else
{
# Reboot.
$anvil->System->call({shell_call => $anvil->data->{path}{exe}{'shutdown'}." --reboot now"});
update_progress($anvil, 100, "message_0040");
}
$anvil->nice_exit({exit_code => 0});
#############################################################################################################
# Functions #
#############################################################################################################
@ -132,13 +143,35 @@ $anvil->nice_exit({exit_code => 0});
# This updates the progress if we were called with a job UUID.
sub update_progress
{
my ($anvil, $progress) = @_;
my ($anvil, $progress, $message) = @_;
# Log the progress percentage.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { progress => $progress }});
if ($anvil->data->{jobs}{job_uuid})
{
# Get the current job_status and append this new one.
my $query = "
SELECT
job_status
FROM
jobs
WHERE
job_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{jobs}{job_uuid})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $job_status = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$job_status = "" if not defined $old_job_status;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { job_status => $job_status }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { message => $message }});
if ($message)
{
$job_status .= $message."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { job_status => $job_status }});
}
my $query = "
UPDATE
jobs
@ -146,6 +179,7 @@ SET
job_picked_up_by = ".$anvil->data->{sys}{database}{use_handle}->quote($$).",
job_updated = now(),
job_progress = ".$progress.",
job_status = ".$anvil->data->{sys}{database}{use_handle}->quote($job_status).",
modified_date = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})."
WHERE
job_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{jobs}{job_uuid})."
@ -169,7 +203,7 @@ sub run_os_update
my $progress = 5;
my $counted_lines = 0;
my $next_step = 0;
my $reboot = 0;
my $verifying = 0;
my $output = "";
my $shell_call = $anvil->data->{path}{exe}{dnf}." clean expire-cache && ".$anvil->data->{path}{exe}{dnf}." -y update; ".$anvil->data->{path}{exe}{echo}." return_code:\$?";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
@ -185,8 +219,31 @@ sub run_os_update
if ($line =~ /^kernel /)
{
# Reboot will be needed.
$reboot = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0199"});
$anvil->Database->insert_or_update_variables({
variable_name => "reboot::needed",
variable_value => "1",
variable_default => "0",
variable_description => "striker_0089",
variable_section => "system",
variable_source_uuid => $anvil->Get->host_uuid,
variable_source_table => "hosts",
update_value_only => 1,
});
}
if ((not $verifying) && ($line =~ /^Verifying /i))
{
# Update done, verifying now.
$verifying = 1;
update_progress($anvil, $progress, "message_0038");
}
if ($line =~ /Running transaction/i)
{
# Done downloading
update_progress($anvil, $progress, "message_0037");
}
if ($line =~ /return_code:(\d+)$/)
@ -210,10 +267,13 @@ sub run_os_update
}});
}
if ($line =~ /Total download size/i)
if ($line =~ /Total download size: (.*)$/i)
{
my $update_size = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update_size => $update_size }});
# Ready to install, update to 5%. The next step will count up to 95%.
update_progress($anvil, $progress);
update_progress($anvil, $progress, "message_0035,!!size!$update_size!!");
# The total (reliable) count of events is (to_update * 3), counting '(x/y): '
# (download), 'Upgrading '/'Installing ' and 'Verifying '. We ignore the scriplet
@ -246,7 +306,7 @@ sub run_os_update
}});
next if $progress > 95;
update_progress($anvil, $progress);
update_progress($anvil, $progress, "");
}
}
}
@ -256,13 +316,11 @@ sub run_os_update
if (not $success)
{
# Nope.
update_progress($anvil, 0);
update_progress($anvil, 0, "message_0036");
print $anvil->Words->string({key => "error_0035", variables => { output => $output }})."\n";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0035", variables => { output => $output } });
$anvil->nice_exit({code => 3});
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }});
return($reboot);
return(0);
};

@ -0,0 +1,13 @@
[Unit]
Description=Anvil! Intelligent Availability Platform - Boot-time tasks
# Don't run until we can talk to our and our peer's databases.
Wants=network.target
Wants=postgresql.service
[Service]
Type=simple
ExecStart=/usr/sbin/anvil-clear-reboot
ExecStop=/bin/kill -WINCH ${MAINPID}
[Install]
WantedBy=default.target
Loading…
Cancel
Save