seasharp/anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Fixed a bug in Database->connect where an empty 'database::<key>' would cause an error. Updated Database->disconnect to delete the 'database' hash key as part of the same fix.

Browse Source 
* Renamed 'database::locking' to 'sys::database::locking' to avoid collisions with 'database' keys.
* Fixed a problem with System->call where reidrects were missing the Proc::Simple method name.
* Updated anvil-daemon to check if there is no database connections on start-up, run prep-database if not, and try connecting again. If it still fails, exit. Also updated the main loop to reconnect to the database(s) and skip if non are available. Did more work on the keep_running() function.

Signed-off-by: Digimer <digimer@alteeve.ca>
This commit is contained in:
Digimer 2018-08-24 00:52:56 -04:00
parent c3a26c32f0
commit 633da25d07
5 changed files with 121 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Anvil/Tools/Database.pm
View File

@ -687,6 +687,9 @@ sub connect
my $successful_connections = [];
foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{database}})
{
# Periodically, autovivication causes and empty key to appear.
next if ((not $uuid) or (not $anvil->Validate->is_uuid({uuid => $uuid})));
if (($db_uuid) && ($db_uuid ne $uuid))
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0191", variables => { db_uuid => $db_uuid, uuid => $uuid }});
@ -1198,6 +1201,9 @@ sub disconnect
delete $anvil->data->{sys}{database}{use_handle};
delete $anvil->data->{sys}{database}{read_uuid};
# Delete any database information (reconnects should re-read anvil.conf anyway).
delete $anvil->data->{database};
# Set the connection count to 0.
$anvil->data->{sys}{database}{connections} = 0;
@ -4095,13 +4101,13 @@ sub locking
}});
# Make sure we have a sane lock age
if ((not defined $anvil->data->{database}{locking}{reap_age}) or
(not $anvil->data->{database}{locking}{reap_age}) or
($anvil->data->{database}{locking}{reap_age} =~ /\D/)
if ((not defined $anvil->data->{sys}{database}{locking}{reap_age}) or
(not $anvil->data->{sys}{database}{locking}{reap_age}) or
($anvil->data->{sys}{database}{locking}{reap_age} =~ /\D/)
)
{
$anvil->data->{database}{locking}{reap_age} = $anvil->data->{defaults}{database}{locking}{reap_age};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "database::locking::reap_age" => $anvil->data->{database}{locking}{reap_age} }});
$anvil->data->{sys}{database}{locking}{reap_age} = $anvil->data->{defaults}{database}{locking}{reap_age};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::locking::reap_age" => $anvil->data->{sys}{database}{locking}{reap_age} }});
}
# If I have been asked to check, we will return the variable_uuid if a lock is set.
@ -4180,7 +4186,7 @@ sub locking
}
# We always check for, and then wait for, locks. Read in the locks, if any. If any are set and they are
# younger than database::locking::reap_age, we'll hold.
# younger than sys::database::locking::reap_age, we'll hold.
my $waiting = 1;
while ($waiting)
{
@ -4201,7 +4207,7 @@ sub locking
my $lock_source_uuid = $2;
my $lock_time = $3;
my $current_time = time;
my $timeout_time = $lock_time + $anvil->data->{database}{locking}{reap_age};
my $timeout_time = $lock_time + $anvil->data->{sys}{database}{locking}{reap_age};
my $lock_age = $current_time - $lock_time;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
lock_source_name => $lock_source_name,

6
Anvil/Tools/System.pm
View File

@ -233,15 +233,15 @@ sub call
# Setup output files
if (($stderr_file) && ($stdout_file))
{
$process->($stdout_file, $stderr_file);
$process->redirect_output($stdout_file, $stderr_file);
}
elsif ($stdout_file)
{
$process->($stdout_file, undef);
$process->redirect_output($stdout_file, undef);
}
elsif ($stderr_file)
{
$process->(undef, $stderr_file);
$process->redirect_output(undef, $stderr_file);
}
my $status = $process->start($shell_call);

7
share/words.xml
View File

@ -88,6 +88,7 @@ Report if a reboot is needed: #!variable!program!#
</key>
<key name="message_0053">This system needs to be rebooted.</key>
<key name="message_0054">This system does NOT need to be rebooted.</key>
<key name="message_0055">Exiting to '--run-once' switch.</key>
<!-- Log entries -->
<key name="log_0001">Starting: [#!variable!program!#].</key>
@ -166,7 +167,7 @@ Connecting to Database with configuration ID: [#!variable!uuid!#]
<key name="log_0060">Database user: [#!variable!user!#] already exists with ID: [#!variable!uuid!#].</key>
<key name="log_0061"><![CDATA[[ Error ] - The method Get->users_home() was asked to find the home directory for the user: [#!variable!user!#], but was unable to do so.]]></key>
<key name="log_0062">SSH session opened without a password to: [#!variable!target!#].</key>
<key name="log_0063">The database: [#!variable!host!# -> #!variable!name!#] with the ID: [#!variable!uuid!#] did not respond to pings and 'database::#!variable!uuid!#::ping' is not set to '0' in '#!data!path::configs::anvil.conf!#', skipping it.</key>
<key name="log_0063">The database: [#!variable!host!# -> #!variable!name!#] with the UUID: [#!variable!uuid!#] did not respond to pings and 'database::#!variable!uuid!#::ping' is not set to '0' in '#!data!path::configs::anvil.conf!#', skipping it.</key>
<key name="log_0064">[ Warning ] - The database: [#!variable!name!#] on host: [#!variable!host!#] with ID: [#!variable!uuid!#] can not be used, skipping it.</key>
<key name="log_0065">
The database connection error was:
@ -323,6 +324,9 @@ The database connection error was:
<key name="log_0198">The user: [#!variable!user!#] logged out successfully.</key>
<key name="log_0199">A system reboot is required, setting the database flag.</key>
<key name="log_0200">A system reboot is required, setting the database flag.</key>
<key name="log_0201">Unable to connect to any database. Will try to initialize the local system and then try again.</key>
<key name="log_0202">Failed to connect to any databases. Skipping the loop of the daemon.</key>
<key name="log_0203">Disconnected from all databases. Will reconnect when entering the main loop.</key>
<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
<key name="t_0000">Test</key>
@ -514,6 +518,7 @@ The update appears to have not completed successfully. The output was:
#!variable!output!#
====
</key>
<key name="error_0036"></key>
<!-- These are units, words and so on used when displaying information. -->
<key name="unit_0001">Yes</key>

122
tools/anvil-daemon
View File

@ -6,8 +6,10 @@
# Exit codes;
# 0 = Normal exit
# 1 = md5sum of this program changed. Exited to reload.
# 2 = Unable to connect to any database, even after trying to initialize the local system.
#
# TODO:
# - Need to check what kind of machine this is and not prep the database unless its a dashboard.
#
use strict;
@ -32,21 +34,58 @@ my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
# If I have no databases, sleep for a second and then exit (systemd will restart us).
if (not $anvil->data->{sys}{database}{connections})
{
# Try to configure the local database, and then try to connect again.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0201"});
prep_database($anvil);
sleep 1;
# Try connecting again
$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# Still nothing, sleep and exit.
print $anvil->Words->string({key => "error_0003"})."\n";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0003"});
$anvil->nice_exit({exit_code => 2});
}
}
# Read switches
$anvil->data->{switches}{'run-once'} = "";
$anvil->data->{switches}{'no-run-once'} = "";
$anvil->Get->switches;
# There are some things we only want to run on (re)start and don't need to always run.
run_once($anvil);
run_once($anvil) if not $anvil->data->{switches}{'no-run-once'};
# Calculate my sum so that we can exit if it changes later.
$anvil->Storage->record_md5sums;
# Disconnect. We'll reconnect inside the loop
$anvil->Database->disconnect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0203"});
# These are the things we always want running.
while(1)
{
# Loop and sleep for 2s.
keep_running($anvil);
# Connect to the database(s)
$anvil->Storage->read_config({file => "/etc/anvil/anvil.conf"});
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if ($anvil->data->{sys}{database}{connections})
{
# Loop and sleep for 2s.
keep_running($anvil);
}
else
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "log_0202"});
}
# Exit if called with '--run-once'
if ($anvil->data->{switches}{'run-once'})
@ -61,6 +100,13 @@ while(1)
$anvil->nice_exit({code => 1});
}
# Exit if 'run-once' selected.
if ($anvil->data->{switches}{'run-once'})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "warn", key => "message_0055"});
$anvil->nice_exit({code => 0});
}
# Sleep now.
sleep 2;
}
@ -78,13 +124,7 @@ sub run_once
my ($anvil) = @_;
# Check that the database is ready.
my $shell_call = $anvil->data->{path}{exe}{'anvil-prep-database'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
my $database_output = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
if ($database_output)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_output => $database_output }});
}
prep_database($anvil);
# If the uptime is less than ten minutes, clear the reboot flag.
my $uptime = $anvil->Storage->read_file({
@ -115,6 +155,22 @@ sub run_once
return(0);
}
# Configure the local database, if needed.
sub prep_database
{
my ($anvil) = @_;
my $shell_call = $anvil->data->{path}{exe}{'anvil-prep-database'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
my $database_output = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
if ($database_output)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_output => $database_output }});
}
return(0);
}
# These are tools that need to keep running.
sub keep_running
{
@ -193,9 +249,7 @@ SELECT
job_data,
job_picked_up_by,
job_picked_up_at,
extract(epoch from job_picked_up_at),
job_updated,
extract(epoch from modified_date)
job_progress
FROM
jobs
@ -212,44 +266,43 @@ WHERE
}});
foreach my $row (@{$results})
{
my $job_uuid = $row->[0];
my $job_command = $row->[1];
my $job_data = defined $row->[2] ? $row->[2] : "";
my $job_picked_up_by = $row->[3];
my $job_picked_up_at = $row->[4];
my $unix_picked_up = $row->[3];
my $job_updated = $row->[5];
my $unix_updated = $row->[6];
my $job_progress = $row->[7];
my $started_seconds_ago = time - $unix_picked_up;
my $updated_seconds_ago = time - $unix_updated;
my $job_uuid = $row->[0];
my $job_command = $row->[1];
my $job_data = defined $row->[2] ? $row->[2] : "";
my $job_picked_up_by = $row->[3];
my $job_picked_up_at = $row->[4];
my $job_updated = $row->[5];
my $job_progress = $row->[6];
my $started_seconds_ago = $job_picked_up_at ? (time - $job_picked_up_at) : 0;
my $updated_seconds_ago = $job_updated ? (time - $job_updated) : 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
job_uuid => $job_uuid,
job_command => $job_command,
job_data => $job_data,
job_picked_up_by => $job_picked_up_by,
job_picked_up_at => $job_picked_up_at,
unix_picked_up => $unix_picked_up,
job_updated => $job_updated,
unix_updated => $unix_updated,
job_progress => $job_progress,
started_seconds_ago => $started_seconds_ago,
updated_seconds_ago => $updated_seconds_ago,
}});
# This will be appended to the json file
my $json_string = "{ \"job_uuid\":\"".$job_uuid."\", \"job_command\":\"".$job_command."\", \"job_data\":\"".$job_data."\", \"job_picked_up_at\":\"".$job_picked_up_at."\", \"job_updated\":\"".$job_updated."\", \"job_progress\":\"".$job_progress."\", \"job_progress\":\"".$job_progress."\", \"started_seconds_ago\":\"".$started_seconds_ago."\", \"updated_seconds_ago\":\"".$updated_seconds_ago."\" }, \n";
# If the job is done, see if it was recently enough to record in the jobs.json file.
if ($job_progress eq "100")
{
# Record in JSON if it wass last updated less than 5 minutes ago.
if ($updated_seconds_ago < 300)
{
$jobs_file .= "{ \"job_uuid\":\"".$job_uuid."\", \"job_command\":\"".$job_command."\", \"job_data\":\"".$job_data."\", \"job_picked_up_at\":\"".$job_picked_up_at."\", \"job_updated\":\"".$job_updated."\", \"job_progress\":\"".$job_progress."\", \"job_progress\":\"".$job_progress."\", \"started_seconds_ago\":\"".$started_seconds_ago."\", \"updated_seconds_ago\":\"".$updated_seconds_ago."\" }, \n";
$jobs_file .= $json_string;
}
next;
}
# If we're here, the job isn't done. So first, record it.
$jobs_file .= "{ \"job_uuid\":\"".$job_uuid."\", \"job_command\":\"".$job_command."\", \"job_data\":\"".$job_data."\", \"job_picked_up_at\":\"".$job_picked_up_at."\", \"job_updated\":\"".$job_updated."\", \"job_progress\":\"".$job_progress."\", \"job_progress\":\"".$job_progress."\", \"started_seconds_ago\":\"".$started_seconds_ago."\", \"updated_seconds_ago\":\"".$updated_seconds_ago."\" }, \n";
$jobs_file .= $json_string;
# See if the job was picked up by another running instance.
if ($job_picked_up_by)
@ -300,6 +353,19 @@ WHERE
# Close the jobs file.
$jobs_file .= "]}\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { jobs_file => $jobs_file }});
# Write the JSON file
my $output_json = $anvil->data->{path}{directories}{html}."/status/jobs.json";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output_xml => $output_json }});
$anvil->Storage->write_file({
file => $output_json,
body => $jobs_file,
overwrite => 1,
mode => "0644",
user => "apache",
group => "apache"
});
return(0);
}

8
tools/anvil-update-system
View File

@ -35,7 +35,6 @@ my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
$anvil->Storage->read_config({file => "/etc/anvil/anvil.conf"});
# Read switches
$anvil->data->{switches}{'no-reboot'} = 0;
$anvil->data->{switches}{'job-uuid'} = "";
$anvil->Get->switches;
@ -110,8 +109,11 @@ WHERE
# Mark that we're starting
update_progress($anvil, 1, "message_0033");
my ($reboot) = run_os_update($anvil);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }});
print $THIS_FILE." running with PID: [".$$."]\n"
sleep 60;
exit;
run_os_update($anvil);
# We're done updating
my $reboot_needed = $anvil->System->reboot_needed({debug => 2, set => 1});