From 633da25d070b58448361dd5ad161529b732acadf Mon Sep 17 00:00:00 2001 From: Digimer Date: Fri, 24 Aug 2018 00:52:56 -0400 Subject: [PATCH] * Fixed a bug in Database->connect where an empty 'database::' would cause an error. Updated Database->disconnect to delete the 'database' hash key as part of the same fix. * Renamed 'database::locking' to 'sys::database::locking' to avoid collisions with 'database' keys. * Fixed a problem with System->call where reidrects were missing the Proc::Simple method name. * Updated anvil-daemon to check if there is no database connections on start-up, run prep-database if not, and try connecting again. If it still fails, exit. Also updated the main loop to reconnect to the database(s) and skip if non are available. Did more work on the keep_running() function. Signed-off-by: Digimer --- Anvil/Tools/Database.pm | 20 ++++--- Anvil/Tools/System.pm | 6 +- share/words.xml | 7 ++- tools/anvil-daemon | 122 +++++++++++++++++++++++++++++--------- tools/anvil-update-system | 8 ++- 5 files changed, 121 insertions(+), 42 deletions(-) diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index b381e8a9..1fad5432 100755 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -687,6 +687,9 @@ sub connect my $successful_connections = []; foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{database}}) { + # Periodically, autovivication causes and empty key to appear. + next if ((not $uuid) or (not $anvil->Validate->is_uuid({uuid => $uuid}))); + if (($db_uuid) && ($db_uuid ne $uuid)) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0191", variables => { db_uuid => $db_uuid, uuid => $uuid }}); @@ -1198,6 +1201,9 @@ sub disconnect delete $anvil->data->{sys}{database}{use_handle}; delete $anvil->data->{sys}{database}{read_uuid}; + # Delete any database information (reconnects should re-read anvil.conf anyway). + delete $anvil->data->{database}; + # Set the connection count to 0. $anvil->data->{sys}{database}{connections} = 0; @@ -4095,13 +4101,13 @@ sub locking }}); # Make sure we have a sane lock age - if ((not defined $anvil->data->{database}{locking}{reap_age}) or - (not $anvil->data->{database}{locking}{reap_age}) or - ($anvil->data->{database}{locking}{reap_age} =~ /\D/) + if ((not defined $anvil->data->{sys}{database}{locking}{reap_age}) or + (not $anvil->data->{sys}{database}{locking}{reap_age}) or + ($anvil->data->{sys}{database}{locking}{reap_age} =~ /\D/) ) { - $anvil->data->{database}{locking}{reap_age} = $anvil->data->{defaults}{database}{locking}{reap_age}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "database::locking::reap_age" => $anvil->data->{database}{locking}{reap_age} }}); + $anvil->data->{sys}{database}{locking}{reap_age} = $anvil->data->{defaults}{database}{locking}{reap_age}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::locking::reap_age" => $anvil->data->{sys}{database}{locking}{reap_age} }}); } # If I have been asked to check, we will return the variable_uuid if a lock is set. @@ -4180,7 +4186,7 @@ sub locking } # We always check for, and then wait for, locks. Read in the locks, if any. If any are set and they are - # younger than database::locking::reap_age, we'll hold. + # younger than sys::database::locking::reap_age, we'll hold. my $waiting = 1; while ($waiting) { @@ -4201,7 +4207,7 @@ sub locking my $lock_source_uuid = $2; my $lock_time = $3; my $current_time = time; - my $timeout_time = $lock_time + $anvil->data->{database}{locking}{reap_age}; + my $timeout_time = $lock_time + $anvil->data->{sys}{database}{locking}{reap_age}; my $lock_age = $current_time - $lock_time; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { lock_source_name => $lock_source_name, diff --git a/Anvil/Tools/System.pm b/Anvil/Tools/System.pm index 2c101499..2cfb9703 100755 --- a/Anvil/Tools/System.pm +++ b/Anvil/Tools/System.pm @@ -233,15 +233,15 @@ sub call # Setup output files if (($stderr_file) && ($stdout_file)) { - $process->($stdout_file, $stderr_file); + $process->redirect_output($stdout_file, $stderr_file); } elsif ($stdout_file) { - $process->($stdout_file, undef); + $process->redirect_output($stdout_file, undef); } elsif ($stderr_file) { - $process->(undef, $stderr_file); + $process->redirect_output(undef, $stderr_file); } my $status = $process->start($shell_call); diff --git a/share/words.xml b/share/words.xml index 06e3ffa4..6c6d18c8 100644 --- a/share/words.xml +++ b/share/words.xml @@ -88,6 +88,7 @@ Report if a reboot is needed: #!variable!program!# This system needs to be rebooted. This system does NOT need to be rebooted. + Exiting to '--run-once' switch. Starting: [#!variable!program!#]. @@ -166,7 +167,7 @@ Connecting to Database with configuration ID: [#!variable!uuid!#] Database user: [#!variable!user!#] already exists with ID: [#!variable!uuid!#]. users_home() was asked to find the home directory for the user: [#!variable!user!#], but was unable to do so.]]> SSH session opened without a password to: [#!variable!target!#]. - The database: [#!variable!host!# -> #!variable!name!#] with the ID: [#!variable!uuid!#] did not respond to pings and 'database::#!variable!uuid!#::ping' is not set to '0' in '#!data!path::configs::anvil.conf!#', skipping it. + The database: [#!variable!host!# -> #!variable!name!#] with the UUID: [#!variable!uuid!#] did not respond to pings and 'database::#!variable!uuid!#::ping' is not set to '0' in '#!data!path::configs::anvil.conf!#', skipping it. [ Warning ] - The database: [#!variable!name!#] on host: [#!variable!host!#] with ID: [#!variable!uuid!#] can not be used, skipping it. The database connection error was: @@ -323,6 +324,9 @@ The database connection error was: The user: [#!variable!user!#] logged out successfully. A system reboot is required, setting the database flag. A system reboot is required, setting the database flag. + Unable to connect to any database. Will try to initialize the local system and then try again. + Failed to connect to any databases. Skipping the loop of the daemon. + Disconnected from all databases. Will reconnect when entering the main loop. Test @@ -514,6 +518,7 @@ The update appears to have not completed successfully. The output was: #!variable!output!# ==== + Yes diff --git a/tools/anvil-daemon b/tools/anvil-daemon index 605a38d1..db85009d 100755 --- a/tools/anvil-daemon +++ b/tools/anvil-daemon @@ -6,8 +6,10 @@ # Exit codes; # 0 = Normal exit # 1 = md5sum of this program changed. Exited to reload. +# 2 = Unable to connect to any database, even after trying to initialize the local system. # # TODO: +# - Need to check what kind of machine this is and not prep the database unless its a dashboard. # use strict; @@ -32,21 +34,58 @@ my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1}); $anvil->Database->connect(); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); +# If I have no databases, sleep for a second and then exit (systemd will restart us). +if (not $anvil->data->{sys}{database}{connections}) +{ + # Try to configure the local database, and then try to connect again. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0201"}); + prep_database($anvil); + sleep 1; + + # Try connecting again + $anvil->Database->connect(); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); + if (not $anvil->data->{sys}{database}{connections}) + { + # Still nothing, sleep and exit. + print $anvil->Words->string({key => "error_0003"})."\n"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0003"}); + $anvil->nice_exit({exit_code => 2}); + } +} + +# Read switches +$anvil->data->{switches}{'run-once'} = ""; +$anvil->data->{switches}{'no-run-once'} = ""; +$anvil->Get->switches; + # There are some things we only want to run on (re)start and don't need to always run. -run_once($anvil); +run_once($anvil) if not $anvil->data->{switches}{'no-run-once'}; # Calculate my sum so that we can exit if it changes later. $anvil->Storage->record_md5sums; # Disconnect. We'll reconnect inside the loop $anvil->Database->disconnect; -$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0203"}); # These are the things we always want running. while(1) { - # Loop and sleep for 2s. - keep_running($anvil); + # Connect to the database(s) + $anvil->Storage->read_config({file => "/etc/anvil/anvil.conf"}); + $anvil->Database->connect; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); + + if ($anvil->data->{sys}{database}{connections}) + { + # Loop and sleep for 2s. + keep_running($anvil); + } + else + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "log_0202"}); + } # Exit if called with '--run-once' if ($anvil->data->{switches}{'run-once'}) @@ -61,6 +100,13 @@ while(1) $anvil->nice_exit({code => 1}); } + # Exit if 'run-once' selected. + if ($anvil->data->{switches}{'run-once'}) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "warn", key => "message_0055"}); + $anvil->nice_exit({code => 0}); + } + # Sleep now. sleep 2; } @@ -78,13 +124,7 @@ sub run_once my ($anvil) = @_; # Check that the database is ready. - my $shell_call = $anvil->data->{path}{exe}{'anvil-prep-database'}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }}); - my $database_output = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__}); - if ($database_output) - { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_output => $database_output }}); - } + prep_database($anvil); # If the uptime is less than ten minutes, clear the reboot flag. my $uptime = $anvil->Storage->read_file({ @@ -115,6 +155,22 @@ sub run_once return(0); } +# Configure the local database, if needed. +sub prep_database +{ + my ($anvil) = @_; + + my $shell_call = $anvil->data->{path}{exe}{'anvil-prep-database'}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }}); + my $database_output = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__}); + if ($database_output) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_output => $database_output }}); + } + + return(0); +} + # These are tools that need to keep running. sub keep_running { @@ -193,9 +249,7 @@ SELECT job_data, job_picked_up_by, job_picked_up_at, - extract(epoch from job_picked_up_at), job_updated, - extract(epoch from modified_date) job_progress FROM jobs @@ -212,44 +266,43 @@ WHERE }}); foreach my $row (@{$results}) { - my $job_uuid = $row->[0]; - my $job_command = $row->[1]; - my $job_data = defined $row->[2] ? $row->[2] : ""; - my $job_picked_up_by = $row->[3]; - my $job_picked_up_at = $row->[4]; - my $unix_picked_up = $row->[3]; - my $job_updated = $row->[5]; - my $unix_updated = $row->[6]; - my $job_progress = $row->[7]; - my $started_seconds_ago = time - $unix_picked_up; - my $updated_seconds_ago = time - $unix_updated; + my $job_uuid = $row->[0]; + my $job_command = $row->[1]; + my $job_data = defined $row->[2] ? $row->[2] : ""; + my $job_picked_up_by = $row->[3]; + my $job_picked_up_at = $row->[4]; + my $job_updated = $row->[5]; + my $job_progress = $row->[6]; + my $started_seconds_ago = $job_picked_up_at ? (time - $job_picked_up_at) : 0; + my $updated_seconds_ago = $job_updated ? (time - $job_updated) : 0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid, job_command => $job_command, job_data => $job_data, job_picked_up_by => $job_picked_up_by, job_picked_up_at => $job_picked_up_at, - unix_picked_up => $unix_picked_up, job_updated => $job_updated, - unix_updated => $unix_updated, job_progress => $job_progress, started_seconds_ago => $started_seconds_ago, updated_seconds_ago => $updated_seconds_ago, }}); + # This will be appended to the json file + my $json_string = "{ \"job_uuid\":\"".$job_uuid."\", \"job_command\":\"".$job_command."\", \"job_data\":\"".$job_data."\", \"job_picked_up_at\":\"".$job_picked_up_at."\", \"job_updated\":\"".$job_updated."\", \"job_progress\":\"".$job_progress."\", \"job_progress\":\"".$job_progress."\", \"started_seconds_ago\":\"".$started_seconds_ago."\", \"updated_seconds_ago\":\"".$updated_seconds_ago."\" }, \n"; + # If the job is done, see if it was recently enough to record in the jobs.json file. if ($job_progress eq "100") { # Record in JSON if it wass last updated less than 5 minutes ago. if ($updated_seconds_ago < 300) { - $jobs_file .= "{ \"job_uuid\":\"".$job_uuid."\", \"job_command\":\"".$job_command."\", \"job_data\":\"".$job_data."\", \"job_picked_up_at\":\"".$job_picked_up_at."\", \"job_updated\":\"".$job_updated."\", \"job_progress\":\"".$job_progress."\", \"job_progress\":\"".$job_progress."\", \"started_seconds_ago\":\"".$started_seconds_ago."\", \"updated_seconds_ago\":\"".$updated_seconds_ago."\" }, \n"; + $jobs_file .= $json_string; } next; } # If we're here, the job isn't done. So first, record it. - $jobs_file .= "{ \"job_uuid\":\"".$job_uuid."\", \"job_command\":\"".$job_command."\", \"job_data\":\"".$job_data."\", \"job_picked_up_at\":\"".$job_picked_up_at."\", \"job_updated\":\"".$job_updated."\", \"job_progress\":\"".$job_progress."\", \"job_progress\":\"".$job_progress."\", \"started_seconds_ago\":\"".$started_seconds_ago."\", \"updated_seconds_ago\":\"".$updated_seconds_ago."\" }, \n"; + $jobs_file .= $json_string; # See if the job was picked up by another running instance. if ($job_picked_up_by) @@ -300,6 +353,19 @@ WHERE # Close the jobs file. $jobs_file .= "]}\n"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { jobs_file => $jobs_file }}); + + # Write the JSON file + my $output_json = $anvil->data->{path}{directories}{html}."/status/jobs.json"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output_xml => $output_json }}); + $anvil->Storage->write_file({ + file => $output_json, + body => $jobs_file, + overwrite => 1, + mode => "0644", + user => "apache", + group => "apache" + }); return(0); } diff --git a/tools/anvil-update-system b/tools/anvil-update-system index 901b1d10..7e21fd00 100755 --- a/tools/anvil-update-system +++ b/tools/anvil-update-system @@ -35,7 +35,6 @@ my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1}); $anvil->Storage->read_config({file => "/etc/anvil/anvil.conf"}); # Read switches -$anvil->data->{switches}{'no-reboot'} = 0; $anvil->data->{switches}{'job-uuid'} = ""; $anvil->Get->switches; @@ -110,8 +109,11 @@ WHERE # Mark that we're starting update_progress($anvil, 1, "message_0033"); -my ($reboot) = run_os_update($anvil); -$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot => $reboot }}); +print $THIS_FILE." running with PID: [".$$."]\n" +sleep 60; +exit; + +run_os_update($anvil); # We're done updating my $reboot_needed = $anvil->System->reboot_needed({debug => 2, set => 1});