-
-
+
+
#!string!striker_0053!#
diff --git a/share/words.xml b/share/words.xml
index dc02a3c3..c131d398 100644
--- a/share/words.xml
+++ b/share/words.xml
@@ -254,7 +254,7 @@ The database connection error was:
The local machine's UUID was not read properly. It should be stored in: [#!data!sys::host_uuid!#] and contain hexadecimal characters in the format: '012345-6789-abcd-ef01-23456789abcd' and usually matches the output of 'dmidecode --string system-uuid'. If this file exists and if there is a string in the file, please verify that it is structured correctly.
The database with UUID: [#!variable!uuid!#] for: [#!variable!file!#] is behind.
Anvil! database: [#!variable!database!#] already exists.
- The database on: [#!variable!host!#] (UUID: [#!variable!uuid!#]) is behind. A database resync will be requested.
+ The table: [#!variable!table!#] (and possibly others) in the database on: [#!variable!host!#] (UUID: [#!variable!uuid!#]) is behind by: [#!variable!seconds!#] seconds. A database resync will be requested.
[ Warning ] - Failed to delete the temporary postgres password.
insert_or_update_states() was called but the 'state_host_uuid' parameter was not passed or it is empty. Normally this is set to 'sys::data_uuid'.]]>
[ Error ] - Failed to create the Anvil! database: [#!variable!database!#]
@@ -358,7 +358,7 @@ The database connection error was:
update_progress() called without 'job_uuid' being set, and 'jobs::job_uuid' was also not set. Unable to find the job to update.]]>
update_progress() called with the 'job_uuid': [#!variable!job_uuid!#], which was not found. Unable to find the job to update.]]>
update_progress() called with 'progress' set to an invalid value: [#!variable!progress!#]. This must be a whole number between '0' and '100' (fractions not allowed).]]>
-
+
find_matching_ip(), but it failed to resolve to an IP address.]]>
We've been asked to have the new peer add us. We will now wait for the peer to show up in the 'hosts' table and then request the job for it to add us.
The peer: [#!variable!peer_uuid!#] is not yet in 'hosts', continuing to wait.
@@ -367,9 +367,11 @@ The database connection error was:
The #!variable!uuid_name!#: [#!variable!uuid!#] was passed in, but no record with that UUID was found in the database.
The variable with variable_uuid: [#!variable!variable_uuid!#], variable_source_table: [#!variable!variable_source_table!#] and variable_source_uuid: [#!variable!variable_source_uuid!#] was not found in the database, so unable to update.
The variable: [#!variable!name!#] was expected to be an array reference, but it wasn't. It contained (if anything): [#!variable!value!#].
- The database on: [#!variable!host!#] (UUID: [#!variable!uuid!#]) is missing rows. A database resync will be requested.
+ The table: [#!variable!table!#] (and possibly others) in the database on: [#!variable!host!#] (UUID: [#!variable!uuid!#]) is missing: [#!variable!missing!#] row(s). A database resync will be requested.
insert_or_update_jobs() was called with 'update_progress_only' but without a 'job_uuid' being set.]]>
Writing: [#!variable!to_write!#] record(s) to resync the database on: [#!variable!host_name!#].
+ The connection to the database on: [#!variable!host!#] isn't established, trying again...
+ The connection to the database on: [#!variable!host!#] has been successfully established.
Test
diff --git a/tools/anvil-daemon b/tools/anvil-daemon
index 9ecdc949..4e550301 100755
--- a/tools/anvil-daemon
+++ b/tools/anvil-daemon
@@ -27,6 +27,8 @@ use JSON;
use HTML::Strip;
use HTML::FromText;
+use Data::Dumper;
+
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
@@ -111,10 +113,22 @@ my $check_if_database_is_configured = 0;
while(1)
{
# Connect to the database(s)
- $anvil->Storage->read_config({file => $anvil->data->{path}{configs}{'anvil.conf'}});
+ $anvil->Storage->read_config({force_read => 1, file => $anvil->data->{path}{configs}{'anvil.conf'}});
$anvil->Database->connect({check_if_configured => $check_if_database_is_configured});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
+ ### DEBUG:
+ my $db_count = keys %{$anvil->data->{database}};
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { db_count => $db_count }});
+ foreach my $uuid (keys %{$anvil->data->{database}})
+ {
+ my $host = $anvil->data->{database}{$uuid}{host};
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+ "s1:uuid" => $uuid,
+ "s2:host" => $host,
+ }});
+ }
+
# Mark that we don't want to check the database now.
$check_if_database_is_configured = 0;
@@ -168,7 +182,22 @@ while(1)
}
# Disconnect from the database(s) and sleep now.
- $anvil->Database->disconnect();
+ $anvil->Database->disconnect({debug => 2});
+
+ ### DEBUG:
+ $db_count = 0;
+ $db_count = keys %{$anvil->data->{database}};
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { db_count => $db_count }});
+ foreach my $uuid (keys %{$anvil->data->{database}})
+ {
+ my $host = $anvil->data->{database}{$uuid}{host};
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+ "s1:uuid" => $uuid,
+ "s2:host" => $host,
+ }});
+ }
+
+ die;
sleep(1);
}
@@ -387,7 +416,7 @@ sub run_jobs
my $job_status = $hash_ref->{job_status};
my $started_seconds_ago = $job_picked_up_at ? (time - $job_picked_up_at) : 0;
my $updated_seconds_ago = $job_updated ? (time - $job_updated) : 0;
- $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
job_uuid => $job_uuid,
job_command => $job_command,
job_data => $job_data,
@@ -406,7 +435,7 @@ sub run_jobs
if ($job_progress ne "100")
{
$anvil->data->{sys}{jobs_running} = 1;
- $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::jobs_running" => $anvil->data->{sys}{jobs_running} }});
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "sys::jobs_running" => $anvil->data->{sys}{jobs_running} }});
}
# See if the job was picked up by a now-dead instance.
@@ -422,28 +451,46 @@ sub run_jobs
# If the job is done, just clear the 'job_picked_up_by' and be done.
if ($job_progress ne "100")
{
- # The previous job is gone, but the job isn't finished. Start it again.
- $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "striker_warning_0007", variables => {
- command => $job_command,
- pid => $job_picked_up_by,
- percent => $job_progress,
- }});
-
- # Clear some variables.
- $job_progress = 0;
- $job_status = "message_0056";
- $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
- job_progress => $job_progress,
- job_status => $job_status,
- }});
-
- # Clear the job.
- $anvil->Job->clear({debug => 3, job_uuid => $job_uuid});
+ # It's possible that the job updated to 100% and exited after we
+ # gathered the job data, so we won't restart until we've seen it not
+ # running and not at 100% after 5 loops.
+ if ((not exists $anvil->data->{lost_job_count}{$job_uuid}) or (not defined $anvil->data->{lost_job_count}{$job_uuid}))
+ {
+ $anvil->data->{lost_job_count}{$job_uuid} = 0;
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "lost_job_count::${job_uuid}" => $anvil->data->{lost_job_count}{$job_uuid} }});
+ }
+ if ($anvil->data->{lost_job_count}{$job_uuid} > 5)
+ {
+ # The previous job is gone, but the job isn't finished. Start it again.
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "striker_warning_0007", variables => {
+ command => $job_command,
+ pid => $job_picked_up_by,
+ percent => $job_progress,
+ }});
+
+ # Clear some variables.
+ $job_progress = 0;
+ $job_status = "message_0056";
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+ job_progress => $job_progress,
+ job_status => $job_status,
+ }});
+
+ # Clear the job.
+ $anvil->Job->clear({debug => 3, job_uuid => $job_uuid});
+ $anvil->data->{lost_job_count}{$job_uuid} = 0;
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "lost_job_count::${job_uuid}" => $anvil->data->{lost_job_count}{$job_uuid} }});
+ }
+ else
+ {
+ $anvil->data->{lost_job_count}{$job_uuid}++;
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "lost_job_count::${job_uuid}" => $anvil->data->{lost_job_count}{$job_uuid} }});
+ }
}
# Clear the PID
$job_picked_up_by = 0;
- $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_picked_up_by => $job_picked_up_by }});
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { job_picked_up_by => $job_picked_up_by }});
}
}
diff --git a/tools/anvil-manage-striker-peers b/tools/anvil-manage-striker-peers
index 1ed122d8..a476dd31 100755
--- a/tools/anvil-manage-striker-peers
+++ b/tools/anvil-manage-striker-peers
@@ -41,9 +41,15 @@ $| = 1;
my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
# Read switches
-$anvil->data->{switches}{list} = "";
-$anvil->data->{switches}{add} = 0;
-$anvil->data->{switches}{remove} = 0;
+$anvil->data->{switches}{list} = "";
+$anvil->data->{switches}{add} = 0;
+$anvil->data->{switches}{remove} = 0;
+$anvil->data->{switches}{'job-uuid'} = "";
+$anvil->data->{switches}{'host-uuid'} = "";
+$anvil->data->{switches}{'host'} = "";
+$anvil->data->{switches}{'port'} = 5432;
+$anvil->data->{switches}{'password-file'} = "";
+$anvil->data->{switches}{'ping'} = 0;
$anvil->Get->switches;
# Make sure we're running as 'root'
@@ -63,9 +69,9 @@ $anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
# Am I adding, editing or deleting?
-if (not $anvil->data->{switches}{list})
+if ((not $anvil->data->{switches}{list}) && ($anvil->data->{switches}{'host-uuid'}))
{
- process_entry($anvil) ;
+ process_entry($anvil);
}
### Report the peers.
@@ -141,12 +147,12 @@ sub process_entry
{
my ($anvil) = @_;
- my $job_uuid = defined $anvil->data->{switches}{'job-uuid'} ? $anvil->data->{switches}{'job-uuid'} : "";
- my $host_uuid = defined $anvil->data->{switches}{'host-uuid'} ? $anvil->data->{switches}{'host-uuid'} : "";
- my $host = defined $anvil->data->{switches}{'host'} ? $anvil->data->{switches}{'host'} : "";
- my $port = defined $anvil->data->{switches}{'port'} ? $anvil->data->{switches}{'port'} : 5432;
- my $password_file = defined $anvil->data->{switches}{'password-file'} ? $anvil->data->{switches}{'password-file'} : "";
- my $ping = defined $anvil->data->{switches}{'ping'} ? $anvil->data->{switches}{'ping'} : 0;
+ my $job_uuid = $anvil->data->{switches}{'job-uuid'};
+ my $host_uuid = $anvil->data->{switches}{'host-uuid'};
+ my $host = $anvil->data->{switches}{'host'};
+ my $port = $anvil->data->{switches}{'port'};
+ my $password_file = $anvil->data->{switches}{'password-file'};
+ my $ping = $anvil->data->{switches}{'ping'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => 0, level => 2, list => {
job_uuid => $job_uuid,
host_uuid => $host_uuid,
@@ -537,7 +543,35 @@ sub process_entry
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
- # Now loop until we see the peer's host_uuid show up
+ # Now loop until we see the peer's host_uuid show up and we have a connection to the peer's
+ # database.
+ my $peer_connected = 0;
+ until($peer_connected)
+ {
+ my $say_host = $anvil->Get->host_name({host_uuid => $host_uuid});
+ $say_host = $host_uuid if not $say_host;
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+ say_host => $say_host,
+ "cache::database_handle::${host_uuid}" => $anvil->data->{cache}{database_handle}{$host_uuid},
+ }});
+ if ($anvil->data->{cache}{database_handle}{$host_uuid} =~ /^DBI::db=HASH/)
+ {
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0223", variables => { host => $say_host }});
+ $peer_connected = 1;
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { peer_connected => $peer_connected }});
+ }
+ else
+ {
+ # Sleep and then try again to connect.
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0222", variables => { host => $say_host }});
+
+ sleep 1;
+
+ $anvil->Database->connect({db_uuid => $host_uuid});
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
+ }
+ }
+
my $peer_host_name = "";
my $host_seen = 0;
until($host_seen)
|