diff --git a/AN/Tools.pm b/AN/Tools.pm index 3bb6b3c4..f5ce9583 100755 --- a/AN/Tools.pm +++ b/AN/Tools.pm @@ -606,6 +606,7 @@ sub _set_defaults "variables", "alert_sent", "states", + "updated", ], local_lock_active => 0, locking_reap_age => 300, diff --git a/AN/Tools.sql b/AN/Tools.sql index 27316c9d..d74b286d 100644 --- a/AN/Tools.sql +++ b/AN/Tools.sql @@ -253,7 +253,9 @@ CREATE TRIGGER trigger_variables -- These are special tables with no history or tracking UUIDs that simply record transient information. -- -- ------------------------------------------------------------------------------------------------------- -- --- This table records the last time a scan ran. + +-- This table records the last time a scan ran. It's sole purpose is to make sure at least one table's +-- 'modified_date' changes per run, so that database resyncs can be triggered reliably. CREATE TABLE updated ( updated_host_uuid uuid not null, updated_by text not null, -- The name of the agent (or "ScanCore' itself) that updated. diff --git a/AN/Tools/Database.pm b/AN/Tools/Database.pm index 482f19bf..5b9404e0 100755 --- a/AN/Tools/Database.pm +++ b/AN/Tools/Database.pm @@ -2742,7 +2742,7 @@ sub _find_behind_databases my $source = $parameter->{source} ? $parameter->{source} : ""; my $tables = $parameter->{tables} ? $parameter->{tables} : ""; - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { source => $source, tables => $tables, }}); @@ -2760,21 +2760,24 @@ sub _find_behind_databases my $check_tables = []; foreach my $table (@{$an->data->{sys}{database}{core_tables}}) { - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { table => $table }}); + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { table => $table }}); push @{$check_tables}, $table; } if (ref($tables) eq "ARRAY") { foreach my $table (@{$tables}) { - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { table => $table }}); + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { table => $table }}); push @{$check_tables}, $table; + + # This will store the most recent time stamp. + $an->data->{sys}{database}{table}{$table}{last_updated} = 0; + } } # Look at all the databases and find the most recent time stamp (and the ID of the DB). - $an->data->{sys}{database}{source_db_id} = 0; - $an->data->{sys}{database}{source_updated_time} = 0; + my $source_updated_time = 0; foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}}) { $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { @@ -2785,74 +2788,34 @@ sub _find_behind_databases "database::${id}::password" => $an->Log->secure ? $an->data->{database}{$id}{password} : "--", }}); - my $name = $an->data->{database}{$id}{name}; - my $user = $an->data->{database}{$id}{user}; - - # Read the table's last modified_date - my $query = " -SELECT - round(extract(epoch from modified_date)) -FROM - updated -WHERE - updated_host_uuid = ".$an->data->{sys}{use_db_fh}->quote($an->data->{sys}{host_uuid})." -AND - updated_by = ".$an->data->{sys}{use_db_fh}->quote($source).";"; - - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - id => $id, - query => $query, - }}); - my $last_updated = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; - $last_updated = 0 if not defined $last_updated; - - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - last_updated => $last_updated, - "sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time}, - }}); - if ($last_updated > $an->data->{sys}{database}{source_updated_time}) - { - $an->data->{sys}{database}{source_updated_time} = $last_updated; - $an->data->{sys}{database}{source_db_id} = $id; - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "sys::database::source_db_id" => $an->data->{sys}{database}{source_db_id}, - "sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time}, - }}); - } - - # Get the last updated time for this database (and source). - $an->data->{database}{$id}{last_updated} = $last_updated; - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time}, - "sys::database::source_db_id" => $an->data->{sys}{database}{source_db_id}, - "database::${id}::last_updated" => $an->data->{database}{$id}{last_updated} - }}); - + # Loop through the tables in this DB. For each table, we'll record the most recent time + # stamp. Later, We'll look through again and any table/DB with an older time stamp will be + # behind and a resync will be needed. foreach my $table (@{$check_tables}) { # Does this table exist yet? my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'public' AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table).";"; - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); my $count = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { count => $count }}); if ($count == 1) { # Does this table have a '*_host_uuid' column? my $query = "SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND column_name LIKE '\%_host_uuid' AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table).";"; - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); my $host_column = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $host_column = "" if not defined $host_column; - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_column => $host_column }}); + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_column => $host_column }}); # Does this table have a history schema version? $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'history' AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table).";"; - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); my $count = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { count => $count }}); my $schema = $count ? "history" : "public"; @@ -2871,79 +2834,56 @@ WHERE ORDER BY modified_date DESC ;"; - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { id => $id, query => $query, }}); my $last_updated = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; - $last_updated = 0 if not defined $last_updated; + $last_updated = 0 if not defined $last_updated; - $an->data->{database}{$id}{tables}{$table}{last_updated} = $last_updated; + # Record this table's last modified_date for later comparison. + $an->data->{sys}{database}{table}{$table}{id}{$id}{last_updated} = $last_updated; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "database::${id}::tables::${table}::last_updated" => $an->data->{database}{$id}{tables}{$table}{last_updated}, + "sys::database::table::${table}::id::${id}::last_updated" => $an->data->{sys}{database}{table}{$table}{id}{$id}{last_updated}, + "sys::database::table::${table}::last_updated" => $an->data->{sys}{database}{table}{$table}{last_updated}, + }}); + + if ($an->data->{sys}{database}{table}{$table}{id}{$id}{last_updated} > $an->data->{sys}{database}{table}{$table}{last_updated}) + { + $an->data->{sys}{database}{table}{$table}{last_updated} = $an->data->{sys}{database}{table}{$table}{id}{$id}{last_updated}; + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "sys::database::table::${table}::last_updated" => $an->data->{sys}{database}{table}{$table}{last_updated}, + }}); + } } } } - # Find which DB is most up to date. - $an->data->{sys}{database}{to_update} = {}; - foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}}) + # Now loop through each table we've seen and see if the moditied_date differs for any of the + # databases. If it has, trigger a resync. + foreach my $table (sort {$a cmp $b} keys %{$an->data->{sys}{database}{table}}) { $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time}, - "database::${id}::last_updated" => $an->data->{database}{$id}{last_updated}, + "sys::database::table::${table}::last_updated" => $an->data->{sys}{database}{table}{$table}{last_updated}, }}); - if ($an->data->{sys}{database}{source_updated_time} > $an->data->{database}{$id}{last_updated}) + foreach my $id (sort {$a cmp $b} keys %{$an->data->{sys}{database}{table}{$table}{id}}) { - # This database is behind - $an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0104", variables => { - id => $id, - source => $source, - }}); - - # A database is behind, resync - $an->Database->_mark_database_as_behind({id => $id}); - } - else - { - # This database is up to date (so far). - $an->data->{sys}{database}{to_update}{$id}{behind} = 0; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "sys::database::to_update::${id}::behind" => $an->data->{sys}{database}{to_update}{$id}{behind}, + "sys::database::table::${table}::id::${id}::last_updated" => $an->data->{sys}{database}{table}{$table}{id}{$id}{last_updated}, }}); - } - - # If we don't yet need a resync, and if we were passed one or more tables, check those tables - # for differences - if ((not $an->data->{sys}{database}{resync_needed}) && (ref($tables) eq "HASH")) - { - foreach my $table (sort {$a cmp $b} keys %{$tables}) + if ($an->data->{sys}{database}{table}{$table}{last_updated} > $an->data->{sys}{database}{table}{$table}{id}{$id}{last_updated}) { - if (not defined $an->data->{sys}{database}{tables}{$table}{last_updated}) - { - # First we've seen, set the general updated time to this entry - $an->data->{sys}{database}{tables}{$table}{last_updated} = $an->data->{database}{$id}{tables}{$table}{last_updated}; - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "sys::database::tables::${table}::last_updated" => $an->data->{sys}{database}{tables}{$table}{last_updated} - }}); - } - - if ($an->data->{sys}{database}{tables}{$table}{last_updated} > $an->data->{database}{$id}{tables}{$table}{last_updated}) - { - # This database is behind - $an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0106", variables => { - id => $id, - source => $source, - table => $table, - }}); - } + # Resync needed. + $an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0106", variables => { id => $id }}); # Mark it as behind. $an->Database->_mark_database_as_behind({id => $id}); + last; } } + last if $an->data->{sys}{database}{resync_needed}; } return(0); diff --git a/AN/an-tools.xml b/AN/an-tools.xml index f089b9cf..c3dad266 100644 --- a/AN/an-tools.xml +++ b/AN/an-tools.xml @@ -174,7 +174,7 @@ The database connection error was: The local machine's UUID was not read properly. It should be stored in: [#!data!sys::host_uuid!#] and contain hexadecimal characters in the format: '012345-6789-abcd-ef01-23456789abcd' and usually matches the output of 'dmidecode --string system-uuid'. If this file exists and if there is a string in the file, please verify that it is structured correctly. The database with ID: [#!variable!id!#] for: [#!variable!file!#] is behind. ScanCore database: [#!variable!database!#] already exists. - The database with ID: [#!variable!id!#] for: [#!variable!file!#] and table: [#!variable!table!#] is behind. + The database with ID: [#!variable!id!#] is behind. A database esync will be requested. [ Warning ] - Failed to delete the temporary postgres password. insert_or_update_states() was called but the 'state_host_uuid' parameter was not passed or it is empty. Normally this is set to 'sys::data_uuid'.]]> [ Error ] - Failed to create the ScanCore database: [#!variable!database!#]