seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Fixed Database->_find_behind_databases(), making it a lot simpler in the process.

Browse Source 
Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 7 years ago
parent 3ce3a80c98
commit ce607238f8
4 changed files with 48 additions and 105 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      AN/Tools.pm
  2. 4
      AN/Tools.sql
  3. 146
      AN/Tools/Database.pm
  4. 2
      AN/an-tools.xml

1
AN/Tools.pm
Unescape View File

@ -606,6 +606,7 @@ sub _set_defaults
"variables", "variables",
"alert_sent", "alert_sent",
"states", "states",
"updated",
], ],
local_lock_active => 0, local_lock_active => 0,
locking_reap_age => 300, locking_reap_age => 300,

4
AN/Tools.sql
Unescape View File

@ -253,7 +253,9 @@ CREATE TRIGGER trigger_variables
-- These are special tables with no history or tracking UUIDs that simply record transient information. -- -- These are special tables with no history or tracking UUIDs that simply record transient information. --
-- ------------------------------------------------------------------------------------------------------- -- -- ------------------------------------------------------------------------------------------------------- --
-- This table records the last time a scan ran.
-- This table records the last time a scan ran. It's sole purpose is to make sure at least one table's
-- 'modified_date' changes per run, so that database resyncs can be triggered reliably.
CREATE TABLE updated ( CREATE TABLE updated (
updated_host_uuid uuid not null, updated_host_uuid uuid not null,
updated_by text not null, -- The name of the agent (or "ScanCore' itself) that updated. updated_by text not null, -- The name of the agent (or "ScanCore' itself) that updated.

146
AN/Tools/Database.pm
Unescape View File

@ -2742,7 +2742,7 @@ sub _find_behind_databases
my $source = $parameter->{source} ? $parameter->{source} : ""; my $source = $parameter->{source} ? $parameter->{source} : "";
my $tables = $parameter->{tables} ? $parameter->{tables} : ""; my $tables = $parameter->{tables} ? $parameter->{tables} : "";
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
source => $source, source => $source,
tables => $tables, tables => $tables,
}}); }});
@ -2760,21 +2760,24 @@ sub _find_behind_databases
my $check_tables = []; my $check_tables = [];
foreach my $table (@{$an->data->{sys}{database}{core_tables}}) foreach my $table (@{$an->data->{sys}{database}{core_tables}})
{ {
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { table => $table }}); $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { table => $table }});
push @{$check_tables}, $table; push @{$check_tables}, $table;
} }
if (ref($tables) eq "ARRAY") if (ref($tables) eq "ARRAY")
{ {
foreach my $table (@{$tables}) foreach my $table (@{$tables})
{ {
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { table => $table }}); $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { table => $table }});
push @{$check_tables}, $table; push @{$check_tables}, $table;
# This will store the most recent time stamp.
$an->data->{sys}{database}{table}{$table}{last_updated} = 0;
} }
} }
# Look at all the databases and find the most recent time stamp (and the ID of the DB). # Look at all the databases and find the most recent time stamp (and the ID of the DB).
$an->data->{sys}{database}{source_db_id} = 0; my $source_updated_time = 0;
$an->data->{sys}{database}{source_updated_time} = 0;
foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}}) foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}})
{ {
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
@ -2785,74 +2788,34 @@ sub _find_behind_databases
"database::${id}::password" => $an->Log->secure ? $an->data->{database}{$id}{password} : "--", "database::${id}::password" => $an->Log->secure ? $an->data->{database}{$id}{password} : "--",
}}); }});
my $name = $an->data->{database}{$id}{name}; # Loop through the tables in this DB. For each table, we'll record the most recent time
my $user = $an->data->{database}{$id}{user}; # stamp. Later, We'll look through again and any table/DB with an older time stamp will be
# behind and a resync will be needed.
# Read the table's last modified_date
my $query = "
SELECT
round(extract(epoch from modified_date))
FROM
updated
WHERE
updated_host_uuid = ".$an->data->{sys}{use_db_fh}->quote($an->data->{sys}{host_uuid})."
AND
updated_by = ".$an->data->{sys}{use_db_fh}->quote($source).";";
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
id => $id,
query => $query,
}});
my $last_updated = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$last_updated = 0 if not defined $last_updated;
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
last_updated => $last_updated,
"sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time},
}});
if ($last_updated > $an->data->{sys}{database}{source_updated_time})
{
$an->data->{sys}{database}{source_updated_time} = $last_updated;
$an->data->{sys}{database}{source_db_id} = $id;
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"sys::database::source_db_id" => $an->data->{sys}{database}{source_db_id},
"sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time},
}});
}
# Get the last updated time for this database (and source).
$an->data->{database}{$id}{last_updated} = $last_updated;
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time},
"sys::database::source_db_id" => $an->data->{sys}{database}{source_db_id},
"database::${id}::last_updated" => $an->data->{database}{$id}{last_updated}
}});
foreach my $table (@{$check_tables}) foreach my $table (@{$check_tables})
{ {
# Does this table exist yet? # Does this table exist yet?
my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'public' AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table).";"; my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'public' AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table).";";
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $count = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; my $count = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { count => $count }});
if ($count == 1) if ($count == 1)
{ {
# Does this table have a '*_host_uuid' column? # Does this table have a '*_host_uuid' column?
my $query = "SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND column_name LIKE '\%_host_uuid' AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table).";"; my $query = "SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND column_name LIKE '\%_host_uuid' AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table).";";
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $host_column = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; my $host_column = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$host_column = "" if not defined $host_column; $host_column = "" if not defined $host_column;
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_column => $host_column }}); $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_column => $host_column }});
# Does this table have a history schema version? # Does this table have a history schema version?
$query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'history' AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table).";"; $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'history' AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table).";";
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $count = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; my $count = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { count => $count }});
my $schema = $count ? "history" : "public"; my $schema = $count ? "history" : "public";
@ -2871,79 +2834,56 @@ WHERE
ORDER BY ORDER BY
modified_date DESC modified_date DESC
;"; ;";
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
id => $id, id => $id,
query => $query, query => $query,
}}); }});
my $last_updated = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; my $last_updated = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$last_updated = 0 if not defined $last_updated; $last_updated = 0 if not defined $last_updated;
$an->data->{database}{$id}{tables}{$table}{last_updated} = $last_updated; # Record this table's last modified_date for later comparison.
$an->data->{sys}{database}{table}{$table}{id}{$id}{last_updated} = $last_updated;
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"database::${id}::tables::${table}::last_updated" => $an->data->{database}{$id}{tables}{$table}{last_updated}, "sys::database::table::${table}::id::${id}::last_updated" => $an->data->{sys}{database}{table}{$table}{id}{$id}{last_updated},
"sys::database::table::${table}::last_updated" => $an->data->{sys}{database}{table}{$table}{last_updated},
}}); }});
if ($an->data->{sys}{database}{table}{$table}{id}{$id}{last_updated} > $an->data->{sys}{database}{table}{$table}{last_updated})
{
$an->data->{sys}{database}{table}{$table}{last_updated} = $an->data->{sys}{database}{table}{$table}{id}{$id}{last_updated};
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"sys::database::table::${table}::last_updated" => $an->data->{sys}{database}{table}{$table}{last_updated},
}});
}
} }
} }
} }
# Find which DB is most up to date. # Now loop through each table we've seen and see if the moditied_date differs for any of the
$an->data->{sys}{database}{to_update} = {}; # databases. If it has, trigger a resync.
foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}}) foreach my $table (sort {$a cmp $b} keys %{$an->data->{sys}{database}{table}})
{ {
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time}, "sys::database::table::${table}::last_updated" => $an->data->{sys}{database}{table}{$table}{last_updated},
"database::${id}::last_updated" => $an->data->{database}{$id}{last_updated},
}}); }});
if ($an->data->{sys}{database}{source_updated_time} > $an->data->{database}{$id}{last_updated}) foreach my $id (sort {$a cmp $b} keys %{$an->data->{sys}{database}{table}{$table}{id}})
{ {
# This database is behind
$an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0104", variables => {
id => $id,
source => $source,
}});
# A database is behind, resync
$an->Database->_mark_database_as_behind({id => $id});
}
else
{
# This database is up to date (so far).
$an->data->{sys}{database}{to_update}{$id}{behind} = 0;
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"sys::database::to_update::${id}::behind" => $an->data->{sys}{database}{to_update}{$id}{behind}, "sys::database::table::${table}::id::${id}::last_updated" => $an->data->{sys}{database}{table}{$table}{id}{$id}{last_updated},
}}); }});
} if ($an->data->{sys}{database}{table}{$table}{last_updated} > $an->data->{sys}{database}{table}{$table}{id}{$id}{last_updated})
# If we don't yet need a resync, and if we were passed one or more tables, check those tables
# for differences
if ((not $an->data->{sys}{database}{resync_needed}) && (ref($tables) eq "HASH"))
{
foreach my $table (sort {$a cmp $b} keys %{$tables})
{ {
if (not defined $an->data->{sys}{database}{tables}{$table}{last_updated}) # Resync needed.
{ $an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0106", variables => { id => $id }});
# First we've seen, set the general updated time to this entry
$an->data->{sys}{database}{tables}{$table}{last_updated} = $an->data->{database}{$id}{tables}{$table}{last_updated};
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"sys::database::tables::${table}::last_updated" => $an->data->{sys}{database}{tables}{$table}{last_updated}
}});
}
if ($an->data->{sys}{database}{tables}{$table}{last_updated} > $an->data->{database}{$id}{tables}{$table}{last_updated})
{
# This database is behind
$an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0106", variables => {
id => $id,
source => $source,
table => $table,
}});
}
# Mark it as behind. # Mark it as behind.
$an->Database->_mark_database_as_behind({id => $id}); $an->Database->_mark_database_as_behind({id => $id});
last;
} }
} }
last if $an->data->{sys}{database}{resync_needed};
} }
return(0); return(0);

2
AN/an-tools.xml
Unescape View File

@ -174,7 +174,7 @@ The database connection error was:
<key name="log_0103">The local machine's UUID was not read properly. It should be stored in: [#!data!sys::host_uuid!#] and contain hexadecimal characters in the format: '012345-6789-abcd-ef01-23456789abcd' and usually matches the output of 'dmidecode --string system-uuid'. If this file exists and if there is a string in the file, please verify that it is structured correctly.</key> <key name="log_0103">The local machine's UUID was not read properly. It should be stored in: [#!data!sys::host_uuid!#] and contain hexadecimal characters in the format: '012345-6789-abcd-ef01-23456789abcd' and usually matches the output of 'dmidecode --string system-uuid'. If this file exists and if there is a string in the file, please verify that it is structured correctly.</key>
<key name="log_0104">The database with ID: [#!variable!id!#] for: [#!variable!file!#] is behind.</key> <key name="log_0104">The database with ID: [#!variable!id!#] for: [#!variable!file!#] is behind.</key>
<key name="log_0105">ScanCore database: [#!variable!database!#] already exists.</key> <key name="log_0105">ScanCore database: [#!variable!database!#] already exists.</key>
<key name="log_0106">The database with ID: [#!variable!id!#] for: [#!variable!file!#] and table: [#!variable!table!#] is behind.</key> <key name="log_0106">The database with ID: [#!variable!id!#] is behind. A database esync will be requested.</key>
<key name="log_0107">[ Warning ] - Failed to delete the temporary postgres password.</key> <key name="log_0107">[ Warning ] - Failed to delete the temporary postgres password.</key>
<key name="log_0108"><![CDATA[[ Error ] - The method Database->insert_or_update_states() was called but the 'state_host_uuid' parameter was not passed or it is empty. Normally this is set to 'sys::data_uuid'.]]></key> <key name="log_0108"><![CDATA[[ Error ] - The method Database->insert_or_update_states() was called but the 'state_host_uuid' parameter was not passed or it is empty. Normally this is set to 'sys::data_uuid'.]]></key>
<key name="log_0109">[ Error ] - Failed to create the ScanCore database: [#!variable!database!#]</key> <key name="log_0109">[ Error ] - Failed to create the ScanCore database: [#!variable!database!#]</key>

Write Preview
Loading…
Cancel
Save