seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #228 from ClusterLabs/anvil-tools-dev

Browse Source 
Reworked Database->_find_behind_databases to loop through tables, the…
main
digimer-bot 3 years ago committed by GitHub
parent 875aa4fea3 0b41029db2
commit 1036235585
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 133 additions and 141 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 101
      Anvil/Tools/Database.pm
  2. 3
      tools/anvil-daemon

101
Anvil/Tools/Database.pm
Unescape View File

@ -17697,23 +17697,10 @@ sub _find_behind_databases
}}); }});
} }
# Look at all the databases and find the most recent time stamp (and the ID of the DB). # Look at all the databases and find the most recent time stamp (and the ID of the DB). Do this by
# table then by database to keep the counts close together and reduce the chance of tables changing
# between counts.
my $source_updated_time = 0; my $source_updated_time = 0;
foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
{
my $database_name = defined $anvil->data->{database}{$uuid}{name} ? $anvil->data->{database}{$uuid}{name} : "#!string!log_0185!#";
my $database_user = defined $anvil->data->{database}{$uuid}{user} ? $anvil->data->{database}{$uuid}{user} : "#!string!log_0185!#";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"database::${uuid}::host" => $anvil->data->{database}{$uuid}{host},
"database::${uuid}::port" => $anvil->data->{database}{$uuid}{port},
"database::${uuid}::name" => $database_name,
"database::${uuid}::user" => $database_user,
"database::${uuid}::password" => $anvil->Log->is_secure($anvil->data->{database}{$uuid}{password}),
}});
# Loop through the tables in this DB. For each table, we'll record the most recent time
# stamp. Later, We'll look through again and any table/DB with an older time stamp will be
# behind and a resync will be needed.
foreach my $table (@{$anvil->data->{sys}{database}{check_tables}}) foreach my $table (@{$anvil->data->{sys}{database}{check_tables}})
{ {
# We don't sync 'states' or 'oui' as it's transient and sometimes per-DB. # We don't sync 'states' or 'oui' as it's transient and sometimes per-DB.
@ -17724,14 +17711,15 @@ sub _find_behind_databases
my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'public' AND table_name = ".$anvil->Database->quote($table).";"; my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'public' AND table_name = ".$anvil->Database->quote($table).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; # If not, skip. It'll get sync'ed later when the table is added.
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
next if not $count;
if ($count == 1) ### Get the host_uuid column (we can get this from whatever DB we're reading from.
{ # Some tables, like 'servers', has a host_uuid column, but it's not used to restrict data to
# Some tables, like 'servers', has a host_uuid column, but it's not used to # a host, but instead show which host a movable resource is on. This prevents us from using
# restrict data to a host, but instead show which host a movable resource is # the column by accident.
# on. This prevents us from using the column by accident.
my $host_column = ""; my $host_column = "";
if (($table ne "servers") && if (($table ne "servers") &&
($table ne "jobs")) ($table ne "jobs"))
@ -17755,7 +17743,7 @@ sub _find_behind_databases
# See if there is a column that ends in '_host_uuid'. If there is, we'll use # See if there is a column that ends in '_host_uuid'. If there is, we'll use
# it later to restrict resync activity to these columns with the local # it later to restrict resync activity to these columns with the local
# 'sys::host_uuid'. # 'sys::host_uuid'.
my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
if ($count) if ($count)
@ -17771,10 +17759,22 @@ sub _find_behind_databases
$query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'history' AND table_name = ".$anvil->Database->quote($table).";"; $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'history' AND table_name = ".$anvil->Database->quote($table).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; my $has_history = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { has_history => $has_history }});
my $schema = $count ? "history" : "public"; foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
{
my $database_name = defined $anvil->data->{database}{$uuid}{name} ? $anvil->data->{database}{$uuid}{name} : "#!string!log_0185!#";
my $database_user = defined $anvil->data->{database}{$uuid}{user} ? $anvil->data->{database}{$uuid}{user} : "#!string!log_0185!#";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"database::${uuid}::host" => $anvil->data->{database}{$uuid}{host},
"database::${uuid}::port" => $anvil->data->{database}{$uuid}{port},
"database::${uuid}::name" => $database_name,
"database::${uuid}::user" => $database_user,
"database::${uuid}::password" => $anvil->Log->is_secure($anvil->data->{database}{$uuid}{password}),
}});
my $schema = $has_history ? "history" : "public";
$query = " $query = "
SELECT DISTINCT SELECT DISTINCT
round(extract(epoch from modified_date)) AS unix_modified_date round(extract(epoch from modified_date)) AS unix_modified_date
@ -17837,14 +17837,9 @@ ORDER BY
}}); }});
} }
} }
else
{
### NOTE: We could recover a lost table here if we tried to find the table in
### a .sql file and load it. Might be worth adding later.
}
}
} }
# Are being asked to trigger a resync? # Are being asked to trigger a resync?
foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}}) foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
{ {
@ -17884,27 +17879,27 @@ ORDER BY
"sys::database::table::${table}::uuid::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated}, "sys::database::table::${table}::uuid::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated},
"sys::database::table::${table}::uuid::${uuid}::row_count" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count}, "sys::database::table::${table}::uuid::${uuid}::row_count" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count},
}}); }});
# if ($anvil->data->{sys}{database}{table}{$table}{last_updated} > $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated}) if ($anvil->data->{sys}{database}{table}{$table}{last_updated} > $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated})
# { {
# # Resync needed. # Resync needed.
# my $difference = $anvil->data->{sys}{database}{table}{$table}{last_updated} - $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated}; my $difference = $anvil->data->{sys}{database}{table}{$table}{last_updated} - $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated};
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
# "s1:difference" => $anvil->Convert->add_commas({number => $difference }), "s1:difference" => $anvil->Convert->add_commas({number => $difference }),
# "s2:sys::database::table::${table}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{last_updated}, "s2:sys::database::table::${table}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{last_updated},
# "s3:sys::database::table::${table}::uuid::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated}, "s3:sys::database::table::${table}::uuid::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated},
# }}); }});
#
# $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0106", variables => { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0106", variables => {
# seconds => $difference, seconds => $difference,
# table => $table, table => $table,
# uuid => $uuid, uuid => $uuid,
# host => $anvil->Get->host_name_from_uuid({host_uuid => $uuid}), host => $anvil->Get->host_name_from_uuid({host_uuid => $uuid}),
# }}); }});
#
# # Mark it as behind. # Mark it as behind.
# $anvil->Database->_mark_database_as_behind({debug => $debug, uuid => $uuid}); $anvil->Database->_mark_database_as_behind({debug => $debug, uuid => $uuid});
# last; last;
# } }
if ($anvil->data->{sys}{database}{table}{$table}{row_count} > $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count}) if ($anvil->data->{sys}{database}{table}{$table}{row_count} > $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count})
{ {
# Resync needed. # Resync needed.

3
tools/anvil-daemon
Unescape View File

@ -879,9 +879,6 @@ AND
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
$anvil->Database->write({debug => 2, query => $query, source => $THIS_FILE, line => __LINE__}); $anvil->Database->write({debug => 2, query => $query, source => $THIS_FILE, line => __LINE__});
} }
### TODO: What are the chances of a PID being reused in the minute between
### the program's death and us detecting it? Should we filter the
### 'pids::<pid>::command' value against our programs and scan agents?
} }
} }

Write Preview
Loading…
Cancel
Save