seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #102 from ClusterLabs/anvil-tools-dev

Browse Source 
The theme of this commit is database cleanup and fixes.
main
digimer-bot 4 years ago committed by GitHub
parent b862ff4895 8807915bb7
commit 6de1824fe7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 152 additions and 68 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 113
      Anvil/Tools/Database.pm
  2. 2
      Anvil/Tools/Job.pm
  3. 2
      Anvil/Tools/System.pm
  4. 6
      anvil.conf
  5. 3
      share/words.xml
  6. 3
      tools/anvil-daemon
  7. 2
      tools/striker-get-peer-data
  8. 2
      tools/striker-initialize-host
  9. 2
      tools/striker-manage-install-target
  10. 5
      tools/striker-manage-peers
  11. 2
      tools/striker-purge-target
  12. 2
      tools/striker-scan-network

113
Anvil/Tools/Database.pm
Unescape View File

@ -15522,7 +15522,9 @@ sub write
=head2 _age_out_data =head2 _age_out_data
This deletes any data considered transient (power, thermal, etc) after C<< scancore::database::age_out >> hours old. This deletes any data considered transient (power, thermal, etc) after C<< scancore::database::age_out >> hours old. The exception are completed jobs that are more than 2 hours old, which are purged.
B<< Note >>: Scan agents can have fast-growing tabled purged as well. This is done by setting the appropriate values in the C<< $to_clean >> hash contained within. This is hard coded so the source needs to be updated as the number of agents grow.
=cut =cut
sub _age_out_data sub _age_out_data
@ -15533,6 +15535,10 @@ sub _age_out_data
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->_age_out_data()" }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->_age_out_data()" }});
# Log our start, as this takes some time to run.
my $start_time = time;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0623"});
# Get the timestamp to delete jobs and processed alert records older than 2h # Get the timestamp to delete jobs and processed alert records older than 2h
my $query = "SELECT now() - '2h'::interval"; my $query = "SELECT now() - '2h'::interval";
my $old_timestamp = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; my $old_timestamp = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
@ -15717,25 +15723,68 @@ sub _age_out_data
} }
### Looks for scan agent data that grows quickly. ### Looks for scan agent data that grows quickly.
# scan-ipmitool # We don't use 'anvil->data' to prevent injecting SQL queries in anvil.conf
$query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename='scan_ipmitool_values' AND schemaname='public';"; my $to_clean = {};
# scan_apc_pdu
$to_clean->{table}{scan_apc_pdus}{child_table}{scan_apc_pdu_phases}{uuid_column} = "scan_apc_pdu_phase_uuid";
$to_clean->{table}{scan_apc_pdus}{child_table}{scan_apc_pdu_variables}{uuid_column} = "scan_apc_pdu_variable_uuid";
# scan_apc_ups
$to_clean->{table}{scan_apc_upses}{child_table}{scan_apc_ups_batteries}{uuid_column} = "scan_apc_ups_battery_uuid";
$to_clean->{table}{scan_apc_upses}{child_table}{scan_apc_ups_input}{uuid_column} = "scan_apc_ups_input_uuid";
$to_clean->{table}{scan_apc_upses}{child_table}{scan_apc_ups_output}{uuid_column} = "scan_apc_ups_output_uuid";
# scan_filesystems
$to_clean->{table}{scan_filesystems}{child_table}{scan_filesystems}{uuid_column} = "scan_filesystem_uuid";
# scan_hardware
$to_clean->{table}{scan_hardware}{child_table}{scan_hardware}{uuid_column} = "scan_hardware_uuid";
$to_clean->{table}{scan_hardware}{child_table}{scan_hardware}{uuid_column} = "scan_hardware_uuid";
# scan_hpacucli
$to_clean->{table}{scan_hpacucli_variables}{child_table}{scan_hpacucli_variables}{uuid_column} = "scan_hpacucli_variable_uuid";
# scan_ipmitool
$to_clean->{table}{scan_ipmitool}{child_table}{scan_ipmitool_values}{uuid_column} = "scan_ipmitool_value_uuid";
# scan_storcli
$to_clean->{table}{scan_storcli_variables}{child_table}{scan_storcli_variables}{uuid_column} = "scan_storcli_variable_uuid";
my $vacuum = 0;
foreach my $table (sort {$a cmp $b} keys %{$to_clean->{table}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { table => $table }});
# Does the table exist?
$query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename='scan_apc_pdus' AND schemaname='public';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
if ($count) if ($count)
{ {
# The table exists, clean up child tables.
foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}}) foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
{ {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid => $uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
foreach my $child_table (sort {$a cmp $b} keys %{$to_clean->{table}{$table}{child_table}})
{
my $uuid_column = $to_clean->{table}{$table}{child_table}{$child_table}{uuid_column};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
child_table => $child_table,
uuid_column => $uuid_column,
}});
# Get a list of all records.
my $queries = []; my $queries = [];
my $query = "SELECT scan_ipmitool_value_uuid FROM scan_ipmitool_values;"; my $query = "SELECT ".$uuid_column." FROM ".$child_table.";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results}; my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results, results => $results,
count => $count, count => $count,
}}); }});
@ -15744,21 +15793,34 @@ sub _age_out_data
my $column_uuid = $row->[0]; my $column_uuid = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { column_uuid => $column_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { column_uuid => $column_uuid }});
# Find out of there are any records to remove at all.
my $query = "SELECT history_id FROM history.".$child_table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND modified_date <= '".$old_timestamp."';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
if ($count)
{
# Find how many records will be left. If it's 0, we'll use an OFFSET 1. # Find how many records will be left. If it's 0, we'll use an OFFSET 1.
my $query = "SELECT history_id FROM history.scan_ipmitool_values WHERE scan_ipmitool_value_uuid = ".$anvil->Database->quote($column_uuid)." AND modified_date > '".$old_timestamp."';"; my $query = "SELECT history_id FROM history.".$child_table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND modified_date > '".$old_timestamp."';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results}; my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results, results => $results,
count => $count, count => $count,
}}); }});
if ($count) if ($count)
{ {
# At least one record will be left. # At least one record will be left, we can do a simple delete.
my $query = "DELETE FROM history.scan_ipmitool_values WHERE scan_ipmitool_value_uuid = ".$anvil->Database->quote($column_uuid)." AND modified_date <= '".$old_timestamp."';"; my $query = "DELETE FROM history.".$child_table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND modified_date <= '".$old_timestamp."';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
push @{$queries}, $query; push @{$queries}, $query;
} }
else else
@ -15769,19 +15831,31 @@ sub _age_out_data
my $history_id = $row->[0]; my $history_id = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { history_id => $history_id }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { history_id => $history_id }});
my $query = "DELETE FROM history.scan_ipmitool_values WHERE scan_ipmitool_value_uuid = ".$anvil->Database->quote($column_uuid)." AND hostory_id = '".$history_id."';"; my $query = "DELETE FROM history.".$child_table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND history_id = '".$history_id."';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
push @{$queries}, $query; push @{$queries}, $query;
} }
} }
} }
}
my $commits = @{$queries}; my $commits = @{$queries};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { commits => $commits }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { commits => $commits }});
if ($commits) if ($commits)
{ {
# Commit the DELETEs. # Commit the DELETEs.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0622", variables => {
age => $age,
table => $child_table,
database => $anvil->Get->host_name_from_uuid({host_uuid => $uuid, debug => $debug}),
}});
$anvil->Database->write({debug => $debug, uuid => $uuid, query => $queries, source => $THIS_FILE, line => __LINE__}); $anvil->Database->write({debug => $debug, uuid => $uuid, query => $queries, source => $THIS_FILE, line => __LINE__});
$vacuum += $commits;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { vacuum => $vacuum }});
undef $queries;
}
}
} }
} }
} }
@ -15796,6 +15870,9 @@ sub _age_out_data
$anvil->Database->write({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); $anvil->Database->write({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
} }
my $runtime = time - $start_time;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0624", variables => { runtime => $runtime }});
return(0); return(0);
} }
@ -16133,7 +16210,7 @@ sub _find_column
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results}; my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
table => $table, table => $table,
count => $count, count => $count,
}}); }});

2
Anvil/Tools/Job.pm
Unescape View File

@ -616,7 +616,7 @@ sub update_progress
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { message => $message, picked_up_by => $picked_up_by }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { message => $message, picked_up_by => $picked_up_by }});
if ($message eq "clear") if ($message eq "clear")
{ {
$picked_up_by = 0; $picked_up_by = $$;
$clear_status = 1; $clear_status = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
picked_up_by => $picked_up_by, picked_up_by => $picked_up_by,

2
Anvil/Tools/System.pm
Unescape View File

@ -4933,7 +4933,7 @@ sub update_hosts
# Matches, we don't need to deal with this name. # Matches, we don't need to deal with this name.
delete $anvil->data->{hosts}{needed}{$name}; delete $anvil->data->{hosts}{needed}{$name};
} }
else elsif ($ip_address ne "127.0.0.1")
{ {
# The IP has changed. Skip this name (which removes it from the list). # The IP has changed. Skip this name (which removes it from the list).
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0481", variables => { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0481", variables => {

6
anvil.conf
Unescape View File

@ -76,9 +76,9 @@ feature::scancore::disable::preventative-live-migration = 0
# NOTE: If the archive directory doesn't exist, Anvil! will create it # NOTE: If the archive directory doesn't exist, Anvil! will create it
# automatically the first time it is needed. # automatically the first time it is needed.
sys::database::archive::compress = 1 sys::database::archive::compress = 1
sys::database::archive::trigger = 50000 sys::database::archive::trigger = 100000
sys::database::archive::count = 25000 sys::database::archive::count = 50000
sys::database::archive::division = 30000 sys::database::archive::division = 75000
sys::database::archive::directory = /usr/local/anvil/archives/ sys::database::archive::directory = /usr/local/anvil/archives/
# This puts a limit on how many queries (writes, generally) to make in a single batch transaction. This is # This puts a limit on how many queries (writes, generally) to make in a single batch transaction. This is

3
share/words.xml
Unescape View File

@ -1787,6 +1787,9 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
<key name="log_0619">The host: [#!variable!host_name!#] has shut down for thermal reasons: [#!variable!count!#] times. To prevent a frequent boot / thermal excursion / shutdown loop, we will wait: [#!variable!wait_for!#] before marking it's temperature as being OK again.</key> <key name="log_0619">The host: [#!variable!host_name!#] has shut down for thermal reasons: [#!variable!count!#] times. To prevent a frequent boot / thermal excursion / shutdown loop, we will wait: [#!variable!wait_for!#] before marking it's temperature as being OK again.</key>
<key name="log_0620">This host has been running for: [#!variable!uptime!#]. The cluster will not be started (uptime must be less than 10 minutes for 'anvil-safe-start' to be called automatically).</key> <key name="log_0620">This host has been running for: [#!variable!uptime!#]. The cluster will not be started (uptime must be less than 10 minutes for 'anvil-safe-start' to be called automatically).</key>
<key name="log_0621">- The Scan agent: [#!variable!agent_name!#] ran a bit long, exiting after: [#!variable!runtime!#] seconds with the return code: [#!variable!return_code!#].</key> <key name="log_0621">- The Scan agent: [#!variable!agent_name!#] ran a bit long, exiting after: [#!variable!runtime!#] seconds with the return code: [#!variable!return_code!#].</key>
<key name="log_0622">Aging out one or more records that are more than: [#!variable!age!#] hours old from the table: [#!variable!table!#] on the database host: [#!variable!database!#].</key>
<key name="log_0623">Starting the process of aging out old data. This can take about a minute, please be patient.</key>
<key name="log_0624">Aging out old data completed after: [#!variable!runtime!#] seconds.</key>
<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. --> <!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key> <key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>

3
tools/anvil-daemon
Unescape View File

@ -462,6 +462,9 @@ sub handle_periodic_tasks
### recently enough. ### recently enough.
if ($type eq "striker") if ($type eq "striker")
{ {
# Age out old data. This takes up to a minute.
$anvil->Database->_age_out_data();
# Record a job, don't call it directly. It takes too long to run. # Record a job, don't call it directly. It takes too long to run.
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
file => $THIS_FILE, file => $THIS_FILE,

2
tools/striker-get-peer-data
Unescape View File

@ -315,7 +315,7 @@ sub get_password
} }
# We'll pick up the peer's password from the database. # We'll pick up the peer's password from the database.
$anvil->Database->connect(); $anvil->Database->connect({check_for_resync => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections}) if (not $anvil->data->{sys}{database}{connections})
{ {

2
tools/striker-initialize-host
Unescape View File

@ -43,7 +43,7 @@ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list
}}); }});
# Connect to the database(s). # Connect to the database(s).
$anvil->Database->connect; $anvil->Database->connect({check_for_resync => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections}) if (not $anvil->data->{sys}{database}{connections})
{ {

2
tools/striker-manage-install-target
Unescape View File

@ -123,7 +123,7 @@ if ($anvil->data->{switches}{status})
} }
# Connect to the database(s). # Connect to the database(s).
$anvil->Database->connect; $anvil->Database->connect({check_for_resync => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections}) if (not $anvil->data->{sys}{database}{connections})
{ {

5
tools/striker-manage-peers
Unescape View File

@ -63,7 +63,7 @@ if (($< != 0) && ($> != 0))
} }
# We'll try to connect in case we're adding additional peers. # We'll try to connect in case we're adding additional peers.
$anvil->Database->connect(); $anvil->Database->connect({check_for_resync => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
# Am I adding, editing or deleting? # Am I adding, editing or deleting?
@ -543,6 +543,7 @@ sub process_entry
# Connect, and configure, if needed. # Connect, and configure, if needed.
$anvil->Database->connect({ $anvil->Database->connect({
debug => 3, debug => 3,
check_for_resync => 1,
check_if_configured => $host_uuid eq $anvil->Get->host_uuid ? 1 : 0, check_if_configured => $host_uuid eq $anvil->Get->host_uuid ? 1 : 0,
}); });
@ -623,7 +624,7 @@ sub process_entry
sleep 1; sleep 1;
$anvil->Database->connect({db_uuid => $host_uuid}); $anvil->Database->connect({check_for_resync => 1, db_uuid => $host_uuid});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
} }
} }

2
tools/striker-purge-target
Unescape View File

@ -20,7 +20,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
my $anvil = Anvil::Tools->new(); my $anvil = Anvil::Tools->new();
$anvil->Database->connect({debug => 3}); $anvil->Database->connect({check_for_resync => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections}) if (not $anvil->data->{sys}{database}{connections})
{ {

2
tools/striker-scan-network
Unescape View File

@ -40,7 +40,7 @@ if (($< != 0) && ($> != 0))
$anvil->nice_exit({exit_code => 5}); $anvil->nice_exit({exit_code => 5});
} }
$anvil->Database->connect; $anvil->Database->connect({check_for_resync => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections}) if (not $anvil->data->{sys}{database}{connections})
{ {

Write Preview
Loading…
Cancel
Save