seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #209 from ClusterLabs/anvil-tools-dev

Browse Source 
Anvil tools dev
main
Digimer 3 years ago committed by GitHub
parent c7a1dbdebd 032f20a267
commit 861d565cce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 406 additions and 17 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 5
      Anvil/Tools/Alert.pm
  2. 26
      Anvil/Tools/Database.pm
  3. 8
      scancore-agents/scan-network/scan-network
  4. 3
      share/words.xml
  5. 381
      tools/striker-db-report

5
Anvil/Tools/Alert.pm
Unescape View File

@ -295,6 +295,7 @@ sub check_condition_age
# See if this variable has been set yet. # See if this variable has been set yet.
my ($variable_value, $variable_uuid, $epoch_modified_date, $modified_date) = $anvil->Database->read_variable({ my ($variable_value, $variable_uuid, $epoch_modified_date, $modified_date) = $anvil->Database->read_variable({
debug => $debug,
variable_name => $name, variable_name => $name,
variable_source_table => $source_table, variable_source_table => $source_table,
variable_source_uuid => $host_uuid, variable_source_uuid => $host_uuid,
@ -319,8 +320,8 @@ sub check_condition_age
}); });
} }
# if the value was 'clear', change it to 'set'. # if the 'clear' parameter isn't set, and the value is 'clear', change it to 'set'.
if ($variable_value eq "clear") if (($variable_value eq "clear") && (not $clear))
{ {
# Set it. # Set it.
$variable_uuid = $anvil->Database->insert_or_update_variables({ $variable_uuid = $anvil->Database->insert_or_update_variables({

26
Anvil/Tools/Database.pm
Unescape View File

@ -16677,10 +16677,11 @@ sub _age_out_data
# We don't use 'anvil->data' to prevent injecting SQL queries in anvil.conf # We don't use 'anvil->data' to prevent injecting SQL queries in anvil.conf
my $to_clean = {}; my $to_clean = {};
# Power, temperatures and ip addresses # Power, temperatures, ip addresses and variables
$to_clean->{table}{temperature}{child_table}{temperature}{uuid_column} = "temperature_uuid"; $to_clean->{table}{temperature}{child_table}{temperature}{uuid_column} = "temperature_uuid";
$to_clean->{table}{power}{child_table}{power}{uuid_column} = "power_uuid"; $to_clean->{table}{power}{child_table}{power}{uuid_column} = "power_uuid";
$to_clean->{table}{ip_addresses}{child_table}{ip_addresses}{uuid_column} = "ip_address_uuid"; $to_clean->{table}{ip_addresses}{child_table}{ip_addresses}{uuid_column} = "ip_address_uuid";
$to_clean->{table}{variables}{child_table}{variables}{uuid_column} = "variable_uuid";
# scan_apc_pdu # scan_apc_pdu
$to_clean->{table}{scan_apc_pdus}{child_table}{scan_apc_pdu_phases}{uuid_column} = "scan_apc_pdu_phase_uuid"; $to_clean->{table}{scan_apc_pdus}{child_table}{scan_apc_pdu_phases}{uuid_column} = "scan_apc_pdu_phase_uuid";
@ -16760,7 +16761,7 @@ sub _age_out_data
count => $count, count => $count,
}}); }});
if ($count) if ($count > 1)
{ {
# Find how many records will be left. If it's 0, we'll use an OFFSET 1. # Find how many records will be left. If it's 0, we'll use an OFFSET 1.
my $query = "SELECT history_id FROM history.".$child_table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND modified_date > '".$old_timestamp."';"; my $query = "SELECT history_id FROM history.".$child_table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND modified_date > '".$old_timestamp."';";
@ -16781,16 +16782,17 @@ sub _age_out_data
} }
else else
{ {
# This would delete everything, reserve at least one record. # This would delete everything, reserve at
foreach my $row (@{$results}) # least one record.
{ my $query = "SELECT history_id FROM history.".$child_table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." ORDER BY modified_date DESC LIMIT 1;";
my $history_id = $row->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { history_id => $history_id }});
my $history_id = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
my $query = "DELETE FROM history.".$child_table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND history_id = '".$history_id."';"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
push @{$queries}, $query; $query = "DELETE FROM history.".$child_table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND modified_date <= '".$old_timestamp."' AND history_id != '".$history_id."';";
} $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
push @{$queries}, $query;
} }
} }
} }

8
scancore-agents/scan-network/scan-network
Unescape View File

@ -3457,9 +3457,11 @@ AND
# Don't set / clear interfaces that appear down but aren't named ifn/bcn/sn as they're probably # Don't set / clear interfaces that appear down but aren't named ifn/bcn/sn as they're probably
# unconfigured/unusued interfaces. # unconfigured/unusued interfaces.
my $problem = 0; my $problem = 0;
my $check = 0; my $check = 0;
if ($anvil->Network->is_our_interface({interface => $network_interface_name})) my $monitored = $anvil->Network->is_our_interface({interface => $network_interface_name});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { monitored => $monitored }});
if ($monitored)
{ {
# One we monitor # One we monitor
$check = 1; $check = 1;

3
share/words.xml
Unescape View File

@ -2435,6 +2435,8 @@ Are you sure that you want to delete the server: [#!variable!server_name!#]? [Ty
<key name="message_0265">Failed to get server VM screenshot; got non-zero return code.</key> <key name="message_0265">Failed to get server VM screenshot; got non-zero return code.</key>
<key name="message_0266">Finished attempting to get server VM screenshot; no operations happened because requirements not met.</key>>>> master <key name="message_0266">Finished attempting to get server VM screenshot; no operations happened because requirements not met.</key>>>> master
<key name="message_0267">Preparing to manage DR for a server.</key> <key name="message_0267">Preparing to manage DR for a server.</key>
<key name="message_0268">UUID Column counts for: [history.#!variable!table!#]:</key>
<key name="message_0269">Counting entries for each unique: [#!variable!column!#] in the table [#!variable!table!#]. Please be patient.</key>
<!-- Success messages shown to the user --> <!-- Success messages shown to the user -->
<key name="ok_0001">Saved the mail server information successfully!</key> <key name="ok_0001">Saved the mail server information successfully!</key>
@ -3098,6 +3100,7 @@ We will sleep a bit and try again.
<key name="warning_0130">[ Warning ] - The storage group: [#!variable!storage_group_name!#] had the host: [#!variable!host_name!#] as a member. This host is not a member (anymore?) of the Anvil!: [#!variable!anvil_name!#]. Removing it from the storage group now.</key> <key name="warning_0130">[ Warning ] - The storage group: [#!variable!storage_group_name!#] had the host: [#!variable!host_name!#] as a member. This host is not a member (anymore?) of the Anvil!: [#!variable!anvil_name!#]. Removing it from the storage group now.</key>
<key name="warning_0131">[ Warning ] - The postgresql server is not installed yet. Sleeping for a bit, then will check again.</key> <key name="warning_0131">[ Warning ] - The postgresql server is not installed yet. Sleeping for a bit, then will check again.</key>
<key name="warning_0132">[ Warning ] - Failed to build or install the DRBD kernel module! It is very unlikely that this machine will be able to run any servers until this is fixed.</key> <key name="warning_0132">[ Warning ] - Failed to build or install the DRBD kernel module! It is very unlikely that this machine will be able to run any servers until this is fixed.</key>
<key name="warning_0133">[ Warning ] - Table: [history.#!variable!table!#] not found.</key>
<!-- The entries below here are not sequential, but use a key to find the entry. --> <!-- The entries below here are not sequential, but use a key to find the entry. -->
<!-- Run 'striker-parse-os-list to find new entries. --> <!-- Run 'striker-parse-os-list to find new entries. -->

381
tools/striker-db-report
Unescape View File

@ -0,0 +1,381 @@
#!/usr/bin/perl
#
# This tool looks at the database and counts how many records are in each database. Optionally, if given a
# table name, it will count the number of entries exist in the history schema for each record in the public
# schema. The goal being to help quickly identifying rapidly growing tables.
#
use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
use Text::Diff;
$| = 1;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
my $anvil = Anvil::Tools->new();
$anvil->Database->connect({debug => 3, check_for_resync => 0});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003"});
$anvil->nice_exit({exit_code => 1});
}
# When set, records are counted in the public table, the the number of history entries for each columng is
# shown, sorted by frequency.
$anvil->data->{switches}{table} = "";
# When set, tables with less than the minium are ignored.
$anvil->data->{switches}{minimum} = 0;
$anvil->Get->switches();
$anvil->data->{switches}{minimum} =~ s/,//g;
if ($anvil->data->{switches}{table})
{
count_table($anvil);
}
else
{
count_all($anvil);
}
$anvil->nice_exit({exit_code => 0});
#############################################################################################################
# Functions #
#############################################################################################################
sub count_table
{
my ($anvil) = @_;
# Make sure the table exists.
my $table = $anvil->Database->quote($anvil->data->{switches}{table});
$table =~ s/^\s+//;
$table =~ s/\s.*//;
$table =~ s/^'(.*)'$/$1/;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { table => $table }});
my $query = "
SELECT
COUNT(*)
FROM
information_schema.tables
WHERE
table_schema = 'history'
AND
table_name = '".$table."'
AND
table_catalog = 'anvil'
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
if (not $count)
{
# Table doesn't exist.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "warning_0133", variables => { table => $table }});
$anvil->nice_exit({exit_code => 1});
}
my $uuid_width = 0;
my $count_width = 0;
my $column1 = $table."_uuid";
my $column2 = "";
my $column3 = "";
my $column4 = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column1 => $column1 }});
if ($table =~ /^(.*)s$/)
{
$column2 = $1."_uuid";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column2 => $column2 }});
}
if ($table =~ /^(.*)es$/)
{
$column3 = $1."_uuid";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column3 => $column3 }});
}
if ($table =~ /^(.*)ies$/)
{
$column4 = $1."y_uuid";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column4 => $column4 }});
}
$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND column_name = ".$anvil->Database->quote($column1).";";
if ($column4)
{
$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2)." OR column_name = ".$anvil->Database->quote($column3)." OR column_name = ".$anvil->Database->quote($column4).");";
}
elsif ($column3)
{
$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2)." OR column_name = ".$anvil->Database->quote($column3).");";
}
elsif ($column2)
{
$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2).");";
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $uuid_column = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$uuid_column = "" if not defined $uuid_column;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid_column => $uuid_column }});
if (not $uuid_column)
{
# This is a problem
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "err", key => "error_0311", variables => { table => $table }});
$anvil->nice_exit({exit_code => 1});
}
# This can take a while, ask the user to be patient.
print $anvil->Words->string({key => "message_0269", variables => {
table => $table,
column => $uuid_column,
}})."\n";
# Count how many entries exist for each UUID.
$query = "
SELECT
DISTINCT ".$uuid_column."
FROM
history.".$table."
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $column_uuid = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column_uuid => $column_uuid }});
if (length($column_uuid) > $uuid_width)
{
$uuid_width = length($column_uuid);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid_width => $uuid_width }});
}
my $query = "
SELECT
COUNT(*)
FROM
history.".$table."
WHERE
".$uuid_column." = ".$anvil->Database->quote($column_uuid)."
;";
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
my $comma_count = $anvil->Convert->add_commas({number => $count});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
count => $count,
comma_count => $comma_count,
}});
$anvil->data->{db_counts}{count}{$count}{$column_uuid} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"db_counts::count::${count}::${column_uuid}" => $anvil->data->{db_counts}{count}{$count}{$column_uuid},
}});
if (length($comma_count) > $count_width)
{
$count_width = length($comma_count);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count_width => $count_width }});
}
print ".";
}
print "\n";
my $queries = [];
my $divider = "-";
for (1..$uuid_width) { $divider .= "-"; }
$divider .= "-+-";
for (1..$count_width) { $divider .= "-"; }
$divider .= "-";
print $anvil->Words->string({key => "message_0268", variables => { table => $table }})."\n";
print $divider."\n";
foreach my $count (sort {$a <=> $b} keys %{$anvil->data->{db_counts}{count}})
{
my $comma_count = $anvil->Convert->add_commas({number => $count});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
count => $count,
comma_count => $comma_count,
}});
if (($anvil->data->{switches}{minimum}) && ($anvil->data->{switches}{minimum} =~ /^\d+$/) && ($count < $anvil->data->{switches}{minimum}))
{
# Skip it.
next;
}
# Sorting by UUID doesn't really make sense, but it provides consistency run over run.
foreach my $column_uuid (sort {$a cmp $b} keys %{$anvil->data->{db_counts}{count}{$count}})
{
print " ".sprintf("%${uuid_width}s", $column_uuid)." | ".sprintf("%${count_width}s", $comma_count)." \n";
# This will need to be updated by the person debugging a table.
#push @{$queries}, "SELECT variable_name, variable_value, variable_source_table, variable_source_uuid FROM variables WHERE variable_uuid = '".$column_uuid."';";
}
}
print $divider."\n";
# Enable this if you're trying to figure out what data is growing, it needs to be edited on a
# per-table basis.
if (0)
{
foreach my $query (@{$queries})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $variable_name = $row->[0];
my $variable_value = $row->[1];
my $source_table = $row->[2];
my $source_uuid = $row->[3];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:variable_name' => $variable_name,
's2:variable_value' => $variable_value,
's3:source_table' => $source_table,
's4:source_uuid' => $source_uuid,
}});
if ($source_table eq "hosts")
{
my $host_name = $anvil->Get->host_name_from_uuid({host_uuid => $source_uuid});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }});
}
}
}
}
return(0);
}
sub count_all
{
my ($anvil) = @_;
my $longest_table = 0;
my $longest_public = 0;
my $longest_history = 0;
my $query = "
SELECT
table_schema,
table_name
FROM
information_schema.tables
WHERE
(table_schema = 'public' OR table_schema = 'history')
AND
table_catalog = 'anvil'
ORDER BY
table_name ASC,
table_schema DESC;
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $table_schema = $row->[0];
my $table_name = $row->[1];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
table_schema => $table_schema,
table_name => $table_name,
}});
if (not exists $anvil->data->{db_counts}{table}{$table_name})
{
$anvil->data->{db_counts}{table}{$table_name}{public} = 0;
$anvil->data->{db_counts}{table}{$table_name}{history} = -1;
}
if (length($table_name) > $longest_table)
{
$longest_table = length($table_name);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_table => $longest_table }});
}
my $query = "SELECT COUNT(*) FROM ".$table_schema.".".$table_name.";";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
my $comma_count = $anvil->Convert->add_commas({number => $count});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
count => $count,
comma_count => $comma_count,
}});
if ($table_schema eq "public")
{
if (length($comma_count) > $longest_public)
{
$longest_public = length($comma_count);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_public => $longest_public }});
}
}
else
{
if (length($comma_count) > $longest_history)
{
$longest_history = length($comma_count);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_history => $longest_history }});
}
}
$anvil->data->{db_counts}{table}{$table_name}{$table_schema} = $count;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"db_counts::table::${table_name}::${table_schema}" => $anvil->data->{db_counts}{table}{$table_name}{$table_schema},
}});
}
my $say_table = $anvil->Words->string({key => "header_0062"});
my $say_public = $anvil->Words->string({key => "header_0063"});
my $say_history = $anvil->Words->string({key => "header_0064"});
my $divider = "-";
for (1..$longest_table) { $divider .= "-"; }
$divider .= "-+-";
for (1..$longest_public) { $divider .= "-"; }
$divider .= "-+-";
for (1..$longest_history) { $divider .= "-"; }
$divider .= "-";
print " ".sprintf("%${longest_table}s", "Table")." | ".sprintf("%${longest_public}s", $say_public)." | ".sprintf("%${longest_history}s", $say_history)." \n";
print $divider."\n";
foreach my $table_name (sort {$a cmp $b} keys %{$anvil->data->{db_counts}{table}})
{
if (($anvil->data->{switches}{minimum}) && ($anvil->data->{switches}{minimum} =~ /^\d+$/))
{
if (($anvil->data->{db_counts}{table}{$table_name}{public} < $anvil->data->{switches}{minimum}) &&
($anvil->data->{db_counts}{table}{$table_name}{history} < $anvil->data->{switches}{minimum}))
{
# Skip it.
next;
}
}
my $public = $anvil->Convert->add_commas({number => $anvil->data->{db_counts}{table}{$table_name}{public}});
my $history = $anvil->data->{db_counts}{table}{$table_name}{history} == -1 ? "--" : $anvil->Convert->add_commas({number => $anvil->data->{db_counts}{table}{$table_name}{history}});
print " ".sprintf("%${longest_table}s", $table_name)." | ".sprintf("%${longest_public}s", $public)." | ".sprintf("%${longest_history}s", $history)." \n";
}
print $divider."\n";
return(0);
}
Write Preview
Loading…
Cancel
Save