From 0a9f81d852b756abd7fa08401a2b9397088c122d Mon Sep 17 00:00:00 2001 From: Digimer Date: Thu, 13 Jan 2022 20:00:37 -0500 Subject: [PATCH 1/3] * Created the new striker-db-report that shows how many records are in each database table, and if passed a table name, report how many times each record has been recorded in the history schema. Signed-off-by: Digimer --- share/words.xml | 3 + tools/striker-db-report | 381 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 384 insertions(+) create mode 100755 tools/striker-db-report diff --git a/share/words.xml b/share/words.xml index ad92ad88..bec84e10 100644 --- a/share/words.xml +++ b/share/words.xml @@ -2435,6 +2435,8 @@ Are you sure that you want to delete the server: [#!variable!server_name!#]? [Ty Failed to get server VM screenshot; got non-zero return code. Finished attempting to get server VM screenshot; no operations happened because requirements not met.>>> master Preparing to manage DR for a server. + UUID Column counts for: [history.#!variable!table!#]: + Counting entries for each unique: [#!variable!column!#] in the table [#!variable!table!#]. Please be patient. Saved the mail server information successfully! @@ -3098,6 +3100,7 @@ We will sleep a bit and try again. [ Warning ] - The storage group: [#!variable!storage_group_name!#] had the host: [#!variable!host_name!#] as a member. This host is not a member (anymore?) of the Anvil!: [#!variable!anvil_name!#]. Removing it from the storage group now. [ Warning ] - The postgresql server is not installed yet. Sleeping for a bit, then will check again. [ Warning ] - Failed to build or install the DRBD kernel module! It is very unlikely that this machine will be able to run any servers until this is fixed. + [ Warning ] - Table: [history.#!variable!table!#] not found. diff --git a/tools/striker-db-report b/tools/striker-db-report new file mode 100755 index 00000000..ace40df5 --- /dev/null +++ b/tools/striker-db-report @@ -0,0 +1,381 @@ +#!/usr/bin/perl +# +# This tool looks at the database and counts how many records are in each database. Optionally, if given a +# table name, it will count the number of entries exist in the history schema for each record in the public +# schema. The goal being to help quickly identifying rapidly growing tables. +# + +use strict; +use warnings; +use Anvil::Tools; +use Data::Dumper; +use Text::Diff; + +$| = 1; + +my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; +my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; +if (($running_directory =~ /^\./) && ($ENV{PWD})) +{ + $running_directory =~ s/^\./$ENV{PWD}/; +} + +my $anvil = Anvil::Tools->new(); + +$anvil->Database->connect({debug => 3, check_for_resync => 0}); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); +if (not $anvil->data->{sys}{database}{connections}) +{ + # No databases, exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003"}); + $anvil->nice_exit({exit_code => 1}); +} + +# When set, records are counted in the public table, the the number of history entries for each columng is +# shown, sorted by frequency. +$anvil->data->{switches}{table} = ""; +# When set, tables with less than the minium are ignored. +$anvil->data->{switches}{minimum} = 0; +$anvil->Get->switches(); +$anvil->data->{switches}{minimum} =~ s/,//g; + +if ($anvil->data->{switches}{table}) +{ + count_table($anvil); +} +else +{ + count_all($anvil); +} + +$anvil->nice_exit({exit_code => 0}); + + + +############################################################################################################# +# Functions # +############################################################################################################# + +sub count_table +{ + my ($anvil) = @_; + + # Make sure the table exists. + my $table = $anvil->Database->quote($anvil->data->{switches}{table}); + $table =~ s/^\s+//; + $table =~ s/\s.*//; + $table =~ s/^'(.*)'$/$1/; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { table => $table }}); + my $query = " +SELECT + COUNT(*) +FROM + information_schema.tables +WHERE + table_schema = 'history' +AND + table_name = '".$table."' +AND + table_catalog = 'anvil' +;"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); + my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); + if (not $count) + { + # Table doesn't exist. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "warning_0133", variables => { table => $table }}); + $anvil->nice_exit({exit_code => 1}); + } + + my $uuid_width = 0; + my $count_width = 0; + my $column1 = $table."_uuid"; + my $column2 = ""; + my $column3 = ""; + my $column4 = ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column1 => $column1 }}); + if ($table =~ /^(.*)s$/) + { + $column2 = $1."_uuid"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column2 => $column2 }}); + } + if ($table =~ /^(.*)es$/) + { + $column3 = $1."_uuid"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column3 => $column3 }}); + } + if ($table =~ /^(.*)ies$/) + { + $column4 = $1."y_uuid"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column4 => $column4 }}); + } + $query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND column_name = ".$anvil->Database->quote($column1).";"; + if ($column4) + { + $query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2)." OR column_name = ".$anvil->Database->quote($column3)." OR column_name = ".$anvil->Database->quote($column4).");"; + } + elsif ($column3) + { + $query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2)." OR column_name = ".$anvil->Database->quote($column3).");"; + } + elsif ($column2) + { + $query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2).");"; + } + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); + my $uuid_column = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $uuid_column = "" if not defined $uuid_column; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid_column => $uuid_column }}); + if (not $uuid_column) + { + # This is a problem + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "err", key => "error_0311", variables => { table => $table }}); + $anvil->nice_exit({exit_code => 1}); + } + + # This can take a while, ask the user to be patient. + print $anvil->Words->string({key => "message_0269", variables => { + table => $table, + column => $uuid_column, + }})."\n"; + + # Count how many entries exist for each UUID. + $query = " +SELECT + DISTINCT ".$uuid_column." +FROM + history.".$table." +;"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $column_uuid = $row->[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column_uuid => $column_uuid }}); + + if (length($column_uuid) > $uuid_width) + { + $uuid_width = length($column_uuid); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid_width => $uuid_width }}); + } + + my $query = " +SELECT + COUNT(*) +FROM + history.".$table." +WHERE + ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." +;"; + my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $comma_count = $anvil->Convert->add_commas({number => $count}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + count => $count, + comma_count => $comma_count, + }}); + + $anvil->data->{db_counts}{count}{$count}{$column_uuid} = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "db_counts::count::${count}::${column_uuid}" => $anvil->data->{db_counts}{count}{$count}{$column_uuid}, + }}); + + if (length($comma_count) > $count_width) + { + $count_width = length($comma_count); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count_width => $count_width }}); + } + print "."; + } + print "\n"; + + my $queries = []; + my $divider = "-"; + for (1..$uuid_width) { $divider .= "-"; } + $divider .= "-+-"; + for (1..$count_width) { $divider .= "-"; } + $divider .= "-"; + print $anvil->Words->string({key => "message_0268", variables => { table => $table }})."\n"; + print $divider."\n"; + foreach my $count (sort {$a <=> $b} keys %{$anvil->data->{db_counts}{count}}) + { + my $comma_count = $anvil->Convert->add_commas({number => $count}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + count => $count, + comma_count => $comma_count, + }}); + + if (($anvil->data->{switches}{minimum}) && ($anvil->data->{switches}{minimum} =~ /^\d+$/) && ($count < $anvil->data->{switches}{minimum})) + { + # Skip it. + next; + } + + # Sorting by UUID doesn't really make sense, but it provides consistency run over run. + foreach my $column_uuid (sort {$a cmp $b} keys %{$anvil->data->{db_counts}{count}{$count}}) + { + print " ".sprintf("%${uuid_width}s", $column_uuid)." | ".sprintf("%${count_width}s", $comma_count)." \n"; + + # This will need to be updated by the person debugging a table. + #push @{$queries}, "SELECT variable_name, variable_value, variable_source_table, variable_source_uuid FROM variables WHERE variable_uuid = '".$column_uuid."';"; + } + } + print $divider."\n"; + + # Enable this if you're trying to figure out what data is growing, it needs to be edited on a + # per-table basis. + if (0) + { + foreach my $query (@{$queries}) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0124", variables => { query => $query }}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $variable_name = $row->[0]; + my $variable_value = $row->[1]; + my $source_table = $row->[2]; + my $source_uuid = $row->[3]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:variable_name' => $variable_name, + 's2:variable_value' => $variable_value, + 's3:source_table' => $source_table, + 's4:source_uuid' => $source_uuid, + }}); + + if ($source_table eq "hosts") + { + my $host_name = $anvil->Get->host_name_from_uuid({host_uuid => $source_uuid}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }}); + } + } + } + } + + return(0); +} + +sub count_all +{ + my ($anvil) = @_; + + my $longest_table = 0; + my $longest_public = 0; + my $longest_history = 0; + my $query = " +SELECT + table_schema, + table_name +FROM + information_schema.tables +WHERE + (table_schema = 'public' OR table_schema = 'history') +AND + table_catalog = 'anvil' +ORDER BY + table_name ASC, + table_schema DESC; +;"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $table_schema = $row->[0]; + my $table_name = $row->[1]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + table_schema => $table_schema, + table_name => $table_name, + }}); + + if (not exists $anvil->data->{db_counts}{table}{$table_name}) + { + $anvil->data->{db_counts}{table}{$table_name}{public} = 0; + $anvil->data->{db_counts}{table}{$table_name}{history} = -1; + } + + if (length($table_name) > $longest_table) + { + $longest_table = length($table_name); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_table => $longest_table }}); + } + + my $query = "SELECT COUNT(*) FROM ".$table_schema.".".$table_name.";"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); + my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $comma_count = $anvil->Convert->add_commas({number => $count}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + count => $count, + comma_count => $comma_count, + }}); + + if ($table_schema eq "public") + { + if (length($comma_count) > $longest_public) + { + $longest_public = length($comma_count); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_public => $longest_public }}); + } + } + else + { + if (length($comma_count) > $longest_history) + { + $longest_history = length($comma_count); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_history => $longest_history }}); + } + } + + $anvil->data->{db_counts}{table}{$table_name}{$table_schema} = $count; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "db_counts::table::${table_name}::${table_schema}" => $anvil->data->{db_counts}{table}{$table_name}{$table_schema}, + }}); + } + + my $say_table = $anvil->Words->string({key => "header_0062"}); + my $say_public = $anvil->Words->string({key => "header_0063"}); + my $say_history = $anvil->Words->string({key => "header_0064"}); + my $divider = "-"; + for (1..$longest_table) { $divider .= "-"; } + $divider .= "-+-"; + for (1..$longest_public) { $divider .= "-"; } + $divider .= "-+-"; + for (1..$longest_history) { $divider .= "-"; } + $divider .= "-"; + + print " ".sprintf("%${longest_table}s", "Table")." | ".sprintf("%${longest_public}s", $say_public)." | ".sprintf("%${longest_history}s", $say_history)." \n"; + print $divider."\n"; + foreach my $table_name (sort {$a cmp $b} keys %{$anvil->data->{db_counts}{table}}) + { + if (($anvil->data->{switches}{minimum}) && ($anvil->data->{switches}{minimum} =~ /^\d+$/)) + { + if (($anvil->data->{db_counts}{table}{$table_name}{public} < $anvil->data->{switches}{minimum}) && + ($anvil->data->{db_counts}{table}{$table_name}{history} < $anvil->data->{switches}{minimum})) + { + # Skip it. + next; + } + } + my $public = $anvil->Convert->add_commas({number => $anvil->data->{db_counts}{table}{$table_name}{public}}); + my $history = $anvil->data->{db_counts}{table}{$table_name}{history} == -1 ? "--" : $anvil->Convert->add_commas({number => $anvil->data->{db_counts}{table}{$table_name}{history}}); + print " ".sprintf("%${longest_table}s", $table_name)." | ".sprintf("%${longest_public}s", $public)." | ".sprintf("%${longest_history}s", $history)." \n"; + } + print $divider."\n"; + + return(0); +} + From 796814531e42ef2c7a6cfbeaca85a87fe86688cc Mon Sep 17 00:00:00 2001 From: Digimer Date: Thu, 13 Jan 2022 21:07:25 -0500 Subject: [PATCH 2/3] Fixed a bug in Alert->check_condition_age() where, when the 'clear' parameter was set and the value was already 'clear', it would flip to 'set' erroniously. Signed-off-by: Digimer --- Anvil/Tools/Alert.pm | 5 +++-- scancore-agents/scan-network/scan-network | 8 +++++--- tools/striker-db-report | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Anvil/Tools/Alert.pm b/Anvil/Tools/Alert.pm index b49b0944..dc1d322f 100644 --- a/Anvil/Tools/Alert.pm +++ b/Anvil/Tools/Alert.pm @@ -295,6 +295,7 @@ sub check_condition_age # See if this variable has been set yet. my ($variable_value, $variable_uuid, $epoch_modified_date, $modified_date) = $anvil->Database->read_variable({ + debug => $debug, variable_name => $name, variable_source_table => $source_table, variable_source_uuid => $host_uuid, @@ -319,8 +320,8 @@ sub check_condition_age }); } - # if the value was 'clear', change it to 'set'. - if ($variable_value eq "clear") + # if the 'clear' parameter isn't set, and the value is 'clear', change it to 'set'. + if (($variable_value eq "clear") && (not $clear)) { # Set it. $variable_uuid = $anvil->Database->insert_or_update_variables({ diff --git a/scancore-agents/scan-network/scan-network b/scancore-agents/scan-network/scan-network index 7267fd8e..78c5a2f2 100755 --- a/scancore-agents/scan-network/scan-network +++ b/scancore-agents/scan-network/scan-network @@ -3457,9 +3457,11 @@ AND # Don't set / clear interfaces that appear down but aren't named ifn/bcn/sn as they're probably # unconfigured/unusued interfaces. - my $problem = 0; - my $check = 0; - if ($anvil->Network->is_our_interface({interface => $network_interface_name})) + my $problem = 0; + my $check = 0; + my $monitored = $anvil->Network->is_our_interface({interface => $network_interface_name}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { monitored => $monitored }}); + if ($monitored) { # One we monitor $check = 1; diff --git a/tools/striker-db-report b/tools/striker-db-report index ace40df5..b738c468 100755 --- a/tools/striker-db-report +++ b/tools/striker-db-report @@ -233,7 +233,7 @@ WHERE { foreach my $query (@{$queries}) { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0124", variables => { query => $query }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { From 032f20a267001793a5a368f5de08d6a54886372e Mon Sep 17 00:00:00 2001 From: Digimer Date: Thu, 13 Jan 2022 22:49:31 -0500 Subject: [PATCH 3/3] * Fixed a bug in Database->_age_out_data() where, when all records in the history schema would be purged, the most recent record would not be preserved. The result is that nothing was purged, allow tables to grow dramatically. The 'variables' table was also added to this age-out list. Signed-off-by: Digimer --- Anvil/Tools/Database.pm | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 4fa02d39..d962a618 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -16677,10 +16677,11 @@ sub _age_out_data # We don't use 'anvil->data' to prevent injecting SQL queries in anvil.conf my $to_clean = {}; - # Power, temperatures and ip addresses + # Power, temperatures, ip addresses and variables $to_clean->{table}{temperature}{child_table}{temperature}{uuid_column} = "temperature_uuid"; $to_clean->{table}{power}{child_table}{power}{uuid_column} = "power_uuid"; $to_clean->{table}{ip_addresses}{child_table}{ip_addresses}{uuid_column} = "ip_address_uuid"; + $to_clean->{table}{variables}{child_table}{variables}{uuid_column} = "variable_uuid"; # scan_apc_pdu $to_clean->{table}{scan_apc_pdus}{child_table}{scan_apc_pdu_phases}{uuid_column} = "scan_apc_pdu_phase_uuid"; @@ -16760,7 +16761,7 @@ sub _age_out_data count => $count, }}); - if ($count) + if ($count > 1) { # Find how many records will be left. If it's 0, we'll use an OFFSET 1. my $query = "SELECT history_id FROM history.".$child_table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND modified_date > '".$old_timestamp."';"; @@ -16781,16 +16782,17 @@ sub _age_out_data } else { - # This would delete everything, reserve at least one record. - foreach my $row (@{$results}) - { - my $history_id = $row->[0]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { history_id => $history_id }}); - - my $query = "DELETE FROM history.".$child_table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND history_id = '".$history_id."';"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - push @{$queries}, $query; - } + # This would delete everything, reserve at + # least one record. + my $query = "SELECT history_id FROM history.".$child_table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." ORDER BY modified_date DESC LIMIT 1;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + + my $history_id = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); + + $query = "DELETE FROM history.".$child_table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND modified_date <= '".$old_timestamp."' AND history_id != '".$history_id."';"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + push @{$queries}, $query; } } }