Digimer
3 years ago
committed by
GitHub
commit
861d565cce
5 changed files with 406 additions and 17 deletions
@ -0,0 +1,381 @@ |
||||
#!/usr/bin/perl |
||||
# |
||||
# This tool looks at the database and counts how many records are in each database. Optionally, if given a |
||||
# table name, it will count the number of entries exist in the history schema for each record in the public |
||||
# schema. The goal being to help quickly identifying rapidly growing tables. |
||||
# |
||||
|
||||
use strict; |
||||
use warnings; |
||||
use Anvil::Tools; |
||||
use Data::Dumper; |
||||
use Text::Diff; |
||||
|
||||
$| = 1; |
||||
|
||||
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; |
||||
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; |
||||
if (($running_directory =~ /^\./) && ($ENV{PWD})) |
||||
{ |
||||
$running_directory =~ s/^\./$ENV{PWD}/; |
||||
} |
||||
|
||||
my $anvil = Anvil::Tools->new(); |
||||
|
||||
$anvil->Database->connect({debug => 3, check_for_resync => 0}); |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); |
||||
if (not $anvil->data->{sys}{database}{connections}) |
||||
{ |
||||
# No databases, exit. |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003"}); |
||||
$anvil->nice_exit({exit_code => 1}); |
||||
} |
||||
|
||||
# When set, records are counted in the public table, the the number of history entries for each columng is |
||||
# shown, sorted by frequency. |
||||
$anvil->data->{switches}{table} = ""; |
||||
# When set, tables with less than the minium are ignored. |
||||
$anvil->data->{switches}{minimum} = 0; |
||||
$anvil->Get->switches(); |
||||
$anvil->data->{switches}{minimum} =~ s/,//g; |
||||
|
||||
if ($anvil->data->{switches}{table}) |
||||
{ |
||||
count_table($anvil); |
||||
} |
||||
else |
||||
{ |
||||
count_all($anvil); |
||||
} |
||||
|
||||
$anvil->nice_exit({exit_code => 0}); |
||||
|
||||
|
||||
|
||||
############################################################################################################# |
||||
# Functions # |
||||
############################################################################################################# |
||||
|
||||
sub count_table |
||||
{ |
||||
my ($anvil) = @_; |
||||
|
||||
# Make sure the table exists. |
||||
my $table = $anvil->Database->quote($anvil->data->{switches}{table}); |
||||
$table =~ s/^\s+//; |
||||
$table =~ s/\s.*//; |
||||
$table =~ s/^'(.*)'$/$1/; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { table => $table }}); |
||||
my $query = " |
||||
SELECT |
||||
COUNT(*) |
||||
FROM |
||||
information_schema.tables |
||||
WHERE |
||||
table_schema = 'history' |
||||
AND |
||||
table_name = '".$table."' |
||||
AND |
||||
table_catalog = 'anvil' |
||||
;"; |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); |
||||
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); |
||||
if (not $count) |
||||
{ |
||||
# Table doesn't exist. |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "warning_0133", variables => { table => $table }}); |
||||
$anvil->nice_exit({exit_code => 1}); |
||||
} |
||||
|
||||
my $uuid_width = 0; |
||||
my $count_width = 0; |
||||
my $column1 = $table."_uuid"; |
||||
my $column2 = ""; |
||||
my $column3 = ""; |
||||
my $column4 = ""; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column1 => $column1 }}); |
||||
if ($table =~ /^(.*)s$/) |
||||
{ |
||||
$column2 = $1."_uuid"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column2 => $column2 }}); |
||||
} |
||||
if ($table =~ /^(.*)es$/) |
||||
{ |
||||
$column3 = $1."_uuid"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column3 => $column3 }}); |
||||
} |
||||
if ($table =~ /^(.*)ies$/) |
||||
{ |
||||
$column4 = $1."y_uuid"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column4 => $column4 }}); |
||||
} |
||||
$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND column_name = ".$anvil->Database->quote($column1).";"; |
||||
if ($column4) |
||||
{ |
||||
$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2)." OR column_name = ".$anvil->Database->quote($column3)." OR column_name = ".$anvil->Database->quote($column4).");"; |
||||
} |
||||
elsif ($column3) |
||||
{ |
||||
$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2)." OR column_name = ".$anvil->Database->quote($column3).");"; |
||||
} |
||||
elsif ($column2) |
||||
{ |
||||
$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2).");"; |
||||
} |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); |
||||
my $uuid_column = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; |
||||
$uuid_column = "" if not defined $uuid_column; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid_column => $uuid_column }}); |
||||
if (not $uuid_column) |
||||
{ |
||||
# This is a problem |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "err", key => "error_0311", variables => { table => $table }}); |
||||
$anvil->nice_exit({exit_code => 1}); |
||||
} |
||||
|
||||
# This can take a while, ask the user to be patient. |
||||
print $anvil->Words->string({key => "message_0269", variables => { |
||||
table => $table, |
||||
column => $uuid_column, |
||||
}})."\n"; |
||||
|
||||
# Count how many entries exist for each UUID. |
||||
$query = " |
||||
SELECT |
||||
DISTINCT ".$uuid_column." |
||||
FROM |
||||
history.".$table." |
||||
;"; |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); |
||||
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); |
||||
$count = @{$results}; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
results => $results, |
||||
count => $count, |
||||
}}); |
||||
foreach my $row (@{$results}) |
||||
{ |
||||
my $column_uuid = $row->[0]; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column_uuid => $column_uuid }}); |
||||
|
||||
if (length($column_uuid) > $uuid_width) |
||||
{ |
||||
$uuid_width = length($column_uuid); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid_width => $uuid_width }}); |
||||
} |
||||
|
||||
my $query = " |
||||
SELECT |
||||
COUNT(*) |
||||
FROM |
||||
history.".$table." |
||||
WHERE |
||||
".$uuid_column." = ".$anvil->Database->quote($column_uuid)." |
||||
;"; |
||||
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; |
||||
my $comma_count = $anvil->Convert->add_commas({number => $count}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
count => $count, |
||||
comma_count => $comma_count, |
||||
}}); |
||||
|
||||
$anvil->data->{db_counts}{count}{$count}{$column_uuid} = 1; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
"db_counts::count::${count}::${column_uuid}" => $anvil->data->{db_counts}{count}{$count}{$column_uuid}, |
||||
}}); |
||||
|
||||
if (length($comma_count) > $count_width) |
||||
{ |
||||
$count_width = length($comma_count); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count_width => $count_width }}); |
||||
} |
||||
print "."; |
||||
} |
||||
print "\n"; |
||||
|
||||
my $queries = []; |
||||
my $divider = "-"; |
||||
for (1..$uuid_width) { $divider .= "-"; } |
||||
$divider .= "-+-"; |
||||
for (1..$count_width) { $divider .= "-"; } |
||||
$divider .= "-"; |
||||
print $anvil->Words->string({key => "message_0268", variables => { table => $table }})."\n"; |
||||
print $divider."\n"; |
||||
foreach my $count (sort {$a <=> $b} keys %{$anvil->data->{db_counts}{count}}) |
||||
{ |
||||
my $comma_count = $anvil->Convert->add_commas({number => $count}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
count => $count, |
||||
comma_count => $comma_count, |
||||
}}); |
||||
|
||||
if (($anvil->data->{switches}{minimum}) && ($anvil->data->{switches}{minimum} =~ /^\d+$/) && ($count < $anvil->data->{switches}{minimum})) |
||||
{ |
||||
# Skip it. |
||||
next; |
||||
} |
||||
|
||||
# Sorting by UUID doesn't really make sense, but it provides consistency run over run. |
||||
foreach my $column_uuid (sort {$a cmp $b} keys %{$anvil->data->{db_counts}{count}{$count}}) |
||||
{ |
||||
print " ".sprintf("%${uuid_width}s", $column_uuid)." | ".sprintf("%${count_width}s", $comma_count)." \n"; |
||||
|
||||
# This will need to be updated by the person debugging a table. |
||||
#push @{$queries}, "SELECT variable_name, variable_value, variable_source_table, variable_source_uuid FROM variables WHERE variable_uuid = '".$column_uuid."';"; |
||||
} |
||||
} |
||||
print $divider."\n"; |
||||
|
||||
# Enable this if you're trying to figure out what data is growing, it needs to be edited on a |
||||
# per-table basis. |
||||
if (0) |
||||
{ |
||||
foreach my $query (@{$queries}) |
||||
{ |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); |
||||
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); |
||||
my $count = @{$results}; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { |
||||
results => $results, |
||||
count => $count, |
||||
}}); |
||||
foreach my $row (@{$results}) |
||||
{ |
||||
my $variable_name = $row->[0]; |
||||
my $variable_value = $row->[1]; |
||||
my $source_table = $row->[2]; |
||||
my $source_uuid = $row->[3]; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
's1:variable_name' => $variable_name, |
||||
's2:variable_value' => $variable_value, |
||||
's3:source_table' => $source_table, |
||||
's4:source_uuid' => $source_uuid, |
||||
}}); |
||||
|
||||
if ($source_table eq "hosts") |
||||
{ |
||||
my $host_name = $anvil->Get->host_name_from_uuid({host_uuid => $source_uuid}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }}); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
return(0); |
||||
} |
||||
|
||||
sub count_all |
||||
{ |
||||
my ($anvil) = @_; |
||||
|
||||
my $longest_table = 0; |
||||
my $longest_public = 0; |
||||
my $longest_history = 0; |
||||
my $query = " |
||||
SELECT |
||||
table_schema, |
||||
table_name |
||||
FROM |
||||
information_schema.tables |
||||
WHERE |
||||
(table_schema = 'public' OR table_schema = 'history') |
||||
AND |
||||
table_catalog = 'anvil' |
||||
ORDER BY |
||||
table_name ASC, |
||||
table_schema DESC; |
||||
;"; |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); |
||||
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); |
||||
my $count = @{$results}; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
results => $results, |
||||
count => $count, |
||||
}}); |
||||
foreach my $row (@{$results}) |
||||
{ |
||||
my $table_schema = $row->[0]; |
||||
my $table_name = $row->[1]; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
table_schema => $table_schema, |
||||
table_name => $table_name, |
||||
}}); |
||||
|
||||
if (not exists $anvil->data->{db_counts}{table}{$table_name}) |
||||
{ |
||||
$anvil->data->{db_counts}{table}{$table_name}{public} = 0; |
||||
$anvil->data->{db_counts}{table}{$table_name}{history} = -1; |
||||
} |
||||
|
||||
if (length($table_name) > $longest_table) |
||||
{ |
||||
$longest_table = length($table_name); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_table => $longest_table }}); |
||||
} |
||||
|
||||
my $query = "SELECT COUNT(*) FROM ".$table_schema.".".$table_name.";"; |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); |
||||
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; |
||||
my $comma_count = $anvil->Convert->add_commas({number => $count}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
count => $count, |
||||
comma_count => $comma_count, |
||||
}}); |
||||
|
||||
if ($table_schema eq "public") |
||||
{ |
||||
if (length($comma_count) > $longest_public) |
||||
{ |
||||
$longest_public = length($comma_count); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_public => $longest_public }}); |
||||
} |
||||
} |
||||
else |
||||
{ |
||||
if (length($comma_count) > $longest_history) |
||||
{ |
||||
$longest_history = length($comma_count); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_history => $longest_history }}); |
||||
} |
||||
} |
||||
|
||||
$anvil->data->{db_counts}{table}{$table_name}{$table_schema} = $count; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
"db_counts::table::${table_name}::${table_schema}" => $anvil->data->{db_counts}{table}{$table_name}{$table_schema}, |
||||
}}); |
||||
} |
||||
|
||||
my $say_table = $anvil->Words->string({key => "header_0062"}); |
||||
my $say_public = $anvil->Words->string({key => "header_0063"}); |
||||
my $say_history = $anvil->Words->string({key => "header_0064"}); |
||||
my $divider = "-"; |
||||
for (1..$longest_table) { $divider .= "-"; } |
||||
$divider .= "-+-"; |
||||
for (1..$longest_public) { $divider .= "-"; } |
||||
$divider .= "-+-"; |
||||
for (1..$longest_history) { $divider .= "-"; } |
||||
$divider .= "-"; |
||||
|
||||
print " ".sprintf("%${longest_table}s", "Table")." | ".sprintf("%${longest_public}s", $say_public)." | ".sprintf("%${longest_history}s", $say_history)." \n"; |
||||
print $divider."\n"; |
||||
foreach my $table_name (sort {$a cmp $b} keys %{$anvil->data->{db_counts}{table}}) |
||||
{ |
||||
if (($anvil->data->{switches}{minimum}) && ($anvil->data->{switches}{minimum} =~ /^\d+$/)) |
||||
{ |
||||
if (($anvil->data->{db_counts}{table}{$table_name}{public} < $anvil->data->{switches}{minimum}) && |
||||
($anvil->data->{db_counts}{table}{$table_name}{history} < $anvil->data->{switches}{minimum})) |
||||
{ |
||||
# Skip it. |
||||
next; |
||||
} |
||||
} |
||||
my $public = $anvil->Convert->add_commas({number => $anvil->data->{db_counts}{table}{$table_name}{public}}); |
||||
my $history = $anvil->data->{db_counts}{table}{$table_name}{history} == -1 ? "--" : $anvil->Convert->add_commas({number => $anvil->data->{db_counts}{table}{$table_name}{history}}); |
||||
print " ".sprintf("%${longest_table}s", $table_name)." | ".sprintf("%${longest_public}s", $public)." | ".sprintf("%${longest_history}s", $history)." \n"; |
||||
} |
||||
print $divider."\n"; |
||||
|
||||
return(0); |
||||
} |
||||
|
||||
Loading…
Reference in new issue