#!/usr/bin/perl
#
# This tool looks at the database and counts how many records are in each database. Optionally, if given a
# table name, it will count the number of entries exist in the history schema for each record in the public
# schema. The goal being to help quickly identifying rapidly growing tables.
#
use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
use Text::Diff;
$| = 1;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
my $anvil = Anvil::Tools->new();
$anvil->Database->connect({debug => 3, check_for_resync => 0});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003"});
$anvil->nice_exit({exit_code => 1});
}
# When set, records are counted in the public table, the the number of history entries for each columng is
# shown, sorted by frequency.
$anvil->data->{switches}{table} = "";
# When set, tables with less than the minium are ignored.
$anvil->data->{switches}{minimum} = 0;
$anvil->Get->switches();
$anvil->data->{switches}{minimum} =~ s/,//g;
if ($anvil->data->{switches}{table})
{
count_table($anvil);
}
else
{
count_all($anvil);
}
$anvil->nice_exit({exit_code => 0});
#############################################################################################################
# Functions #
#############################################################################################################
sub count_table
{
my ($anvil) = @_;
# Make sure the table exists.
my $table = $anvil->Database->quote($anvil->data->{switches}{table});
$table =~ s/^\s+//;
$table =~ s/\s.*//;
$table =~ s/^'(.*)'$/$1/;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { table => $table }});
my $query = "
SELECT
COUNT(*)
FROM
information_schema.tables
WHERE
table_schema = 'history'
AND
table_name = '".$table."'
AND
table_catalog = 'anvil'
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
if (not $count)
{
# Table doesn't exist.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "warning_0133", variables => { table => $table }});
$anvil->nice_exit({exit_code => 1});
}
my $uuid_width = 0;
my $count_width = 0;
my $column1 = $table."_uuid";
my $column2 = "";
my $column3 = "";
my $column4 = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column1 => $column1 }});
if ($table =~ /^(.*)s$/)
{
$column2 = $1."_uuid";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column2 => $column2 }});
}
if ($table =~ /^(.*)es$/)
{
$column3 = $1."_uuid";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column3 => $column3 }});
}
if ($table =~ /^(.*)ies$/)
{
$column4 = $1."y_uuid";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column4 => $column4 }});
}
$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND column_name = ".$anvil->Database->quote($column1).";";
if ($column4)
{
$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2)." OR column_name = ".$anvil->Database->quote($column3)." OR column_name = ".$anvil->Database->quote($column4).");";
}
elsif ($column3)
{
$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2)." OR column_name = ".$anvil->Database->quote($column3).");";
}
elsif ($column2)
{
$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2).");";
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $uuid_column = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$uuid_column = "" if not defined $uuid_column;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid_column => $uuid_column }});
if (not $uuid_column)
{
# This is a problem
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "err", key => "error_0311", variables => { table => $table }});
$anvil->nice_exit({exit_code => 1});
}
# This can take a while, ask the user to be patient.
print $anvil->Words->string({key => "message_0269", variables => {
table => $table,
column => $uuid_column,
}})."\n";
# Count how many entries exist for each UUID.
$query = "
SELECT
DISTINCT ".$uuid_column."
FROM
history.".$table."
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $column_uuid = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column_uuid => $column_uuid }});
if (length($column_uuid) > $uuid_width)
{
$uuid_width = length($column_uuid);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid_width => $uuid_width }});
}
my $query = "
SELECT
COUNT(*)
FROM
history.".$table."
WHERE
".$uuid_column." = ".$anvil->Database->quote($column_uuid)."
;";
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
my $comma_count = $anvil->Convert->add_commas({number => $count});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
count => $count,
comma_count => $comma_count,
}});
$anvil->data->{db_counts}{count}{$count}{$column_uuid} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"db_counts::count::${count}::${column_uuid}" => $anvil->data->{db_counts}{count}{$count}{$column_uuid},
}});
if (length($comma_count) > $count_width)
{
$count_width = length($comma_count);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count_width => $count_width }});
}
print ".";
}
print "\n";
my $queries = [];
my $divider = "-";
for (1..$uuid_width) { $divider .= "-"; }
$divider .= "-+-";
for (1..$count_width) { $divider .= "-"; }
$divider .= "-";
print $anvil->Words->string({key => "message_0268", variables => { table => $table }})."\n";
print $divider."\n";
foreach my $count (sort {$a <=> $b} keys %{$anvil->data->{db_counts}{count}})
{
my $comma_count = $anvil->Convert->add_commas({number => $count});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
count => $count,
comma_count => $comma_count,
}});
if (($anvil->data->{switches}{minimum}) && ($anvil->data->{switches}{minimum} =~ /^\d+$/) && ($count < $anvil->data->{switches}{minimum}))
{
# Skip it.
next;
}
# Sorting by UUID doesn't really make sense, but it provides consistency run over run.
foreach my $column_uuid (sort {$a cmp $b} keys %{$anvil->data->{db_counts}{count}{$count}})
{
print " ".sprintf("%${uuid_width}s", $column_uuid)." | ".sprintf("%${count_width}s", $comma_count)." \n";
# This will need to be updated by the person debugging a table.
#push @{$queries}, "SELECT variable_name, variable_value, variable_source_table, variable_source_uuid FROM variables WHERE variable_uuid = '".$column_uuid."';";
}
}
print $divider."\n";
# Enable this if you're trying to figure out what data is growing, it needs to be edited on a
# per-table basis.
if (0)
{
foreach my $query (@{$queries})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $variable_name = $row->[0];
my $variable_value = $row->[1];
my $source_table = $row->[2];
my $source_uuid = $row->[3];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:variable_name' => $variable_name,
's2:variable_value' => $variable_value,
's3:source_table' => $source_table,
's4:source_uuid' => $source_uuid,
}});
if ($source_table eq "hosts")
{
my $host_name = $anvil->Get->host_name_from_uuid({host_uuid => $source_uuid});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }});
}
}
}
}
return(0);
}
sub count_all
{
my ($anvil) = @_;
my $longest_table = 0;
my $longest_public = 0;
my $longest_history = 0;
my $query = "
SELECT
table_schema,
table_name
FROM
information_schema.tables
WHERE
(table_schema = 'public' OR table_schema = 'history')
AND
table_catalog = 'anvil'
ORDER BY
table_name ASC,
table_schema DESC;
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $table_schema = $row->[0];
my $table_name = $row->[1];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
table_schema => $table_schema,
table_name => $table_name,
}});
if (not exists $anvil->data->{db_counts}{table}{$table_name})
{
$anvil->data->{db_counts}{table}{$table_name}{public} = 0;
$anvil->data->{db_counts}{table}{$table_name}{history} = -1;
}
if (length($table_name) > $longest_table)
{
$longest_table = length($table_name);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_table => $longest_table }});
}
my $query = "SELECT COUNT(*) FROM ".$table_schema.".".$table_name.";";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
my $comma_count = $anvil->Convert->add_commas({number => $count});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
count => $count,
comma_count => $comma_count,
}});
if ($table_schema eq "public")
{
if (length($comma_count) > $longest_public)
{
$longest_public = length($comma_count);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_public => $longest_public }});
}
}
else
{
if (length($comma_count) > $longest_history)
{
$longest_history = length($comma_count);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_history => $longest_history }});
}
}
$anvil->data->{db_counts}{table}{$table_name}{$table_schema} = $count;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"db_counts::table::${table_name}::${table_schema}" => $anvil->data->{db_counts}{table}{$table_name}{$table_schema},
}});
}
my $say_table = $anvil->Words->string({key => "header_0062"});
my $say_public = $anvil->Words->string({key => "header_0063"});
my $say_history = $anvil->Words->string({key => "header_0064"});
my $divider = "-";
for (1..$longest_table) { $divider .= "-"; }
$divider .= "-+-";
for (1..$longest_public) { $divider .= "-"; }
$divider .= "-+-";
for (1..$longest_history) { $divider .= "-"; }
$divider .= "-";
print " ".sprintf("%${longest_table}s", "Table")." | ".sprintf("%${longest_public}s", $say_public)." | ".sprintf("%${longest_history}s", $say_history)." \n";
print $divider."\n";
foreach my $table_name (sort {$a cmp $b} keys %{$anvil->data->{db_counts}{table}})
{
if (($anvil->data->{switches}{minimum}) && ($anvil->data->{switches}{minimum} =~ /^\d+$/))
{
if (($anvil->data->{db_counts}{table}{$table_name}{public} < $anvil->data->{switches}{minimum}) &&
($anvil->data->{db_counts}{table}{$table_name}{history} < $anvil->data->{switches}{minimum}))
{
# Skip it.
next;
}
}
my $public = $anvil->Convert->add_commas({number => $anvil->data->{db_counts}{table}{$table_name}{public}});
my $history = $anvil->data->{db_counts}{table}{$table_name}{history} == -1 ? "--" : $anvil->Convert->add_commas({number => $anvil->data->{db_counts}{table}{$table_name}{history}});
print " ".sprintf("%${longest_table}s", $table_name)." | ".sprintf("%${longest_public}s", $public)." | ".sprintf("%${longest_history}s", $history)." \n";
}
print $divider."\n";
return(0);
}
