anvil/tools/striker-db-report

#!/usr/bin/perl
# 
# This tool looks at the database and counts how many records are in each database. Optionally, if given a 
# table name, it will count the number of entries exist in the history schema for each record in the public 
# schema. The goal being to help quickly identifying rapidly growing tables.
# 

use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
use Text::Diff;

$| = 1;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

my $anvil = Anvil::Tools->new();

$anvil->Database->connect({debug => 3, check_for_resync => 0});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003"});
	$anvil->nice_exit({exit_code => 1});
}

# When set, records are counted in the public table, the the number of history entries for each columng is 
# shown, sorted by frequency.
$anvil->data->{switches}{table}   = "";
# When set, tables with less than the minium are ignored.
$anvil->data->{switches}{minimum} = 0;
$anvil->Get->switches();
$anvil->data->{switches}{minimum} =~ s/,//g;

if ($anvil->data->{switches}{table})
{
	count_table($anvil);
}
else
{
	count_all($anvil);
}

$anvil->nice_exit({exit_code => 0});



#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

sub count_table
{
	my ($anvil) = @_;
	
	# Make sure the table exists.
	my $table =  $anvil->Database->quote($anvil->data->{switches}{table});
	   $table =~ s/^\s+//;
	   $table =~ s/\s.*//;
	   $table =~ s/^'(.*)'$/$1/;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { table => $table }});
	my $query = "
SELECT 
    COUNT(*) 
FROM 
    information_schema.tables 
WHERE 
    table_schema  = 'history'
AND 
    table_name    = '".$table."'
AND 
    table_catalog = 'anvil' 
;";
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
	my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
	if (not $count)
	{
		# Table doesn't exist.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "warning_0133", variables => { table => $table }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	my $uuid_width  = 0;
	my $count_width = 0;
	my $column1     = $table."_uuid";
	my $column2     = "";
	my $column3     = "";
	my $column4     = "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column1 => $column1 }});
	if ($table =~ /^(.*)s$/)
	{
		$column2 = $1."_uuid";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column2 => $column2 }});
	}
	if ($table =~ /^(.*)es$/)
	{
		$column3 = $1."_uuid";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column3 => $column3 }});
	}
	if ($table =~ /^(.*)ies$/)
	{
		$column4 = $1."y_uuid";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column4 => $column4 }});
	}
	$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND column_name = ".$anvil->Database->quote($column1).";";
	if ($column4)
	{
		$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2)." OR column_name = ".$anvil->Database->quote($column3)." OR column_name = ".$anvil->Database->quote($column4).");";
	}
	elsif ($column3)
	{
		$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2)." OR column_name = ".$anvil->Database->quote($column3).");";
	}
	elsif ($column2)
	{
		$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = '".$table."' AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$anvil->Database->quote($column1)." OR column_name = ".$anvil->Database->quote($column2).");";
	}
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
	my $uuid_column = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
	   $uuid_column = "" if not defined $uuid_column;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid_column => $uuid_column }});
	if (not $uuid_column)
	{
		# This is a problem
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "err", key => "error_0311", variables => { table => $table }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# This can take a while, ask the user to be patient.
	print $anvil->Words->string({key => "message_0269", variables => { 
		table  => $table,
		column => $uuid_column,
	}})."\n";
	
	# Count how many entries exist for each UUID.
	$query = "
SELECT 
    DISTINCT ".$uuid_column." 
FROM 
    history.".$table."
;";
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
	   $count   = @{$results};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		results => $results, 
		count   => $count,
	}});
	foreach my $row (@{$results})
	{
		my $column_uuid = $row->[0];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column_uuid => $column_uuid }});
		
		if (length($column_uuid) > $uuid_width)
		{
			$uuid_width = length($column_uuid);
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid_width => $uuid_width }});
		}
	
		my $query = "
SELECT 
    COUNT(*) 
FROM 
    history.".$table." 
WHERE 
    ".$uuid_column." = ".$anvil->Database->quote($column_uuid)."
;";
		my $count       = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
		my $comma_count = $anvil->Convert->add_commas({number => $count});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			count       => $count, 
			comma_count => $comma_count,
		}});
		
		$anvil->data->{db_counts}{count}{$count}{$column_uuid} = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"db_counts::count::${count}::${column_uuid}" => $anvil->data->{db_counts}{count}{$count}{$column_uuid},
		}});
		
		if (length($comma_count) > $count_width)
		{
			$count_width = length($comma_count);
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count_width => $count_width }});
		}
		print ".";
	}
	print "\n";
	
	my $queries = [];
	my $divider = "-";
	for (1..$uuid_width) { $divider .= "-"; }
	$divider .= "-+-";
	for (1..$count_width) { $divider .= "-"; }
	$divider .= "-";
	print $anvil->Words->string({key => "message_0268", variables => { table => $table }})."\n";
	print $divider."\n";
	foreach my $count (sort {$a <=> $b} keys %{$anvil->data->{db_counts}{count}})
	{
		my $comma_count = $anvil->Convert->add_commas({number => $count});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			count       => $count, 
			comma_count => $comma_count,
		}});
		
		if (($anvil->data->{switches}{minimum}) && ($anvil->data->{switches}{minimum} =~ /^\d+$/) && ($count < $anvil->data->{switches}{minimum}))
		{
			# Skip it.
			next;
		}
		
		# Sorting by UUID doesn't really make sense, but it provides consistency run over run.
		foreach my $column_uuid (sort {$a cmp $b} keys %{$anvil->data->{db_counts}{count}{$count}})
		{
			print " ".sprintf("%${uuid_width}s", $column_uuid)." | ".sprintf("%${count_width}s", $comma_count)." \n";
			
			# This will need to be updated by the person debugging a table.
			#push @{$queries}, "SELECT variable_name, variable_value, variable_source_table, variable_source_uuid FROM variables WHERE variable_uuid = '".$column_uuid."';";
		}
	}
	print $divider."\n";
	
	# Enable this if you're trying to figure out what data is growing, it needs to be edited on a 
	# per-table basis.
	if (0)
	{
		foreach my $query (@{$queries})
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0124", variables => { query => $query }});
			my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
			my $count   = @{$results};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				results => $results, 
				count   => $count,
			}});
			foreach my $row (@{$results})
			{
				my $variable_name  = $row->[0];
				my $variable_value = $row->[1];
				my $source_table   = $row->[2];
				my $source_uuid    = $row->[3];
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:variable_name'  => $variable_name,
					's2:variable_value' => $variable_value, 
					's3:source_table'   => $source_table, 
					's4:source_uuid'    => $source_uuid, 
				}});
				
				if ($source_table eq "hosts")
				{
					my $host_name = $anvil->Get->host_name_from_uuid({host_uuid => $source_uuid});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }});
				}
			}
		}
	}
	
	return(0);
}

sub count_all
{
	my ($anvil) = @_;
	
	my $longest_table   = 0;
	my $longest_public  = 0;
	my $longest_history = 0;
	my $query           = "
SELECT 
    table_schema, 
    table_name 
FROM 
    information_schema.tables 
WHERE 
    (table_schema = 'public' OR table_schema = 'history') 
AND 
    table_catalog = 'anvil' 
ORDER BY 
    table_name ASC, 
    table_schema DESC;
;";
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
	my $count   = @{$results};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		results => $results, 
		count   => $count,
	}});
	foreach my $row (@{$results})
	{
		my $table_schema = $row->[0];
		my $table_name   = $row->[1];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			table_schema => $table_schema, 
			table_name   => $table_name,
		}});
		
		if (not exists $anvil->data->{db_counts}{table}{$table_name})
		{
			$anvil->data->{db_counts}{table}{$table_name}{public}  = 0;
			$anvil->data->{db_counts}{table}{$table_name}{history} = -1;
		}
		
		if (length($table_name) > $longest_table)
		{
			$longest_table = length($table_name);
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_table => $longest_table }});
		}
		
		my $query = "SELECT COUNT(*) FROM ".$table_schema.".".$table_name.";";
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
		my $count       = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
		my $comma_count = $anvil->Convert->add_commas({number => $count});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			count       => $count,
			comma_count => $comma_count, 
		}});
		
		if ($table_schema eq "public")
		{
			if (length($comma_count) > $longest_public)
			{
				$longest_public = length($comma_count);
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_public => $longest_public }});
			}
		}
		else
		{
			if (length($comma_count) > $longest_history)
			{
				$longest_history = length($comma_count);
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_history => $longest_history }});
			}
		}
		
		$anvil->data->{db_counts}{table}{$table_name}{$table_schema} = $count;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"db_counts::table::${table_name}::${table_schema}" => $anvil->data->{db_counts}{table}{$table_name}{$table_schema},
		}});
	}
	
	my $say_table   = $anvil->Words->string({key => "header_0062"});
	my $say_public  = $anvil->Words->string({key => "header_0063"});
	my $say_history = $anvil->Words->string({key => "header_0064"});
	my $divider     = "-";
	for (1..$longest_table) { $divider .= "-"; }
	$divider .= "-+-";
	for (1..$longest_public) { $divider .= "-"; }
	$divider .= "-+-";
	for (1..$longest_history) { $divider .= "-"; }
	$divider .= "-";
	
	print " ".sprintf("%${longest_table}s", "Table")." | ".sprintf("%${longest_public}s", $say_public)." | ".sprintf("%${longest_history}s", $say_history)." \n";
	print $divider."\n";
	foreach my $table_name (sort {$a cmp $b} keys %{$anvil->data->{db_counts}{table}})
	{
		if (($anvil->data->{switches}{minimum}) && ($anvil->data->{switches}{minimum} =~ /^\d+$/))
		{
			if (($anvil->data->{db_counts}{table}{$table_name}{public}  < $anvil->data->{switches}{minimum}) && 
			    ($anvil->data->{db_counts}{table}{$table_name}{history} < $anvil->data->{switches}{minimum}))
			{
				# Skip it.
				next;
			}
		}
		my $public  = $anvil->Convert->add_commas({number => $anvil->data->{db_counts}{table}{$table_name}{public}});
		my $history = $anvil->data->{db_counts}{table}{$table_name}{history} == -1 ? "--" : $anvil->Convert->add_commas({number => $anvil->data->{db_counts}{table}{$table_name}{history}});
		print " ".sprintf("%${longest_table}s", $table_name)." | ".sprintf("%${longest_public}s", $public)." | ".sprintf("%${longest_history}s", $history)." \n";
	}
	print $divider."\n";
	
	return(0);
}