anvil/tools/striker-parse-oui

#!/usr/bin/perl
# 
# This periodically reads in http://standards-oui.ieee.org/oui/oui.txt, if possible, and parses it to update/
# populate the oui database table.
# 
# TODO: 
# 

use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

my $anvil = Anvil::Tools->new();

$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, exit.
	print $anvil->Words->string({key => "error_0003"})."\n";
	$anvil->nice_exit({exit_code => 2});
}

$anvil->data->{switches}{'job-uuid'} = "";
$anvil->data->{switches}{force}      = "";
$anvil->Get->switches;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
	"switches::job-uuid" => $anvil->data->{switches}{'job-uuid'}, 
	"switches::force"    => $anvil->data->{switches}{force}, 
}});

update_progress($anvil, 0, "clear");
update_progress($anvil, 1, "log_0239,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!");
$anvil->data->{progress} = 1;

check_if_time($anvil);

my $oui_file = $anvil->Get->users_home({debug => 3})."/oui.txt";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { oui_file => $oui_file }});

my $download = 1;
my $process  = 0;
if (-e $oui_file)
{
	# How long ago did we download it?
	my $refresh_time   = 259200;
	my $modified_time  = (stat($oui_file))[9];
	my $size           = (stat($oui_file))[7];
	my $age            = time - $modified_time;
	my $download_after = $refresh_time - $age;
	   $download_after = 0 if $download_after < 0;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:oui_file'       => $oui_file, 
		's2:modified_time'  => $modified_time." (".$anvil->Get->date_and_time({use_time => $modified_time}).")",
		's3:age'            => $anvil->Convert->add_commas({number => $age})." (".$anvil->Convert->time({'time' => $age, translate => 1}).")", 
		's4:refresh_time'   => $anvil->Convert->add_commas({number => $refresh_time})." (".$anvil->Convert->time({'time' => $refresh_time, translate => 1}).")", 
		's5:download_after' => $anvil->Convert->add_commas({number => $download_after})." (".$anvil->Convert->time({'time' => $download_after, translate => 1}).")", 
		's6:size'           => $anvil->Convert->add_commas({number => $size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $size}).")", 
	}});
	if (($download_after) && ($size > 0))
	{
		# It's less than three days old, don't download. Do parse though (for now at least)
		$download = 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { download => $download }});
	}
	if ((not $download) && ($size > 0))
	{
		$process = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { process => $process }});
	}
}

$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { download => $download }});
if ($download)
{
	update_progress($anvil, 10, "log_0447,!!url!".$anvil->data->{path}{urls}{oui_file}."!!,!!file!".$oui_file."!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0095", variables => { 
		url  => $anvil->data->{path}{urls}{oui_file},
		file => $oui_file,
	}});
	my $download_file = $anvil->Network->download({
		debug     => 2,
		url       => $anvil->data->{path}{urls}{oui_file},
		save_to   => $oui_file,
		overwrite => 1,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { download_file => $download_file }});
	
	if (($download_file) && ($download_file eq $oui_file))
	{
		$process = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { process => $process }});
	}
	else
	{
		# Something went wrong. Even if the file exists, there's no sense processing it.
		update_progress($anvil, 50, "error_0095,!!url!".$anvil->data->{path}{urls}{oui_file}."!!,!!file!".$oui_file."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0095", variables => { 
			url  => $anvil->data->{path}{urls}{oui_file},
			file => $oui_file,
		}});
	}
}

$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
	oui_file => $oui_file, 
	process  => $process,
}});
if ((-e $oui_file) && ($process))
{
	process_oui($anvil, $oui_file);
}

# Update the parse time to now.
$anvil->Database->insert_or_update_variables({
	debug                 => 2,
	variable_name         => "parse-oui::parsed", 
	variable_value        => time, 
	variable_default      => "", 
	variable_description  => "striker_0186", 
	variable_section      => "system", 
	variable_source_uuid  => $anvil->Get->host_uuid, 
	variable_source_table => "hosts", 
});

# We're done
print $anvil->Words->string({key => "message_0025"})."\n";
update_progress($anvil, 100, "message_0025");
$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This checks to see if it's time to parse the OUI file yet or not. If not, it will exit the program. If so, 
# it returns with '0'.
sub check_if_time
{
	my ($anvil) = @_;
	
	# NOTE: We only scan once a day, unless 'force' is used.
	if ($anvil->data->{switches}{force})
	{
		return(0);
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		# No job_uuid, so a manual call.
		return(0);
	}
	else
	{
		my ($unixtime, $variable_uuid, $modified_date) = $anvil->Database->read_variable({variable_name => "parse-oui::parsed"});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
			unixtime      => $unixtime, 
			variable_uuid => $variable_uuid, 
			modified_date => $modified_date, 
		}});
		if (($unixtime eq "") or ($unixtime =~ /\D/))
		{
			$unixtime = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { unixtime => $unixtime }});
		}
		
		# If the database variable 'parse-oui::parsed' is not set, or is more than 24 hours old, 
		# proceed with the parse.
		$anvil->data->{'parse-oui'}{'parse-period'} =  86400 if not defined $anvil->data->{'parse-oui'}{'parse-period'};
		$anvil->data->{'parse-oui'}{'parse-period'} =~ s/,//g;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
			'parse-oui::parse-period' => $anvil->data->{'parse-oui'}{'parse-period'}, 
		}});
		if ($anvil->data->{'parse-oui'}{'parse-period'} =~ /\D/)
		{
			$anvil->data->{'parse-oui'}{'parse-period'} = 86400;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				'parse-oui::parse-period' => $anvil->data->{'parse-oui'}{'parse-period'}, 
			}});
		}
		my $time_now   = time;
		my $next_parse  = $unixtime + $anvil->data->{'parse-oui'}{'parse-period'};
		my $difference = ($next_parse - $time_now);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
			's1:time_now'   => $time_now, 
			's2:next_parse'  => $next_parse, 
			's3:difference' => $difference, 
		}});
		if ((not $variable_uuid) or ($unixtime !~ /^\d+/) or ($difference < 0))
		{
			# It's been long enough (or it's the first time), scan.
			return(0);
		}
		elsif ($difference > 0)
		{
			# Log when the next scan will happen and then exit.
			my $say_when = $anvil->Convert->time({'time' => $difference, long => 1});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, 'print' => 1, key => "log_0468", variables => { next_parse => $anvil->Convert->add_commas({number => $say_when}) }});
			update_progress($anvil, 100, "log_0468,!!next_parse!".$anvil->Convert->add_commas({number => $say_when})."!!");
			$anvil->nice_exit({exit_code => 3});
		}
	}
	
	return(0);
}

# If this is being called as a job, this will allow the progress to be updated.
sub update_progress
{
	my ($anvil, $progress, $message) = @_;

	if (not $anvil->data->{switches}{'job-uuid'})
	{
		return(0);
	}
	
	$anvil->Job->update_progress({
		progress => $progress, 
		message  => $message,
		job_uuid => $anvil->data->{switches}{'job-uuid'},
	});
	
	return(0);
}

# This actually processes the OUI file.
sub process_oui
{
	my ($anvil, $oui_file) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { oui_file => $oui_file }});
	
	# Read in the file.
	update_progress($anvil, 25, "log_0448,!!file!".$oui_file."!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0448", variables => { file => $oui_file }});
	my $oui        = "";
	my $vendor     = "";
	my $address    = "";
	my ($oui_body) = $anvil->Storage->read_file({
		debug      => 3,
		file       => $oui_file,
		cache      => 0, 
		force_read => 1
	});
	
	### TODO: For some reason, ending the file on an empty line wasn't triggering the save of the last 
	###       record. So the EOF line/check was added, at least until I can get undumb enough to see what
	###       the real problem is.
	# The OUI list doesn't include an entry for Red Hat / 52:54:00. So we'll inject it here.
	$oui_body .= "
52-54-00   (hex)                QEMU Virtual NIC
525400     (base 16)            QEMU Virtual NIC

EOF
";
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { oui_body => $oui_body }});
	foreach my $line (split/\n/, $oui_body)
	{
		$line = $anvil->Words->clean_spaces({'string' => $line});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
		
		if ((not $line) or ($line eq "EOF"))
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { oui => $oui }});
			if ($oui)
			{
				$address =~ s/, $//;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
					oui     => $oui, 
					vendor  => $vendor, 
					address => $address, 
				}});
				if (not $address)
				{
					# This isn't translated
					$address = "<unknown>";
				}
				
				# NOTE: There are duplicates in the OUI file, so we'll string te entries together
				if ((exists $anvil->data->{oui}{$oui}) && ($anvil->data->{oui}{$oui}{name}))
				{
					$anvil->data->{oui}{$oui}{name}    .= " / ".$vendor;
					$anvil->data->{oui}{$oui}{address} .= " / ". $address;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
						"oui::${oui}::name"    => $anvil->data->{oui}{$oui}{name}, 
						"oui::${oui}::address" => $anvil->data->{oui}{$oui}{address}, 
					}});
				}
				else
				{
					$anvil->data->{oui}{$oui}{name}    = $vendor;
					$anvil->data->{oui}{$oui}{address} = $address;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
						"oui::${oui}::name"    => $anvil->data->{oui}{$oui}{name}, 
						"oui::${oui}::address" => $anvil->data->{oui}{$oui}{address}, 
					}});
				}
			}
			$oui     = "";
			$vendor  = "";
			$address = "";
			next;
		}
		$line =~ s/^\s+//;
		$line =~ s/\s+$//;
		if ($line =~ /^(\w\w-\w\w-\w\w)\s+\(hex\)\s+(.*)/)
		{
			$oui    =  $1;
			$vendor =  $2;
			$oui    =~ s/-/:/g;
			$oui    =  lc($oui);
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				oui    => $oui, 
				vendor => $vendor, 
			}});
			next;
		}
		next if not $oui;
		if ($line =~ /^(\w\w\w\w\w\w)\s+\(base 16\)\s+(.*)/)
		{
			my $oui2    =  $1;
			my $vendor2 =  $2;
			   $oui2    =~ s/-/:/g;
			   $oui2    =~ s/^(\w\w)(\w\w)(\w\w)$/$1:$2:$3/g;
			   $oui2    =  lc($oui2);
			   $oui     =  $oui2    if not $oui;
			   $vendor  =  $vendor2 if not $vendor;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				oui     => $oui, 
				oui2    => $oui2, 
				vendor  => $vendor, 
				vendor2 => $vendor2, 
			}});
			next;
		}
		else
		{
			$address .= "$line, ";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { address => $address }});
		}
	}
	
	# Record the details.
	my $records = keys %{$anvil->data->{oui}};
	   $records = $anvil->Convert->add_commas({number => $records });
	update_progress($anvil, 50, "log_0449,!!records!".$records."!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0449", variables => { records => $records }});
	foreach my $oui (sort {$a cmp $b} keys %{$anvil->data->{oui}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
			"oui::${oui}::name"    => $anvil->data->{oui}{$oui}{name}, 
			"oui::${oui}::address" => $anvil->data->{oui}{$oui}{address}, 
		}});
		my ($oui_uuid) = $anvil->Database->insert_or_update_oui({
			debug               => 3,
			file                => $THIS_FILE, 
			line                => __LINE__, 
			oui_mac_prefix      => $oui, 
			oui_company_address => $anvil->data->{oui}{$oui}{address},
			oui_company_name    => $anvil->data->{oui}{$oui}{name}, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { oui_uuid => $oui_uuid }});
	}
	
	return(0);
}