From 7e960f163230250157c55da5c22b9a96eec30362 Mon Sep 17 00:00:00 2001 From: Digimer Date: Wed, 16 Oct 2019 01:57:59 -0400 Subject: [PATCH] * Created the 'oui' database table that stores the parsed OUI data, as processed by the new 'tools/striker-parse-oui' tool. Also created Database->insert_or_update_oui() to handle inserts and updates. * Fixed a bug in Convert->time() where the suffix was long when it should have been short, and vice-versa. * Updated Network->download() to check if the target file exists and, if so, to abort unless 'overwrite' is given or the existing file is 0-bytes long. Also updated it to not exit on immediate error after the wget call and instead check to see if a zero-byte file exists and remove it, if so. * Created Validate->is_hex() to check hexadecimal strings. * Updated Words->clean_spaces() to remove MS-DOS-style ^M cr/lf characters. * Updated anvil-daemon to have a section for periodic tasks that run daily, and added striker-parse-oui as well as moved striker-manage-install-target refresh to that check. Also made those tools run on dashboards only. Signed-off-by: Digimer --- Anvil/Tools.pm | 1 + Anvil/Tools/Convert.pm | 10 +- Anvil/Tools/Database.pm | 191 +++++++++++++++++++++++++++++++ Anvil/Tools/Network.pm | 117 +++++++++++++++---- Anvil/Tools/Validate.pm | 49 ++++++++ Anvil/Tools/Words.pm | 1 + share/anvil.sql | 22 ++-- share/words.xml | 8 ++ tools/anvil-daemon | 67 +++++++---- tools/striker-parse-oui | 244 ++++++++++++++++++++++++++++++++++++++++ 10 files changed, 653 insertions(+), 57 deletions(-) create mode 100755 tools/striker-parse-oui diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index e6ccf97f..05b09f5b 100644 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -1133,6 +1133,7 @@ sub _set_paths 'striker-initialize-host' => "/usr/sbin/striker-initialize-host", 'striker-manage-install-target' => "/usr/sbin/striker-manage-install-target", 'striker-manage-peers' => "/usr/sbin/striker-manage-peers", + 'striker-parse-oui' => "/usr/sbin/striker-parse-oui", 'striker-prep-database' => "/usr/sbin/striker-prep-database", stty => "/usr/bin/stty", su => "/usr/bin/su", diff --git a/Anvil/Tools/Convert.pm b/Anvil/Tools/Convert.pm index ef39f814..972c9b4e 100644 --- a/Anvil/Tools/Convert.pm +++ b/Anvil/Tools/Convert.pm @@ -1025,11 +1025,11 @@ sub time } # The suffix used for each unit of time will depend on the requested suffix type. - my $suffix_seconds = $long ? " #!string!suffix_0002!#" : " #!string!suffix_0007!#"; - my $suffix_minutes = $long ? " #!string!suffix_0003!#" : " #!string!suffix_0008!#"; - my $suffix_hours = $long ? " #!string!suffix_0004!#" : " #!string!suffix_0009!#"; - my $suffix_days = $long ? " #!string!suffix_0005!#" : " #!string!suffix_0010!#"; - my $suffix_weeks = $long ? " #!string!suffix_0006!#" : " #!string!suffix_0011!#"; + my $suffix_seconds = $long ? " #!string!suffix_0007!#" : " #!string!suffix_0002!#"; + my $suffix_minutes = $long ? " #!string!suffix_0008!#" : " #!string!suffix_0003!#"; + my $suffix_hours = $long ? " #!string!suffix_0009!#" : " #!string!suffix_0004!#"; + my $suffix_days = $long ? " #!string!suffix_0010!#" : " #!string!suffix_0005!#"; + my $suffix_weeks = $long ? " #!string!suffix_0011!#" : " #!string!suffix_0006!#"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { suffix_seconds => $suffix_seconds, suffix_minutes => $suffix_minutes, diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 25d34cab..931ff698 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -35,6 +35,7 @@ my $THIS_FILE = "Database.pm"; # insert_or_update_ip_addresses # insert_or_update_jobs # insert_or_update_network_interfaces +# insert_or_update_oui # insert_or_update_sessions # insert_or_update_states # insert_or_update_users @@ -4518,6 +4519,196 @@ INSERT INTO return($network_interface_uuid); } +=head2 insert_or_update_oui + +This updates (or inserts) a record in the C<< oui >> (Organizationally Unique Identifier) table used for converting network MAC addresses to the company that owns it. The C<< oui_uuid >> referencing the database row will be returned. + +If there is an error, an empty string is returned. + +B<< NOTE >>: This is one of the rare tables that doesn't have an owning host UUID. + +Parameters; + +=head3 oui_uuid (optional) + +If passed, the column with that specific C<< oui_uuid >> will be updated, if it exists. + +=head3 oui_mac_prefix (required) + +This is the first 6 bytes of the MAC address owned by C<< oui_company_name >>. + +=head3 oui_company_address (optional) + +This is the registered address of the company that owns the OUI. + +=head3 oui_company_name (required) + +This is the name of the company that owns the C<< oui_mac_prefix >>. + +=cut +sub insert_or_update_oui +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_oui()" }}); + + my $uuid = defined $parameter->{uuid} ? $parameter->{uuid} : ""; + my $file = defined $parameter->{file} ? $parameter->{file} : ""; + my $line = defined $parameter->{line} ? $parameter->{line} : ""; + my $oui_uuid = defined $parameter->{oui_uuid} ? $parameter->{oui_uuid} : ""; + my $oui_mac_prefix = defined $parameter->{oui_mac_prefix} ? $parameter->{oui_mac_prefix} : ""; + my $oui_company_address = defined $parameter->{oui_company_address} ? $parameter->{oui_company_address} : ""; + my $oui_company_name = defined $parameter->{oui_company_name} ? $parameter->{oui_company_name} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + uuid => $uuid, + file => $file, + line => $line, + oui_uuid => $oui_uuid, + oui_mac_prefix => $oui_mac_prefix, + oui_company_address => $oui_company_address, + oui_company_name => $oui_company_name, + }}); + + if (not $oui_mac_prefix) + { + # No user_uuid Throw an error and return. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_oui()", parameter => "oui_mac_prefix" }}); + return(""); + } + if (not $oui_company_name) + { + # No user_uuid Throw an error and return. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_oui()", parameter => "oui_company_name" }}); + return(""); + } + + # If the MAC isn't 6 or 8 bytes long (8 being xx:xx:xx), or isn't a valid hex string, abort. + if (((length($oui_mac_prefix) != 6) && (length($oui_mac_prefix) != 8)) or (not $anvil->Validate->is_hex({debug => $debug, string => $oui_mac_prefix, sloppy => 1}))) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0096", variables => { oui_mac_prefix => $oui_mac_prefix }}); + return(""); + } + + # If I don't have an oui_uuid, try to find one. + if (not $oui_uuid) + { + my $query = " +SELECT + oui_uuid +FROM + oui +WHERE + oui_mac_prefix = ".$anvil->Database->quote($oui_mac_prefix)." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + + my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + results => $results, + count => $count, + }}); + if ($count) + { + $oui_uuid = $results->[0]->[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { oui_uuid => $oui_uuid }}); + } + } + + # If I have an oui_uuid, see if an update is needed. If there still isn't an oui_uuid, INSERT it. + if ($oui_uuid) + { + # Load the old data and see if anything has changed. + my $query = " +SELECT + oui_mac_prefix, + oui_company_address, + oui_company_name +FROM + oui +WHERE + oui_uuid = ".$anvil->Database->quote($oui_uuid)." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + + my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + results => $results, + count => $count, + }}); + if (not $count) + { + # I have a oui_uuid but no matching record. Probably an error. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0216", variables => { uuid_name => "oui_uuid", uuid => $oui_uuid }}); + return(""); + } + foreach my $row (@{$results}) + { + my $old_oui_mac_prefix = $row->[0]; + my $old_oui_company_address = $row->[1]; + my $old_oui_company_name = $row->[2]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + old_oui_mac_prefix => $old_oui_mac_prefix, + old_oui_company_address => $old_oui_company_address, + oui_company_name => $old_oui_company_name, + }}); + + # Anything change? + if (($old_oui_mac_prefix ne $oui_mac_prefix) or + ($old_oui_company_address ne $oui_company_address) or + ($old_oui_company_name ne $oui_company_name)) + { + # Something changed, save. + my $query = " +UPDATE + oui +SET + oui_mac_prefix = ".$anvil->Database->quote($oui_mac_prefix).", + oui_company_address = ".$anvil->Database->quote($oui_company_address).", + oui_company_name = ".$anvil->Database->quote($oui_company_name).", + modified_date = ".$anvil->Database->quote($anvil->data->{sys}{database}{timestamp})." +WHERE + oui_uuid = ".$anvil->Database->quote($oui_uuid)." +"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + $anvil->Database->write({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); + } + } + } + else + { + # Save it. + $oui_uuid = $anvil->Get->uuid; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { oui_uuid => $oui_uuid }}); + + my $query = " +INSERT INTO + oui +( + oui_uuid, + oui_mac_prefix, + oui_company_address, + oui_company_name, + modified_date +) VALUES ( + ".$anvil->Database->quote($oui_uuid).", + ".$anvil->Database->quote($oui_mac_prefix).", + ".$anvil->Database->quote($oui_company_address).", + ".$anvil->Database->quote($oui_company_name).", + ".$anvil->Database->quote($anvil->data->{sys}{database}{timestamp})." +); +"; + $query =~ s/'NULL'/NULL/g; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + $anvil->Database->write({query => $query, uuid => $uuid, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); + } + + return($oui_uuid); +} + =head2 insert_or_update_sessions This updates (or inserts) a record in the 'sessions' table. The C<< session_uuid >> referencing the database row will be returned. diff --git a/Anvil/Tools/Network.pm b/Anvil/Tools/Network.pm index ce967afb..eb08fc5f 100755 --- a/Anvil/Tools/Network.pm +++ b/Anvil/Tools/Network.pm @@ -332,6 +332,12 @@ On success, the saved file is returned. On failure, an empty string is returned. Parameters; +=head3 overwrite (optional, default '0') + +When set, if the output file already exists, the existing file will be removed before the download is called. + +B<< NOTE >>: If the output file already exists and is 0-bytes, it is removed and the download proceeds regardless of this setting. + =head3 save_to (optional) If set, this is where the file will be downloaded to. If this ends with C<< / >>, the file name is preserved from the C<< url >> and will be saved in the C<< save_to >>'s directory with the original file name. Otherwise, the downlaoded file is saved with the file name given. As such, be careful about the trailing C<< / >>! @@ -355,15 +361,17 @@ sub download my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Network->get_ips()" }}); - my $save_to = defined $parameter->{save_to} ? $parameter->{save_to} : ""; - my $status = defined $parameter->{status} ? $parameter->{status} : 1; - my $url = defined $parameter->{url} ? $parameter->{url} : ""; - my $uuid = $anvil->Get->uuid(); + my $overwrite = defined $parameter->{overwrite} ? $parameter->{overwrite} : 0; + my $save_to = defined $parameter->{save_to} ? $parameter->{save_to} : ""; + my $status = defined $parameter->{status} ? $parameter->{status} : 1; + my $url = defined $parameter->{url} ? $parameter->{url} : ""; + my $uuid = $anvil->Get->uuid(); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - save_to => $save_to, - status => $status, - url => $url, - uuid => $uuid, + overwrite => $overwrite, + save_to => $save_to, + status => $status, + url => $url, + uuid => $uuid, }}); if (not $url) @@ -386,13 +394,35 @@ sub download { $save_to = $anvil->Get->users_home({debug => $debug})."/".$source_file; $save_to =~ s/\/\//\//g; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { save_to => $save_to }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 0, list => { save_to => $save_to }}); } elsif ($save_to =~ /\/$/) { $save_to .= "/".$source_file; $save_to =~ s/\/\//\//g; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { save_to => $save_to }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 0, list => { save_to => $save_to }}); + } + + # Does the download file exist already? + if (-e $save_to) + { + # If overwrite is set, or if the file is zero-bytes, remove it. + my $size = (stat($save_to))[7]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + size => $size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $size}).")", + }}); + if (($overwrite) or ($size == 0)) + { + unlink $save_to; + } + else + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "error_0094", variables => { + url => $url, + save_to => $save_to, + }}); + return(""); + } } ### TODO: Make this work well as a job @@ -407,6 +437,7 @@ sub download my $time_left = 0; # Seconds my $report_interval = 5; # Seconds between status file update my $next_report = time + $report_interval; + my $error = 0; # This should print to a status file print "uuid=$uuid bytes_downloaded=0 percent=0 current_rate=0 average_rate=0 seconds_running=0 seconds_left=0 url=$url save_to=$save_to\n" if $status;; @@ -426,28 +457,55 @@ sub download $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, secure => 0, key => "log_0017", variables => { line => $line }}); if (($line =~ /404/) && ($line =~ /Not Found/i)) { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0086", variables => { urk => $url }}); - return(""); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0086", variables => { url => $url }}); + $error = 1;; } if ($line =~ /Name or service not known/i) { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0087", variables => { urk => $url }}); - return(""); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0087", variables => { url => $url }}); + $error = 1;; } if ($line =~ /Connection refused/i) { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0088", variables => { urk => $url }}); - return(""); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0088", variables => { url => $url }}); + $error = 1;; } if ($line =~ /route to host/i) { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0089", variables => { urk => $url }}); - return(""); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0089", variables => { url => $url }}); + $error = 1;; } if ($line =~ /Network is unreachable/i) { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0090", variables => { urk => $url }}); - return(""); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0090", variables => { url => $url }}); + $error = 1;; + } + if ($line =~ /ERROR (\d+): (.*)$/i) + { + my $error_code = $1; + my $error_message = $2; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + error_code => $error_code, + error_message => $error_message, + }}); + + if ($error_code eq "403") + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0091", variables => { url => $url }}); + } + elsif ($error_code eq "404") + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0092", variables => { url => $url }}); + } + else + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0093", variables => { + url => $url, + error_code => $error_code, + error_message => $error_message, + }}); + } + $error = 1;; } if ($line =~ /^(\d+)K .*? (\d+)% (.*?) (\d+.*)$/) @@ -526,6 +584,25 @@ sub download close $file_handle; chomp($output); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { error => $error }}); + if ($error) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { save_to => $save_to }}); + if (-e $save_to) + { + # Unlink the output file, it's empty. + my $size = (stat($save_to))[7]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + size => $size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $size}).")", + }}); + if (not $size) + { + unlink $save_to; + } + } + return(""); + } + return($save_to); } diff --git a/Anvil/Tools/Validate.pm b/Anvil/Tools/Validate.pm index 0dffc9b2..9ec578d7 100644 --- a/Anvil/Tools/Validate.pm +++ b/Anvil/Tools/Validate.pm @@ -300,6 +300,55 @@ sub is_domain_name return($valid); } +=head2 is_hex + +Checks if the passed-in string contains only hexidecimal characters. A prefix of C<< 0x >> is allowed. + +Parameters; + +=head3 sloppy (optional, default '0') + +If set to C<< 1 >>, the string will be allowed to contain C<< : >> and C<< - >> and a closing C<< h >>characters (as found in MAC addresses, for example). + +=head3 string (required) + +This is the string to validate + +=cut +sub is_hex +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + + my $sloppy = defined $parameter->{sloppy} ? $parameter->{sloppy} : ""; + my $string = defined $parameter->{string} ? $parameter->{string} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + sloppy => $sloppy, + string => $string, + }}); + + if ($sloppy) + { + $string =~ s/-//g; + $string =~ s/://g; + $string =~ s/h$//gi; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { string => $string }}); + } + + my $valid = 1; + if ($string !~ /^(0[xX])*[0-9a-fA-F]+$/) + { + # There's something un-hexxy about this. + $valid = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { valid => $valid }}); + } + + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { valid => $valid }}); + return($valid); +} + =head2 is_ipv4 Checks if the passed-in string is an IPv4 address. Returns 'C<< 1 >>' if OK, 'C<< 0 >>' if not. diff --git a/Anvil/Tools/Words.pm b/Anvil/Tools/Words.pm index 5846a371..85ef2436 100644 --- a/Anvil/Tools/Words.pm +++ b/Anvil/Tools/Words.pm @@ -114,6 +114,7 @@ sub clean_spaces my $string = defined $parameter->{string} ? $parameter->{string} : ""; $string =~ s/^\s+//; $string =~ s/\s+$//; + $string =~ s/\r//g; $string =~ s/\s+/ /g; return($string); diff --git a/share/anvil.sql b/share/anvil.sql index 61e52b38..1c46479c 100644 --- a/share/anvil.sql +++ b/share/anvil.sql @@ -1445,19 +1445,21 @@ CREATE TRIGGER trigger_definitions -- owning company. Data for this comes from http://standards-oui.ieee.org/oui/oui.txt and is stored by -- striker-parse-oui. It is a generic reference table, so it's not bound to any one host. CREATE TABLE oui ( - oui_uuid uuid not null primary key, - oui_mac_prefix text not null, -- This is the first 12 bits / 3 bytes of the MAC address - oui_company_name text not null, -- This is the name of the owning company, as recorded in the OUI list. - modified_date timestamp with time zone not null + oui_uuid uuid not null primary key, + oui_mac_prefix text not null, -- This is the first 12 bits / 3 bytes of the MAC address + oui_company_name text not null, -- This is the name of the owning company, as recorded in the OUI list. + oui_company_address text not null, -- This is the company's registered address. + modified_date timestamp with time zone not null ); ALTER TABLE oui OWNER TO admin; CREATE TABLE history.oui ( - history_id bigserial, - oui_uuid uuid, - oui_mac_prefix text, - oui_company_name text, - modified_date timestamp with time zone not null + history_id bigserial, + oui_uuid uuid, + oui_mac_prefix text, + oui_company_name text, + oui_company_address text, + modified_date timestamp with time zone not null ); ALTER TABLE history.oui OWNER TO admin; @@ -1471,11 +1473,13 @@ BEGIN (oui_uuid, oui_mac_prefix, oui_company_name, + oui_company_address, modified_date) VALUES (history_oui.oui_uuid, history_oui.oui_mac_prefix, history_oui.oui_company_name, + history_oui.oui_company_address, history_oui.modified_date); RETURN NULL; END; diff --git a/share/words.xml b/share/words.xml index 104721d7..d48ef321 100644 --- a/share/words.xml +++ b/share/words.xml @@ -1040,6 +1040,8 @@ Failure! The return code: [#!variable!return_code!#] was received ('0' was expec Setting the host name to: [#!variable!host_name!#]... [ Error ] - The host name: [#!variable!host_name!#] is invalid. Skipping host name setup. [ Error ] - Something went wrong. The host name was set to: [#!variable!host_name!#], but the host name returned was: [#!variable!current_host_name!#]. + OUI Database. + Refresh the 'OUI' database used to cross reference MAC addresses to the companies that own them. The IP address will change. You will need to reconnect after applying these changes. @@ -1162,6 +1164,12 @@ Failed to generate an RSA public key for the user: [#!variable!user!#]. The outp The requested URL: [#!variable!url!#] failed because the remote host refused the connection. The requested URL: [#!variable!url!#] failed because there is no route to that host. The requested URL: [#!variable!url!#] failed because the network is unreachable. + The requested URL: [#!variable!url!#] failed, access was forbidden (error 403). + The requested URL: [#!variable!url!#] failed, the file was not found on the source (error 404). + The requested URL: [#!variable!url!#] failed with HTTP error: [#!variable!error_code!#] (message: [#!variable!error_message!#]). + Aborting the download of: [#!variable!url!#] to: [#!variable!save_to!#]. The target file already exists and 'overwrite' was not set. + There was a problem downloading: [#!variable!url!#] to: [#!variable!file!#]. Aborting parsing of the OUI data. + The 'oui_mac_prefix': [#!variable!oui_mac_prefix!#] string doesn't appear to be a valid 6-byte hex string. Yes diff --git a/tools/anvil-daemon b/tools/anvil-daemon index 52fe6889..a9874370 100755 --- a/tools/anvil-daemon +++ b/tools/anvil-daemon @@ -149,15 +149,17 @@ my $now_time = time; # Once a minute, we'll check the md5sums and see if we should restart. # Once a day, we'll refresh an Install Target's RPM repository (has no effect on non-Striker dashboards). $anvil->data->{timing}{minute_checks} = 60; +$anvil->data->{timing}{daily_checks} = 86400; $anvil->data->{timing}{repo_update_interval} = 86400; $anvil->data->{timing}{next_minute_check} = $now_time - 1; -$anvil->data->{timing}{next_repo_check} = $now_time; # We want to run on daemon startup +$anvil->data->{timing}{next_daily_check} = $now_time - 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "s1:timing::minute_checks" => $anvil->data->{timing}{minute_checks}, + "s1:timing::daily_checks" => $anvil->data->{timing}{daily_checks}, "s2:timing::repo_update_interval" => $anvil->data->{timing}{repo_update_interval}, "s3:now_time" => $now_time, "s4:timing::next_minute_check" => $anvil->data->{timing}{next_minute_check}, - "s5:timing::next_repo_check" => $anvil->data->{timing}{next_repo_check}, + "s4:timing::next_daily_check" => $anvil->data->{timing}{next_daily_check}, }}); # When we periodically check if system files have changed, we'll also ask Database>connect() to check if it @@ -217,7 +219,7 @@ sub handle_periodic_tasks $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "s1:now_time" => $now_time, "s2:timing::next_minute_check" => $anvil->data->{timing}{next_minute_check}, - "s3:timing::next_repo_check" => $anvil->data->{timing}{next_repo_check}, + "s3:timing::next_daily_check" => $anvil->data->{timing}{next_daily_check}, }}); # Time to run once per minute tasks. @@ -286,30 +288,49 @@ sub handle_periodic_tasks } } - ### NOTE: We call it once/day, but this will also trigger on restart of anvil-daemon. As such, we - ### don't use '--force' and let striker-manage-install-target skip the repo update if it happened - ### recently enough. - # Is it time to refresh RPM packages on Install Target hosts? - if ($now_time >= $anvil->data->{timing}{next_repo_check}) + # Now check to see if it's time to run daily tasks. + if ($now_time >= $anvil->data->{timing}{next_daily_check}) { - # Record a job, don't call it directly. It takes too long to run. - my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ - file => $THIS_FILE, - line => __LINE__, - job_command => $anvil->data->{path}{exe}{'striker-manage-install-target'}." --refresh", - job_data => "", - job_name => "install-target::refresh", - job_title => "job_0015", - job_description => "job_0017", - job_progress => 0, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { job_uuid => $job_uuid }}); + ### NOTE: We call it once/day, but this will also trigger on restart of anvil-daemon. As such, we + ### don't use '--force' and let striker-manage-install-target skip the repo update if it happened + ### recently enough. + my $type = $anvil->System->get_host_type(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { type => $type }}); + + if ($type eq "dashboard") + { + # Record a job, don't call it directly. It takes too long to run. + my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ + file => $THIS_FILE, + line => __LINE__, + job_command => $anvil->data->{path}{exe}{'striker-manage-install-target'}." --refresh", + job_data => "", + job_name => "install-target::refresh", + job_title => "job_0015", + job_description => "job_0017", + job_progress => 0, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { job_uuid => $job_uuid }}); + + # Update the OUI data. + ($job_uuid) = $anvil->Database->insert_or_update_jobs({ + file => $THIS_FILE, + line => __LINE__, + job_command => $anvil->data->{path}{exe}{'striker-parse-oui'}, + job_data => "", + job_name => "oui-data::refresh", + job_title => "job_0064", + job_description => "job_0065", + job_progress => 0, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { job_uuid => $job_uuid }}); + } # Update the next check time. - $anvil->data->{timing}{next_repo_check} = $now_time + $anvil->data->{timing}{repo_update_interval}; + $anvil->data->{timing}{next_daily_check} = $now_time + $anvil->data->{timing}{daily_checks}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - "s1:timing::repo_update_interval" => $anvil->data->{timing}{repo_update_interval}, - "s2:timing::next_repo_check" => $anvil->data->{timing}{next_repo_check}, + "s1:timing::daily_checks" => $anvil->data->{timing}{daily_checks}, + "s2:timing::next_daily_check" => $anvil->data->{timing}{next_daily_check}, }}); } diff --git a/tools/striker-parse-oui b/tools/striker-parse-oui new file mode 100755 index 00000000..a1240df4 --- /dev/null +++ b/tools/striker-parse-oui @@ -0,0 +1,244 @@ +#!/usr/bin/perl +# +# This periodically reads in http://standards-oui.ieee.org/oui/oui.txt, if possible, and parses it to update/ +# populate the oui database table. +# +use strict; +use warnings; +use Anvil::Tools; +use Data::Dumper; + +# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete. +$| = 1; + +my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; +my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; +if (($running_directory =~ /^\./) && ($ENV{PWD})) +{ + $running_directory =~ s/^\./$ENV{PWD}/; +} + +my $anvil = Anvil::Tools->new(); +$anvil->Log->level({set => 2}); +$anvil->Log->secure({set => 0}); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }}); + +my $oui_file = $anvil->Get->users_home({debug => 3})."/oui.txt"; +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { oui_file => $oui_file }}); + +my $download = 1; +my $process = 0; +if (-e $oui_file) +{ + # How long ago did we download it? + my $mtime = (stat($oui_file))[9]; + my $size = (stat($oui_file))[7]; + my $age = time - $mtime; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:oui_file' => $oui_file, + 's2:mtime' => $mtime." (".$anvil->Get->date_and_time({use_time => $mtime}).")", + 's3:age' => $anvil->Convert->add_commas({number => $age})." (".$anvil->Convert->time({'time' => $age, translate => 1}).")", + 's4:size' => $anvil->Convert->add_commas({number => $size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $size}).")", + }}); + if (($age < 259200) && ($size > 0)) + { + # It's less than three days old, don't download. Do parse though (for now at least) + $download = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { download => $download }}); + } + if ((not $download) && ($size > 0)) + { + $process = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { process => $process }}); + } +} + +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { download => $download }}); +if ($download) +{ + my $download_file = $anvil->Network->download({ + debug => 2, + url => $anvil->data->{path}{urls}{oui_file}, + save_to => $oui_file, + overwrite => 1, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { download_file => $download_file }}); + + if (($download_file) && ($download_file eq $oui_file)) + { + $process = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { process => $process }}); + } + else + { + # Something went wrong. Even if the file exists, there's no sense processing it. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "error_0095", variables => { + url => $anvil->data->{path}{urls}{oui_file}, + file => $oui_file, + }}); + } +} + +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + oui_file => $oui_file, + process => $process, +}}); +if ((-e $oui_file) && ($process)) +{ + $anvil->Database->connect; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); + if (not $anvil->data->{sys}{database}{connections}) + { + # No databases, exit. + print $anvil->Words->string({key => "error_0003"})."\n"; + $anvil->nice_exit({exit_code => 2}); + } + + process_oui($anvil, $oui_file); +} + +$anvil->nice_exit({exit_code => 0}); + + +############################################################################################################# +# Functions # +############################################################################################################# + +# This actually processes the OUI file. +sub process_oui +{ + my ($anvil, $oui_file) = @_; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { oui_file => $oui_file }}); + + # Read in the file. + my $oui = ""; + my $vendor = ""; + my $address = ""; + my $lines = 0; + my ($oui_body) = $anvil->Storage->read_file({ + debug => 3, + file => $oui_file, + cache => 0, + force_read => 1 + }); + + ### TODO: For some reason, ending the file on an empty line wasn't triggering the save of the last + ### record. So the EOF line/check was added, at least until I can get undumb enough to see what + ### the real problem is. + # The OUI list doesn't include an entry for Red Hat / 52:54:00. So we'll inject it here. + $oui_body .= " +52-54-00 (hex) Red Hat, Inc. +525400 (base 16) Red Hat, Inc. + 100 East Davie Street + Raleigh NC 27601 + US + +EOF +"; + + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { oui_body => $oui_body }}); + foreach my $line (split/\n/, $oui_body) + { + $lines++; + $line = $anvil->Words->clean_spaces({'string' => $line}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 's1:lines' => $lines, 's2:line' => $line }}); + + if ((not $line) or ($line eq "EOF")) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { oui => $oui }}); + if ($oui) + { + $address =~ s/, $//; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + oui => $oui, + vendor => $vendor, + address => $address, + }}); + if (not $address) + { + # This isn't translated + $address = ""; + } + + # NOTE: There are duplicates in the OUI file, so we'll string te entries together + if ((exists $anvil->data->{oui}{$oui}) && ($anvil->data->{oui}{$oui}{name})) + { + $anvil->data->{oui}{$oui}{name} .= " / ".$vendor; + $anvil->data->{oui}{$oui}{address} .= " / ". $address; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + "oui::${oui}::name" => $anvil->data->{oui}{$oui}{name}, + "oui::${oui}::address" => $anvil->data->{oui}{$oui}{address}, + }}); + } + else + { + $anvil->data->{oui}{$oui}{name} = $vendor; + $anvil->data->{oui}{$oui}{address} = $address; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + "oui::${oui}::name" => $anvil->data->{oui}{$oui}{name}, + "oui::${oui}::address" => $anvil->data->{oui}{$oui}{address}, + }}); + } + } + $oui = ""; + $vendor = ""; + $address = ""; + next; + } + $line =~ s/^\s+//; + $line =~ s/\s+$//; + if ($line =~ /^(\w\w-\w\w-\w\w)\s+\(hex\)\s+(.*)/) + { + $oui = $1; + $vendor = $2; + $oui =~ s/-/:/g; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + oui => $oui, + vendor => $vendor, + }}); + next; + } + next if not $oui; + if ($line =~ /^(\w\w\w\w\w\w)\s+\(base 16\)\s+(.*)/) + { + my $oui2 = $1; + my $vendor2 = $2; + $oui2 =~ s/-/:/g; + $oui2 =~ s/^(\w\w)(\w\w)(\w\w)$/$1:$2:$3/g; + $oui = $oui2 if not $oui; + $vendor = $vendor2 if not $vendor; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + oui => $oui, + oui2 => $oui2, + vendor => $vendor, + vendor2 => $vendor2, + }}); + next; + } + else + { + $address .= "$line, "; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { address => $address }}); + } + } + + # Record the details. + foreach my $oui (sort {$a cmp $b} keys %{$anvil->data->{oui}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + "oui::${oui}::name" => $anvil->data->{oui}{$oui}{name}, + "oui::${oui}::address" => $anvil->data->{oui}{$oui}{address}, + }}); + my ($oui_uuid) = $anvil->Database->insert_or_update_oui({ + debug => 3, + file => $THIS_FILE, + line => __LINE__, + oui_mac_prefix => $oui, + oui_company_address => $anvil->data->{oui}{$oui}{address}, + oui_company_name => $anvil->data->{oui}{$oui}{name}, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { oui_uuid => $oui_uuid }}); + } + + return(0); +}