diff --git a/Anvil/Tools/Alert.pm b/Anvil/Tools/Alert.pm index ff46e3fd..d1053652 100644 --- a/Anvil/Tools/Alert.pm +++ b/Anvil/Tools/Alert.pm @@ -77,16 +77,14 @@ sub parent =head2 check_alert_sent +This method is used to see if an event that might last some time has had an alert send already to recipients. + This is used by programs, usually scancore scan agents, that need to track whether an alert was sent when a sensor dropped below/rose above a set alert threshold. For example, if a sensor alerts at 20°C and clears at 25°C, this will be called when either value is passed. When passing the warning threshold, the alert is registered and sent to the user. Once set, no further warning alerts are sent. When the value passes over the clear threshold, this is checked and if an alert was previously registered, it is removed and an "all clear" message is sent. In this way, multiple alerts will not go out if a sensor floats around the warning threshold and a "cleared" message won't be sent unless a "warning" message was previously sent. If there is a problem, C<< !!error!! >> is returned. Parameters; -=head3 modified_date (optional) - -By default, this is set to C<< sys::database::timestamp >>. If you want to force a different timestamp, you can do so with this parameter. - =head3 name (required) This is the name of the alert. So for an alert related to a critically high temperature, this might get set to C<< temperature_high_critical >>. It is meant to compliment the C<< record_locator >> parameter. @@ -116,27 +114,17 @@ sub check_alert_sent my $anvil = $self->parent; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Alert->check_alert_sent()" }}); - my $modified_date = defined $parameter->{modified_date} ? $parameter->{modified_date} : $anvil->data->{sys}{database}{timestamp}; my $name = defined $parameter->{name} ? $parameter->{name} : ""; my $record_locator = defined $parameter->{record_locator} ? $parameter->{record_locator} : ""; my $set_by = defined $parameter->{set_by} ? $parameter->{set_by} : ""; my $type = defined $parameter->{type} ? $parameter->{type} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - modified_date => $modified_date, name => $name, record_locator => $record_locator, set_by => $set_by, type => $type, }}); - # Do we have a timestamp? - if (not $modified_date) - { - # Nope - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0093"}); - return("!!error!!"); - } - # Do we have an alert name? if (not $name) { @@ -170,7 +158,7 @@ sub check_alert_sent } # This will get set to '1' if an alert is added or removed. - my $set = 0; + my $changed = 0; my $query = " SELECT @@ -225,7 +213,6 @@ WHERE set_by => $set_by, record_locator => $record_locator, name => $name, - modified_date => $modified_date, }}); return("!!error!!"); } @@ -236,8 +223,8 @@ WHERE } } - $set = 1; - my $query = " + $changed = 1; + my $query = " INSERT INTO alert_sent ( @@ -257,30 +244,30 @@ INSERT INTO ); "; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - query => $query, - set => $set, + query => $query, + changed => $changed, }}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); } elsif (($type eq "clear") && ($alert_sent_uuid)) { # Alert previously existed, clear it. - $set = 1; - my $query = " + $changed = 1; + my $query = " DELETE FROM alert_sent WHERE alert_sent_uuid = ".$anvil->Database->quote($alert_sent_uuid)." ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - query => $query, - set => $set, + query => $query, + changed => $changed, }}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); } - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { set => $set }}); - return($set); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { changed => $changed }}); + return($changed); } =head2 register diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 18fefbf8..875d3797 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -17,6 +17,7 @@ my $THIS_FILE = "Database.pm"; ### Methods; # archive_database # check_lock_age +# check_for_schema # configure_pgsql # connect # disconnect @@ -358,7 +359,7 @@ sub check_lock_age This reads in a SQL schema file and checks if the first table seen exists in the database. If it isn't, the schema file is loaded into the database main. -If the table exists (and loading isn't needed), C<< 0 >> is returned. If the schema is loaded, C<< 1 >> is returned. If there is any problem, C<< !!error!! >> is returned. +If the table exists (and loading isn't needed), C<< 0 >> is returned. If the schema is loaded, an array reference of the host UUIDs that were loaded is returned. If there is any problem, C<< !!error!! >> is returned. B<< Note >>: This does not check for schema changes! @@ -385,14 +386,93 @@ sub check_for_schema file => $file, }}); + # We only test that a file was passed in. Storage->read will catch errors with the file not existing, + # permission issues, etc. if (not $file) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->check_for_schema()", parameter => "file" }}); return("!!error!!"); } - my $body = $anvil->Storage->read_file({file => $file}); + my $table = ""; + my $schema = "public"; + my $body = $anvil->Storage->read_file({file => $file}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { body => $body }}); + foreach my $line (split/\n/, $body) + { + $line =~ s/--.*$//; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }}); + + if ($line =~ /CREATE TABLE (.*?) \(/i) + { + $table = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { table => $table }}); + + if ($table =~ /^(.*?)\.(.*)$/) + { + $schema = $1; + $table = $2; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + table => $table, + schema => $schema, + }}); + } + last; + } + } + + # Did we find a table? + if (not $table) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0050"}); + return("!!error!!"); + } + + my $query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename=".$anvil->Database->quote($table)." AND schemaname=".$anvil->Database->quote($schema).";"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + + # We have to query each DB individually. + foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}}) + { + my $host_name = $anvil->Database->get_host_from_uuid({debug => $debug, short => 1, host_uuid => $uuid}); + my $count = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:count' => $count, + 's2:host_name' => $host_name, + }}); + + if ($count) + { + # No need to add. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0544", variables => { + table => $schema.".".$table, + host => $host_name, + }}); + } + else + { + # Load the schema. + if ($loaded eq "0") + { + $loaded = []; + } + push @{$loaded}, $uuid; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0545", variables => { + table => $schema.".".$table, + host => $host_name, + file => $file, + }}); + + # Write out the schema now. + $anvil->Database->write({ + uuid => $uuid, + transaction => 1, + query => $body, + source => $THIS_FILE, + line => __LINE__, + }); + } + } return($loaded); } @@ -11445,7 +11525,39 @@ sub resync_databases =head2 write -This records data to one or all of the databases. If an ID is passed, the query is written to one database only. Otherwise, it will be written to all DBs. +This records data to one or all of the databases. If a UUID is passed, the query is written to one database only. Otherwise, it will be written to all DBs. + +Parameters; + +=head3 line (optional) + +If you want errors to be traced back to the query called, this can be set (usually to C<< __LINE__ >>) along with the C<< source >> parameter. In such a case, if there is an error in this method, the caller's file and line are displayed in the logs. + +=head3 transaction (optional, default 0) + +Normally, if C<< query >> is an array reference, a C<< BEGIN TRANSACTION; >> is called before the queries are written, and closed off with a C<< COMMIT; >>. In this way, either all queries succeed or none do. In some cases, like loading a schema, multiple queries are passed as a single line. In these cases, you can set this to C<< 1 >> to wrap the query in a transaction block. + +=head3 query (required) + +This is the query or queries to be written. In string context, the query is directly passed to the database handle(s). In array reference context, the queries are wrapped in a transaction block (see tjhe 'transaction' parameter). + +B<< Note >>: If the number of queries are in the array reference is greater than C<< sys::database::maximum_batch_size >>, the queries are "chunked" into smaller transaction blocks. This is done so that very large arrays don't take so long that locks time out or memory becomes an issue. + +=head3 reenter (optional) + +This is used internally to indicate when a very large query array has been broken up and we've re-entered this method to process component chunks. The main effect is that some checks this method performs are skipped. + +=head3 secure (optional, default 0) + +If the query contains sensitive information, like passwords, setting this will ensure that log entries will be appropriately surpressed unless secure logging is enabled. + +=head3 source (optional) + +If you want errors to be traced back to the query called, this can be set (usually to C<< $THIS_FILE >>) along with the C<< line >> parameter. In such a case, if there is an error in this method, the caller's file and line are displayed in the logs. + +=head3 uuid (optional) + +By default, queries go to all connected databases. If a given write should go to only one database, set this to the C<< host_uuid >> of the dataabase host. This is generally only used internally during resync operations. =cut sub write @@ -11456,12 +11568,13 @@ sub write my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->write()" }}); - my $uuid = $parameter->{uuid} ? $parameter->{uuid} : ""; - my $line = $parameter->{line} ? $parameter->{line} : __LINE__; - my $query = $parameter->{query} ? $parameter->{query} : ""; - my $secure = $parameter->{secure} ? $parameter->{secure} : 0; - my $source = $parameter->{source} ? $parameter->{source} : $THIS_FILE; - my $reenter = $parameter->{reenter} ? $parameter->{reenter} : ""; + my $line = $parameter->{line} ? $parameter->{line} : __LINE__; + my $query = $parameter->{query} ? $parameter->{query} : ""; + my $reenter = $parameter->{reenter} ? $parameter->{reenter} : ""; + my $secure = $parameter->{secure} ? $parameter->{secure} : 0; + my $source = $parameter->{source} ? $parameter->{source} : $THIS_FILE; + my $transaction = $parameter->{transaction} ? $parameter->{transaction} : 0; + my $uuid = $parameter->{uuid} ? $parameter->{uuid} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid, line => $line, @@ -11612,7 +11725,7 @@ sub write uuid => $uuid, count => $count, }}); - if ($count) + if (($count) or ($transaction)) { # More than one query, so start a transaction block. $anvil->data->{cache}{database_handle}{$uuid}->begin_work; @@ -11643,7 +11756,7 @@ sub write } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); - if ($count) + if (($count) or ($transaction)) { # Commit the changes. $anvil->data->{cache}{database_handle}{$uuid}->commit(); diff --git a/scancore-agents/scan-hardware/scan-hardware b/scancore-agents/scan-hardware/scan-hardware index c6b65a68..1077b5d6 100755 --- a/scancore-agents/scan-hardware/scan-hardware +++ b/scancore-agents/scan-hardware/scan-hardware @@ -27,6 +27,8 @@ if (($running_directory =~ /^\./) && ($ENV{PWD})) } my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1}); +$anvil->Log->level({set => 2}); +$anvil->Log->secure({set => 1}); $anvil->Storage->read_config(); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0115", variables => { program => $THIS_FILE }}); @@ -34,8 +36,18 @@ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level # Read switches $anvil->Get->switches; +# Connect to DBs. +$anvil->Database->connect; +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); +if (not $anvil->data->{sys}{database}{connections}) +{ + # No databases, exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, secure => 0, key => "error_0003"}); + $anvil->nice_exit({exit_code => 1}); +} + # Make sure our schema is loaded. -$anvil->Database->check_for_schema({file => $anvil->data->{path}{directories}{scan_agents}."/".$THIS_FILE."/".$THIS_FILE.".sql"}); +check_database($anvil); @@ -44,3 +56,87 @@ $anvil->nice_exit({exit_code => 0}); ############################################################################################################# # Functions # ############################################################################################################# + +sub check_database +{ + my ($anvil) = @_; + + my $schema_file = $anvil->data->{path}{directories}{scan_agents}."/".$THIS_FILE."/".$THIS_FILE.".sql"; + my $loaded = $anvil->Database->check_for_schema({ + debug => 2, + file => $schema_file, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + loaded => $loaded, + schema_file => $schema_file, + }}); + if ($loaded) + { + if ($loaded eq "!!error!!") + { + # Something went wrong. + my $changed = $anvil->Alert->check_alert_sent({ + debug => 2, + record_locator => "schema_load_failure", + set_by => $THIS_FILE, + type => "set", + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { changed => $changed }}); + if ($changed) + { + # Log and register an alert. This should never happen, so we set it as a + # warning level alert. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0181", variables => { + agent_name => $THIS_FILE, + file => $schema_file, + }}); + $anvil->Alert->register({ + debug => 2, + alert_level => "warning", + message => "message_0181,!!agent_name!".$THIS_FILE."!!,!!file!".$schema_file."!!", + set_by => $THIS_FILE, + }); + } + } + elsif (ref($loaded) eq "ARRAY") + { + # If there was an alert, clear it. + my $changed = $anvil->Alert->check_alert_sent({ + debug => 2, + record_locator => "schema_load_failure", + set_by => $THIS_FILE, + type => "clear", + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { changed => $changed }}); + if ($changed) + { + # Register an alert cleared message. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0182", variables => { + agent_name => $THIS_FILE, + file => $schema_file, + + }}); + $anvil->Alert->register({ + debug => 2, + alert_level => "warning", + clear_alert => 1, + message => "message_0182,!!agent_name!".$THIS_FILE."!!,!!file!".$schema_file."!!", + set_by => $THIS_FILE, + }); + } + + # Log which databses we loaded our schema into. + foreach my $uuid (@{$loaded}) + { + my $host_name = $anvil->Database->get_host_from_uuid({short => 1, host_uuid => $uuid}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "message_0183", variables => { + agent_name => $THIS_FILE, + host_name => $host_name, + + }}); + } + } + } + + return(0); +} diff --git a/share/words.xml b/share/words.xml index d57ba1de..4b1d8f13 100644 --- a/share/words.xml +++ b/share/words.xml @@ -1028,7 +1028,9 @@ The file: [#!variable!file!#] needs to be updated. The difference is: This host UUID is: [#!variable!uuid!#] and the database identifier is: [#!variable!identifier!#]. Writing out alert email to: [#!variable!file!#]. Sending email to: [#!variable!to!#]. - I was asked to process alerts, but there are no configured email servers. No sense proceeding.. + I was asked to process alerts, but there are no configured email servers. No sense proceeding. + The table: [#!variable!table!#] already exists in the database on the host: [#!variable!host!#], no need to load the schema. + The table: [#!variable!table!#] does NOT exists in the database on the host: [#!variable!host!#]. Will load the schema file: [#!variable!file!#] now. The host name: [#!variable!target!#] does not resolve to an IP address. @@ -1300,6 +1302,9 @@ About to try to download aproximately: [#!variable!packages!#] packages needed t Hosts added or updated by the #!string!brand_0002!# on: [#!variable!date!#]: ScanCore has started. The scan agent: [#!variable!agent_name!#] timed out! It was given: [#!variable!timeout!#] seconds to run, but it didn't return, so it was terminated. + The scan agent: [#!variable!agent_name!#] check if it's schema was loaded! This is likely a problem with the SQL schema in the file: [#!variable!file!#]. Details are likely available in the: [#!data!path::log::main!#] log file. + The scan agent: [#!variable!agent_name!#] has now successfully loaded! Whatever issue existed with: [#!variable!file!#] has been resolved. + The SQL schema for the scan agent: [#!variable!agent_name!#] has been loaded into the database host: [#!variable!host_name!#]. Saved the mail server information successfully!