From a932b44d284e3988cdde70c917856e1ee835d498 Mon Sep 17 00:00:00 2001 From: Digimer Date: Fri, 28 Jul 2017 00:39:53 -0400 Subject: [PATCH] * Fixed a few more problems with the SQL schema and setting up the database when needed. * Fixed more typo-related bugs. * Created System->enable_daemon(). * Changed the 'database::general' to 'sys::database' to avoid accidental collisions with database IDs. Signed-off-by: Digimer --- AN/Tools.sql | 25 +++++++---- AN/Tools/Alert.pm | 4 +- AN/Tools/Database.pm | 105 ++++++++++++++++++++++--------------------- AN/Tools/Log.pm | 9 ++++ AN/Tools/System.pm | 41 +++++++++++++++++ striker.conf | 2 +- 6 files changed, 123 insertions(+), 63 deletions(-) diff --git a/AN/Tools.sql b/AN/Tools.sql index f6b0b756..27316c9d 100644 --- a/AN/Tools.sql +++ b/AN/Tools.sql @@ -2,7 +2,18 @@ -- It expects PostgreSQL v. 9.1+ SET client_encoding = 'UTF8'; -CREATE SCHEMA IF NOT EXISTS history; +-- This doesn't work before 9.3 - CREATE SCHEMA IF NOT EXISTS history; +-- So we'll use the query below until (if) we upgrade. +DO $$ +BEGIN + IF NOT EXISTS( + SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'history' + ) + THEN + EXECUTE 'CREATE SCHEMA history'; + END IF; +END +$$; -- This stores information about the host machine. This is the master table that everything will be linked -- to. @@ -10,9 +21,7 @@ CREATE TABLE hosts ( host_uuid uuid not null primary key, -- This is the single most important record in ScanCore. Everything links back to here. host_name text not null, host_type text not null, -- Either 'node' or 'dashboard'. - modified_date timestamp with time zone not null, - - FOREIGN KEY(host_location_uuid) REFERENCES locations(location_uuid) + modified_date timestamp with time zone not null ); ALTER TABLE hosts OWNER TO #!variable!user!#; @@ -58,9 +67,7 @@ CREATE TABLE host_variable ( host_variable_host_uuid uuid not null, host_variable_name text not null, host_variable_value text, - modified_date timestamp with time zone not null, - - FOREIGN KEY(host_location_uuid) REFERENCES locations(location_uuid) + modified_date timestamp with time zone not null ); ALTER TABLE host_variable OWNER TO #!variable!user!#; @@ -110,7 +117,7 @@ CREATE TABLE alerts ( alert_level text not null, -- debug (log only), info (+ admin email), notice (+ curious users), warning (+ client technical staff), critical (+ all) alert_title_key text not null, -- ScanCore will read in the agents .xml words file and look for this message key alert_title_variables text, -- List of variables to substitute into the message key. Format is 'var1=val1 #!# var2 #!# val2 #!# ... #!# varN=valN'. - alert_message_key text not null -- ScanCore will read in the agents .xml words file and look for this message key + alert_message_key text not null, -- ScanCore will read in the agents .xml words file and look for this message key alert_message_variables text, -- List of variables to substitute into the message key. Format is 'var1=val1 #!# var2 #!# val2 #!# ... #!# varN=valN'. alert_sort text, -- The alerts will sort on this column. It allows for an optional sorting of the messages in the alert. alert_header boolean not null default TRUE, -- This can be set to have the alert be printed with only the contents of the string, no headers. @@ -264,7 +271,7 @@ ALTER TABLE updated OWNER TO #!variable!user!#; CREATE TABLE alert_sent ( alert_sent_host_uuid uuid not null, -- The node associated with this alert alert_set_by text not null, -- name of the program that set this alert - alert_record_locator text, not null -- String used by the agent to identify the source of the alert (ie: UPS serial number) + alert_record_locator text not null, -- String used by the agent to identify the source of the alert (ie: UPS serial number) alert_name text not null, -- A free-form name used by the caller to identify this alert. modified_date timestamp with time zone not null, diff --git a/AN/Tools/Alert.pm b/AN/Tools/Alert.pm index e6210a55..7d96fc55 100755 --- a/AN/Tools/Alert.pm +++ b/AN/Tools/Alert.pm @@ -179,7 +179,7 @@ AND ;"; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); - my $count = $an->DB->do_db_query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $count = $an->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { type => $type, query => $query, @@ -205,7 +205,7 @@ WHERE ;"; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); - my $count = $an->DB->do_db_query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $count = $an->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); if (not $count) diff --git a/AN/Tools/Database.pm b/AN/Tools/Database.pm index 55ca518d..32b641ff 100755 --- a/AN/Tools/Database.pm +++ b/AN/Tools/Database.pm @@ -145,6 +145,10 @@ sub configure_pgsql { # Initialized! $an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0055"}); + + # Enable it on boot. + my $return_code = $an->System->enable_daemon({daemon => "postgresql"}); + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }}); } } } @@ -521,6 +525,8 @@ sub connect test_table => $test_table, }}); + $an->data->{sys}{db_timestamp} = "" if not defined $an->data->{sys}{db_timestamp}; + # We need the host_uuid before we connect. if (not $an->data->{sys}{host_uuid}) { @@ -533,7 +539,7 @@ sub connect $an->data->{sys}{local_db_id} = ""; # This will be set to '1' if either DB needs to be initialized or if the last_updated differs on any node. - $an->data->{database}{general}{resync_needed} = 0; + $an->data->{sys}{database}{resync_needed} = 0; # Now setup or however-many connections my $seen_connections = []; @@ -542,7 +548,6 @@ sub connect my $successful_connections = []; foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}}) { - next if $id eq "general"; # This is used for global values. my $driver = "DBI:Pg"; my $host = $an->data->{database}{$id}{host} ? $an->data->{database}{$id}{host} : ""; # This should fail my $port = $an->data->{database}{$id}{port} ? $an->data->{database}{$id}{port} : 5432; @@ -750,7 +755,7 @@ sub connect # Get a time stamp for this run, if not yet gotten. $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "cache::db_fh::$id" => $an->data->{cache}{db_fh}{$id}, - "sys::db_timestamp" => $an->data->{sys}{db_timestamp} + "sys::db_timestamp" => $an->data->{sys}{db_timestamp}, }}); # Pick a timestamp for this run, if we haven't yet. @@ -795,11 +800,11 @@ sub connect # If I've not sent an alert about this DB loss before, send one now. my $set = $an->Alert->check_alert_sent({ - type => "set", - alert_set_by => $THIS_FILE, - alert_record_locator => $id, - alert_name => "connect_to_db", - modified_date => $an->data->{sys}{db_timestamp}, + type => "set", + set_by => $THIS_FILE, + record_locator => $id, + name => "connect_to_db", + modified_date => $an->data->{sys}{db_timestamp}, }); $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { set => $set }}); @@ -842,11 +847,11 @@ sub connect if ($count > 0) { my $cleared = $an->Alert->check_alert_sent({ - type => "clear", - alert_sent_by => $THIS_FILE, - alert_record_locator => $id, - alert_name => "connect_to_db", - modified_date => $an->data->{sys}{db_timestamp}, + type => "clear", + sent_by => $THIS_FILE, + record_locator => $id, + name => "connect_to_db", + modified_date => $an->data->{sys}{db_timestamp}, }); if ($cleared) { @@ -878,9 +883,9 @@ sub connect # For now, we just find which DBs are behind and let each agent deal with bringing their tables up to # date. - $an->database->_find_behind_databases({ + $an->Database->_find_behind_databases({ source => $source, - tables => $$tables, + tables => $tables, }); # Hold if a lock has been requested. @@ -949,7 +954,6 @@ sub get_local_id my $network_details = $an->Get->network_details; foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}}) { - next if $id eq "general"; # This is used for global values. if ($network_details->{hostname} eq $an->data->{database}{$id}{host}) { $local_id = $id; @@ -964,7 +968,6 @@ sub get_local_id my $subnet_mask = $network_details->{interface}{$interface}{netmask}; foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}}) { - next if $id eq "general"; # This is used for global values. if ($ip_address eq $an->data->{database}{$id}{host}) { $local_id = $id; @@ -1062,7 +1065,7 @@ sub initialize $an->data->{sys}{db_initialized}{$id} = 1; # Mark that we need to update the DB. - $an->data->{database}{general}{resync_needed} = 1; + $an->data->{sys}{database}{resync_needed} = 1; return($success); }; @@ -2157,18 +2160,18 @@ sub write my $limit = 25000; my $count = 0; my $query_set = []; - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "database::general::maximum_batch_size" => $an->data->{database}{general}{maximum_batch_size} }}); - if ($an->data->{database}{general}{maximum_batch_size}) + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::database::maximum_batch_size" => $an->data->{sys}{database}{maximum_batch_size} }}); + if ($an->data->{sys}{database}{maximum_batch_size}) { - if ($an->data->{database}{general}{maximum_batch_size} =~ /\D/) + if ($an->data->{sys}{database}{maximum_batch_size} =~ /\D/) { # Bad value. - $an->data->{database}{general}{maximum_batch_size} = 25000; - $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "database::general::maximum_batch_size" => $an->data->{database}{general}{maximum_batch_size} }}); + $an->data->{sys}{database}{maximum_batch_size} = 25000; + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::database::maximum_batch_size" => $an->data->{sys}{database}{maximum_batch_size} }}); } # Use the set value now. - $limit = $an->data->{database}{general}{maximum_batch_size}; + $limit = $an->data->{sys}{database}{maximum_batch_size}; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { limit => $limit }}); } if (ref($query) eq "ARRAY") @@ -2336,8 +2339,8 @@ sub _find_behind_databases } # Look at all the databases and find the most recent time stamp (and the ID of the DB). - $an->data->{database}{general}{source_db_id} = 0; - $an->data->{database}{general}{source_updated_time} = 0; + $an->data->{sys}{database}{source_db_id} = 0; + $an->data->{sys}{database}{source_updated_time} = 0; foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}}) { my $name = $an->data->{database}{$id}{name}; @@ -2362,25 +2365,25 @@ AND $last_updated = 0 if not defined $last_updated; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - last_updated => $last_updated, - "database::general::source_updated_time" => $an->data->{database}{general}{source_updated_time}, + last_updated => $last_updated, + "sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time}, }}); - if ($last_updated > $an->data->{database}{general}{source_updated_time}) + if ($last_updated > $an->data->{sys}{database}{source_updated_time}) { - $an->data->{database}{general}{source_updated_time} = $last_updated; - $an->data->{database}{general}{source_db_id} = $id; + $an->data->{sys}{database}{source_updated_time} = $last_updated; + $an->data->{sys}{database}{source_db_id} = $id; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "database::general::source_db_id" => $an->data->{database}{general}{source_db_id}, - "database::general::source_updated_time" => $an->data->{database}{general}{source_updated_time}, + "sys::database::source_db_id" => $an->data->{sys}{database}{source_db_id}, + "sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time}, }}); } # Get the last updated time for this database (and source). $an->data->{database}{$id}{last_updated} = $last_updated; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "database::general::source_updated_time" => $an->data->{database}{general}{source_updated_time}, - "database::general::source_db_id" => $an->data->{database}{general}{source_db_id}, - "database::${id}::last_updated" => $an->data->{database}{$id}{last_updated} + "sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time}, + "sys::database::source_db_id" => $an->data->{sys}{database}{source_db_id}, + "database::${id}::last_updated" => $an->data->{database}{$id}{last_updated} }}); # If we have a tables hash, look into them, too. @@ -2431,14 +2434,14 @@ ORDER BY } # Find which DB is most up to date. - $an->data->{database}{general}{to_update} = {}; + $an->data->{sys}{database}{to_update} = {}; foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}}) { $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "database::general::source_updated_time" => $an->data->{database}{general}{source_updated_time}, - "database::${id}::last_updated" => $an->data->{database}{$id}{last_updated}, + "sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time}, + "database::${id}::last_updated" => $an->data->{database}{$id}{last_updated}, }}); - if ($an->data->{database}{general}{source_updated_time} > $an->data->{database}{$id}{last_updated}) + if ($an->data->{sys}{database}{source_updated_time} > $an->data->{database}{$id}{last_updated}) { # This database is behind $an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0104", variables => { @@ -2452,28 +2455,28 @@ ORDER BY else { # This database is up to date (so far). - $an->data->{database}{general}{to_update}{$id}{behind} = 0; + $an->data->{sys}{database}{to_update}{$id}{behind} = 0; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "database::general::to_update::${id}::behind" => $an->data->{database}{general}{to_update}{$id}{behind}, + "sys::database::to_update::${id}::behind" => $an->data->{sys}{database}{to_update}{$id}{behind}, }}); } # If we don't yet need a resync, and if we were passed one or more tables, check those tables # for differences - if ((not $an->data->{database}{general}{resync_needed}) && (ref($tables) eq "HASH")) + if ((not $an->data->{sys}{database}{resync_needed}) && (ref($tables) eq "HASH")) { foreach my $table (sort {$a cmp $b} keys %{$tables}) { - if (not defined $an->data->{database}{general}{tables}{$table}{last_updated}) + if (not defined $an->data->{sys}{database}{tables}{$table}{last_updated}) { # First we've seen, set the general updated time to this entry - $an->data->{database}{general}{tables}{$table}{last_updated} = $an->data->{database}{$id}{tables}{$table}{last_updated}; + $an->data->{sys}{database}{tables}{$table}{last_updated} = $an->data->{database}{$id}{tables}{$table}{last_updated}; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "database::general::tables::${table}::last_updated" => $an->data->{database}{general}{tables}{$table}{last_updated} + "sys::database::tables::${table}::last_updated" => $an->data->{sys}{database}{tables}{$table}{last_updated} }}); } - if ($an->data->{database}{general}{tables}{$table}{last_updated} > $an->data->{database}{$id}{tables}{$table}{last_updated}) + if ($an->data->{sys}{database}{tables}{$table}{last_updated} > $an->data->{database}{$id}{tables}{$table}{last_updated}) { # This database is behind $an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0106", variables => { @@ -2512,11 +2515,11 @@ sub _mark_database_as_behind my $id = $parameter->{id} ? $parameter->{id} : ""; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { id => $id }}); - $an->data->{database}{general}{to_update}{$id}{behind} = 1; - $an->data->{database}{general}{resync_needed} = 1; + $an->data->{sys}{database}{to_update}{$id}{behind} = 1; + $an->data->{sys}{database}{resync_needed} = 1; $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "database::general::to_update::${id}::behind" => $an->data->{database}{general}{to_update}{$id}{behind}, - "database::general::resync_needed" => $an->data->{database}{general}{resync_needed}, + "sys::database::to_update::${id}::behind" => $an->data->{sys}{database}{to_update}{$id}{behind}, + "sys::database::resync_needed" => $an->data->{sys}{database}{resync_needed}, }}); # We can't trust this database for reads, so switch to another database for reads if diff --git a/AN/Tools/Log.pm b/AN/Tools/Log.pm index 8627907e..0efff0af 100755 --- a/AN/Tools/Log.pm +++ b/AN/Tools/Log.pm @@ -528,6 +528,15 @@ sub variables my $entries = keys %{$list}; if ($entries) { + # If the key points to an undefined value, convert it to '!!undef!!' so that we don't scare + # the user with 'undefined variable' warnings. + foreach my $key (sort {$a cmp $b} keys %{$list}) + { + if (not defined $list->{$key}) + { + $list->{$key} = "!!undef!!"; + } + } my $raw = ""; if ($entries < 5) { diff --git a/AN/Tools/System.pm b/AN/Tools/System.pm index cd90b12c..921c28e9 100755 --- a/AN/Tools/System.pm +++ b/AN/Tools/System.pm @@ -15,6 +15,7 @@ my $THIS_FILE = "System.pm"; # call # check_daemon # check_memory +# enable_daemon # ping # read_ssh_config # remote_call @@ -219,6 +220,46 @@ sub check_memory return($used_ram); } +=head2 enable_daemon + +This method enables a daemon (so that it starts when the OS boots). The return code from the start request will be returned. + +If the return code for the enable command wasn't read, C<< undef >> is returned. + +Parameters; + +=head3 daemon (required) + +This is the name of the daemon to enable. + +=cut +sub enable_daemon +{ + my $self = shift; + my $parameter = shift; + my $an = $self->parent; + + my $return = undef; + my $daemon = defined $parameter->{daemon} ? $parameter->{daemon} : ""; + my $say_daemon = $daemon =~ /\.service$/ ? $daemon : $daemon.".service"; + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { daemon => $daemon, say_daemon => $say_daemon }}); + + my $output = $an->System->call({shell_call => $an->data->{path}{exe}{systemctl}." enable ".$say_daemon." 2>&1; ".$an->data->{path}{exe}{'echo'}." return_code:\$?"}); + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output }}); + foreach my $line (split/\n/, $output) + { + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }}); + if ($line =~ /return_code:(\d+)/) + { + $return = $1; + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'return' => $return }}); + } + } + + $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'return' => $return }}); + return($return); +} + =head2 ping This method will attempt to ping a target, by hostname or IP, and returns C<< 1 >> if successful, and C<< 0 >> if not. diff --git a/striker.conf b/striker.conf index 6a1fc7be..fe36b680 100644 --- a/striker.conf +++ b/striker.conf @@ -18,4 +18,4 @@ database::2::ping_before_connect = 1 # This puts a limit on how many queries (writes, generally) to make in a single batch transaction. This is # useful when doing very large transacions, like resync'ing a large table, by limiting how long a given # transaction can take and how much memory is used. -database::general::maximum_batch_size = 25000 +sys::database::maximum_batch_size = 25000