* Fixed a few more problems with the SQL schema and setting up the database when needed.

* Fixed more typo-related bugs.
* Created System->enable_daemon().
* Changed the 'database::general' to 'sys::database' to avoid accidental collisions with database IDs.

Signed-off-by: Digimer <digimer@alteeve.ca>

AN/Tools.sql

@@ -2,7 +2,18 @@
 -- It expects PostgreSQL v. 9.1+
 
 SET client_encoding = 'UTF8';
-CREATE SCHEMA IF NOT EXISTS history;
+-- This doesn't work before 9.3 - CREATE SCHEMA IF NOT EXISTS history;
+-- So we'll use the query below until (if) we upgrade.
+DO $$
+BEGIN
+	IF NOT EXISTS(
+		SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'history'
+	)
+	THEN
+		EXECUTE 'CREATE SCHEMA history';
+	END IF;
+END
+$$;
 
 -- This stores information about the host machine. This is the master table that everything will be linked 
 -- to. 
@@ -10,9 +21,7 @@ CREATE TABLE hosts (
 	host_uuid			uuid				not null	primary key,	-- This is the single most important record in ScanCore. Everything links back to here.
 	host_name			text				not null,
 	host_type			text				not null,			-- Either 'node' or 'dashboard'.
-	modified_date			timestamp with time zone	not null,
-	
-	FOREIGN KEY(host_location_uuid) REFERENCES locations(location_uuid)
+	modified_date			timestamp with time zone	not null
 );
 ALTER TABLE hosts OWNER TO #!variable!user!#;
 
@@ -58,9 +67,7 @@ CREATE TABLE host_variable (
 	host_variable_host_uuid		uuid				not null,
 	host_variable_name		text				not null,
 	host_variable_value		text,
-	modified_date			timestamp with time zone	not null,
-	
-	FOREIGN KEY(host_location_uuid) REFERENCES locations(location_uuid)
+	modified_date			timestamp with time zone	not null
 );
 ALTER TABLE host_variable OWNER TO #!variable!user!#;
 
@@ -110,7 +117,7 @@ CREATE TABLE alerts (
 	alert_level		text				not null,			-- debug (log only), info (+ admin email), notice (+ curious users), warning (+ client technical staff), critical (+ all)
 	alert_title_key		text				not null,			-- ScanCore will read in the agents <name>.xml words file and look for this message key
 	alert_title_variables	text,								-- List of variables to substitute into the message key. Format is 'var1=val1 #!# var2 #!# val2 #!# ... #!# varN=valN'.
-	alert_message_key	text				not null			-- ScanCore will read in the agents <name>.xml words file and look for this message key
+	alert_message_key	text				not null,			-- ScanCore will read in the agents <name>.xml words file and look for this message key
 	alert_message_variables	text,								-- List of variables to substitute into the message key. Format is 'var1=val1 #!# var2 #!# val2 #!# ... #!# varN=valN'.
 	alert_sort		text,								-- The alerts will sort on this column. It allows for an optional sorting of the messages in the alert.
 	alert_header		boolean				not null	default TRUE,	-- This can be set to have the alert be printed with only the contents of the string, no headers.
@@ -264,7 +271,7 @@ ALTER TABLE updated OWNER TO #!variable!user!#;
 CREATE TABLE alert_sent (
 	alert_sent_host_uuid	uuid				not null,			-- The node associated with this alert
 	alert_set_by		text				not null,			-- name of the program that set this alert
-	alert_record_locator	text,				not null			-- String used by the agent to identify the source of the alert (ie: UPS serial number)
+	alert_record_locator	text				not null,			-- String used by the agent to identify the source of the alert (ie: UPS serial number)
 	alert_name		text				not null,			-- A free-form name used by the caller to identify this alert.
 	modified_date		timestamp with time zone	not null,
 	

AN/Tools/Alert.pm

@@ -179,7 +179,7 @@ AND
 ;";
 	$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
 	
-	my $count = $an->DB->do_db_query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
+	my $count = $an->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
 	$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 		type  => $type,
 		query => $query,
@@ -205,7 +205,7 @@ WHERE
 ;";
 			$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
 
-			my $count = $an->DB->do_db_query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
+			my $count = $an->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
 			$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
 			
 			if (not $count)

AN/Tools/Database.pm

@@ -145,6 +145,10 @@ sub configure_pgsql
 			{
 				# Initialized!
 				$an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0055"});
+				
+				# Enable it on boot.
+				my $return_code = $an->System->enable_daemon({daemon => "postgresql"});
+				$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
 			}
 		}
 	}
@@ -521,6 +525,8 @@ sub connect
 		test_table => $test_table, 
 	}});
 	
+	$an->data->{sys}{db_timestamp} = "" if not defined $an->data->{sys}{db_timestamp};
+	
 	# We need the host_uuid before we connect.
 	if (not $an->data->{sys}{host_uuid})
 	{
@@ -533,7 +539,7 @@ sub connect
 	$an->data->{sys}{local_db_id} = "";
 	
 	# This will be set to '1' if either DB needs to be initialized or if the last_updated differs on any node.
-	$an->data->{database}{general}{resync_needed} = 0;
+	$an->data->{sys}{database}{resync_needed} = 0;
 	
 	# Now setup or however-many connections
 	my $seen_connections       = [];
@@ -542,7 +548,6 @@ sub connect
 	my $successful_connections = [];
 	foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}})
 	{
-		next if $id eq "general";	# This is used for global values.
 		my $driver   = "DBI:Pg";
 		my $host     = $an->data->{database}{$id}{host}     ? $an->data->{database}{$id}{host}     : ""; # This should fail
 		my $port     = $an->data->{database}{$id}{port}     ? $an->data->{database}{$id}{port}     : 5432;
@@ -750,7 +755,7 @@ sub connect
 			# Get a time stamp for this run, if not yet gotten.
 			$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 				"cache::db_fh::$id" => $an->data->{cache}{db_fh}{$id}, 
-				"sys::db_timestamp" => $an->data->{sys}{db_timestamp}
+				"sys::db_timestamp" => $an->data->{sys}{db_timestamp},
 			}});
 			
 			# Pick a timestamp for this run, if we haven't yet.
@@ -796,9 +801,9 @@ sub connect
 		# If I've not sent an alert about this DB loss before, send one now.
 		my $set = $an->Alert->check_alert_sent({
 			type		=>	"set",
-			alert_set_by		=>	$THIS_FILE,
-			alert_record_locator	=>	$id,
-			alert_name		=>	"connect_to_db",
+			set_by		=>	$THIS_FILE,
+			record_locator	=>	$id,
+			name		=>	"connect_to_db",
 			modified_date	=>	$an->data->{sys}{db_timestamp},
 		});
 		$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { set => $set }});
@@ -843,9 +848,9 @@ sub connect
 		{
 			my $cleared = $an->Alert->check_alert_sent({
 				type		=>	"clear",
-				alert_sent_by		=>	$THIS_FILE,
-				alert_record_locator	=>	$id,
-				alert_name		=>	"connect_to_db",
+				sent_by		=>	$THIS_FILE,
+				record_locator	=>	$id,
+				name		=>	"connect_to_db",
 				modified_date	=>	$an->data->{sys}{db_timestamp},
 			});
 			if ($cleared)
@@ -878,9 +883,9 @@ sub connect
 	
 	# For now, we just find which DBs are behind and let each agent deal with bringing their tables up to
 	# date.
-	$an->database->_find_behind_databases({
+	$an->Database->_find_behind_databases({
 		source => $source, 
-		tables => $$tables,
+		tables => $tables,
 	});
 	
 	# Hold if a lock has been requested.
@@ -949,7 +954,6 @@ sub get_local_id
 	my $network_details = $an->Get->network_details;
 	foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}})
 	{
-		next if $id eq "general";	# This is used for global values.
 		if ($network_details->{hostname} eq $an->data->{database}{$id}{host})
 		{
 			$local_id = $id;
@@ -964,7 +968,6 @@ sub get_local_id
 			my $subnet_mask = $network_details->{interface}{$interface}{netmask};
 			foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}})
 			{
-				next if $id eq "general";	# This is used for global values.
 				if ($ip_address eq $an->data->{database}{$id}{host})
 				{
 					$local_id = $id;
@@ -1062,7 +1065,7 @@ sub initialize
 	$an->data->{sys}{db_initialized}{$id} = 1;
 	
 	# Mark that we need to update the DB.
-	$an->data->{database}{general}{resync_needed} = 1;
+	$an->data->{sys}{database}{resync_needed} = 1;
 	
 	return($success);
 };
@@ -2157,18 +2160,18 @@ sub write
 	my $limit     = 25000;
 	my $count     = 0;
 	my $query_set = [];
-	$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "database::general::maximum_batch_size" => $an->data->{database}{general}{maximum_batch_size} }});
-	if ($an->data->{database}{general}{maximum_batch_size})
+	$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::database::maximum_batch_size" => $an->data->{sys}{database}{maximum_batch_size} }});
+	if ($an->data->{sys}{database}{maximum_batch_size})
 	{
-		if ($an->data->{database}{general}{maximum_batch_size} =~ /\D/)
+		if ($an->data->{sys}{database}{maximum_batch_size} =~ /\D/)
 		{
 			# Bad value.
-			$an->data->{database}{general}{maximum_batch_size} = 25000;
-			$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "database::general::maximum_batch_size" => $an->data->{database}{general}{maximum_batch_size} }});
+			$an->data->{sys}{database}{maximum_batch_size} = 25000;
+			$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::database::maximum_batch_size" => $an->data->{sys}{database}{maximum_batch_size} }});
 		}
 		
 		# Use the set value now.
-		$limit = $an->data->{database}{general}{maximum_batch_size};
+		$limit = $an->data->{sys}{database}{maximum_batch_size};
 		$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { limit => $limit }});
 	}
 	if (ref($query) eq "ARRAY")
@@ -2336,8 +2339,8 @@ sub _find_behind_databases
 	}
 	
 	# Look at all the databases and find the most recent time stamp (and the ID of the DB).
-	$an->data->{database}{general}{source_db_id}        = 0;
-	$an->data->{database}{general}{source_updated_time} = 0;
+	$an->data->{sys}{database}{source_db_id}        = 0;
+	$an->data->{sys}{database}{source_updated_time} = 0;
 	foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}})
 	{
 		my $name = $an->data->{database}{$id}{name};
@@ -2363,23 +2366,23 @@ AND
 		   
 		$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			last_updated                         => $last_updated, 
-			"database::general::source_updated_time" => $an->data->{database}{general}{source_updated_time}, 
+			"sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time}, 
 		}});
-		if ($last_updated > $an->data->{database}{general}{source_updated_time})
+		if ($last_updated > $an->data->{sys}{database}{source_updated_time})
 		{
-			$an->data->{database}{general}{source_updated_time} = $last_updated;
-			$an->data->{database}{general}{source_db_id}        = $id;
+			$an->data->{sys}{database}{source_updated_time} = $last_updated;
+			$an->data->{sys}{database}{source_db_id}        = $id;
 			$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-				"database::general::source_db_id"        => $an->data->{database}{general}{source_db_id}, 
-				"database::general::source_updated_time" => $an->data->{database}{general}{source_updated_time}, 
+				"sys::database::source_db_id"        => $an->data->{sys}{database}{source_db_id}, 
+				"sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time}, 
 			}});
 		}
 		
 		# Get the last updated time for this database (and source).
 		$an->data->{database}{$id}{last_updated} = $last_updated;
 		$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-			"database::general::source_updated_time" => $an->data->{database}{general}{source_updated_time}, 
-			"database::general::source_db_id"        => $an->data->{database}{general}{source_db_id}, 
+			"sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time}, 
+			"sys::database::source_db_id"        => $an->data->{sys}{database}{source_db_id}, 
 			"database::${id}::last_updated"      => $an->data->{database}{$id}{last_updated}
 		}});
 		
@@ -2431,14 +2434,14 @@ ORDER BY
 	}
 	
 	# Find which DB is most up to date.
-	$an->data->{database}{general}{to_update} = {};
+	$an->data->{sys}{database}{to_update} = {};
 	foreach my $id (sort {$a cmp $b} keys %{$an->data->{database}})
 	{
 		$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-			"database::general::source_updated_time" => $an->data->{database}{general}{source_updated_time}, 
+			"sys::database::source_updated_time" => $an->data->{sys}{database}{source_updated_time}, 
 			"database::${id}::last_updated"      => $an->data->{database}{$id}{last_updated}, 
 		}});
-		if ($an->data->{database}{general}{source_updated_time} > $an->data->{database}{$id}{last_updated})
+		if ($an->data->{sys}{database}{source_updated_time} > $an->data->{database}{$id}{last_updated})
 		{
 			# This database is behind
 			$an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0104", variables => {
@@ -2452,28 +2455,28 @@ ORDER BY
 		else
 		{
 			# This database is up to date (so far).
-			$an->data->{database}{general}{to_update}{$id}{behind} = 0;
+			$an->data->{sys}{database}{to_update}{$id}{behind} = 0;
 			$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-				"database::general::to_update::${id}::behind" => $an->data->{database}{general}{to_update}{$id}{behind}, 
+				"sys::database::to_update::${id}::behind" => $an->data->{sys}{database}{to_update}{$id}{behind}, 
 			}});
 		}
 		
 		# If we don't yet need a resync, and if we were passed one or more tables, check those tables
 		# for differences
-		if ((not $an->data->{database}{general}{resync_needed}) && (ref($tables) eq "HASH"))
+		if ((not $an->data->{sys}{database}{resync_needed}) && (ref($tables) eq "HASH"))
 		{
 			foreach my $table (sort {$a cmp $b} keys %{$tables})
 			{
-				if (not defined $an->data->{database}{general}{tables}{$table}{last_updated})
+				if (not defined $an->data->{sys}{database}{tables}{$table}{last_updated})
 				{
 					# First we've seen, set the general updated time to this entry
-					$an->data->{database}{general}{tables}{$table}{last_updated} = $an->data->{database}{$id}{tables}{$table}{last_updated};
+					$an->data->{sys}{database}{tables}{$table}{last_updated} = $an->data->{database}{$id}{tables}{$table}{last_updated};
 					$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-						"database::general::tables::${table}::last_updated" => $an->data->{database}{general}{tables}{$table}{last_updated}
+						"sys::database::tables::${table}::last_updated" => $an->data->{sys}{database}{tables}{$table}{last_updated}
 					}});
 				}
 				
-				if ($an->data->{database}{general}{tables}{$table}{last_updated} > $an->data->{database}{$id}{tables}{$table}{last_updated})
+				if ($an->data->{sys}{database}{tables}{$table}{last_updated} > $an->data->{database}{$id}{tables}{$table}{last_updated})
 				{
 					# This database is behind
 					$an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0106", variables => {
@@ -2512,11 +2515,11 @@ sub _mark_database_as_behind
 	my $id = $parameter->{id} ? $parameter->{id} : "";
 	$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { id => $id }});
 	
-	$an->data->{database}{general}{to_update}{$id}{behind} = 1;
-	$an->data->{database}{general}{resync_needed}          = 1;
+	$an->data->{sys}{database}{to_update}{$id}{behind} = 1;
+	$an->data->{sys}{database}{resync_needed}          = 1;
 	$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-		"database::general::to_update::${id}::behind" => $an->data->{database}{general}{to_update}{$id}{behind}, 
-		"database::general::resync_needed"            => $an->data->{database}{general}{resync_needed}, 
+		"sys::database::to_update::${id}::behind" => $an->data->{sys}{database}{to_update}{$id}{behind}, 
+		"sys::database::resync_needed"            => $an->data->{sys}{database}{resync_needed}, 
 	}});
 		
 	# We can't trust this database for reads, so switch to another database for reads if

AN/Tools/Log.pm

@@ -528,6 +528,15 @@ sub variables
 	my $entries = keys %{$list};
 	if ($entries)
 	{
+		# If the key points to an undefined value, convert it to '!!undef!!' so that we don't scare
+		# the user with 'undefined variable' warnings.
+		foreach my $key (sort {$a cmp $b} keys %{$list})
+		{
+			if (not defined $list->{$key})
+			{
+				$list->{$key} = "!!undef!!";
+			}
+		}
 		my $raw = "";
 		if ($entries < 5)
 		{

AN/Tools/System.pm

@@ -15,6 +15,7 @@ my $THIS_FILE = "System.pm";
 # call
 # check_daemon
 # check_memory
+# enable_daemon
 # ping
 # read_ssh_config
 # remote_call
@@ -219,6 +220,46 @@ sub check_memory
 	return($used_ram);
 }
 
+=head2 enable_daemon
+
+This method enables a daemon (so that it starts when the OS boots). The return code from the start request will be returned.
+
+If the return code for the enable command wasn't read, C<< undef >> is returned.
+
+Parameters;
+
+=head3 daemon (required)
+
+This is the name of the daemon to enable.
+
+=cut
+sub enable_daemon
+{
+	my $self      = shift;
+	my $parameter = shift;
+	my $an        = $self->parent;
+	
+	my $return     = undef;
+	my $daemon     = defined $parameter->{daemon} ? $parameter->{daemon} : "";
+	my $say_daemon = $daemon =~ /\.service$/ ? $daemon : $daemon.".service";
+	$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { daemon => $daemon, say_daemon => $say_daemon }});
+	
+	my $output = $an->System->call({shell_call => $an->data->{path}{exe}{systemctl}." enable ".$say_daemon." 2>&1; ".$an->data->{path}{exe}{'echo'}." return_code:\$?"});
+	$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output }});
+	foreach my $line (split/\n/, $output)
+	{
+		$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
+		if ($line =~ /return_code:(\d+)/)
+		{
+			$return = $1;
+			$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'return' => $return }});
+		}
+	}
+	
+	$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'return' => $return }});
+	return($return);
+}
+
 =head2 ping
 
 This method will attempt to ping a target, by hostname or IP, and returns C<< 1 >> if successful, and C<< 0 >> if not.

striker.conf

@@ -18,4 +18,4 @@ database::2::ping_before_connect	=	1
 # This puts a limit on how many queries (writes, generally) to make in a single batch transaction. This is 
 # useful when doing very large transacions, like resync'ing a large table, by limiting how long a given 
 # transaction can take and how much memory is used. 
-database::general::maximum_batch_size	=	25000
+sys::database::maximum_batch_size	=	25000