* Fixed a bug where setting the debug level to 3 caused a deep recursion and a system hang.

* Update Anvil::Tools->new() to access the parameters 'log_level', 'log_secure' and 'debug', streamlining the frequent calls to $anvil->Log->level and ->secure in program startup, and allowing the values to take effect during the ->new constructor.
* Passed 'debug' to child method calls in more places (still more to do though).
* Fixed a bug where 'test_table' wasn't set in the right place, causing the database to try to initialize repeatedly.
* Made Database->archive_database only run if called with root access.
* Now the number of database connections are stored in 'sys::db_connections' instead of checking the returned number, and that is cleared on disconnect.
* Started working more on 'anvil-daemon', including adding support for System->call being taking 'background', 'stderr_file' and 'stdout_file' paramters which, when set, used Proc::Simple to background the process.
* Did some more work on database archiving, though still far from done.

Signed-off-by: Digimer <digimer@alteeve.ca>

Anvil/Tools.pm

@@ -147,22 +147,6 @@ sub new
 	my $anvil = $self; 
 	weaken($anvil);	# Helps avoid memory leaks. See Scalar::Utils
 	
-	# Record the start time.
-	$anvil->data->{ENV_VALUES}{START_TIME} = Time::HiRes::time;
-	
-	# Set passed parameters if needed.
-	if (ref($parameter) eq "HASH")
-	{
-		### TODO: Calls to allow the user to override defaults...
-		# Local parameters...
-	}
-	elsif ($parameter)
-	{
-		# Um...
-		print $THIS_FILE." ".__LINE__."; Anvil::Tools->new() invoked with an invalid parameter. Expected a hash reference, but got: [$parameter]\n";
-		exit(1);
-	}
-	
 	# Get a handle on the various submodules
 	$anvil->Account->parent($anvil);
 	$anvil->Alert->parent($anvil);
@@ -178,8 +162,38 @@ sub new
 	$anvil->Validate->parent($anvil);
 	
 	# Set some system paths and system default variables
-	$anvil->_set_paths;
+	$anvil->_set_paths();
-	$anvil->_set_defaults;
+	$anvil->_set_defaults();
+	
+	# Record the start time.
+	$anvil->data->{ENV_VALUES}{START_TIME} = Time::HiRes::time;
+	
+	# Set passed parameters if needed.
+	my $debug = 3;
+	if (ref($parameter) eq "HASH")
+	{
+		# Local parameters...
+		if ($parameter->{debug})
+		{
+			$debug = $parameter->{debug};
+		}
+		if ($parameter->{log_secure})
+		{
+			$anvil->Log->secure({set => $parameter->{log_secure}});
+		}
+	}
+	elsif ($parameter)
+	{
+		# Um...
+		print $THIS_FILE." ".__LINE__."; Anvil::Tools->new() invoked with an invalid parameter. Expected a hash reference, but got: [$parameter]\n";
+		exit(1);
+	}
+	
+	# If the user passed a custom log level, sit it now.
+	if ($parameter->{log_level})
+	{
+		$anvil->Log->level({set => $parameter->{log_level}});
+	}
 	
 	# This will help clean up if we catch a signal.
 	$SIG{INT}  = sub { $anvil->catch_sig({signal => "INT"});  };
@@ -202,22 +216,33 @@ sub new
 	$anvil->data($parameter->{data}) if $parameter->{data};
 	
 	# Initialize the list of directories to seach.
-	$anvil->Storage->search_directories({initialize => 1});
+	$anvil->Storage->search_directories({debug => $debug, initialize => 1});
 	
 	# I need to read the initial words early.
-	$anvil->Words->read({file  => $anvil->data->{path}{words}{'words.xml'}});
+	$anvil->Words->read({debug => $debug, file  => $anvil->data->{path}{words}{'words.xml'}});
 	
 	# If the local './tools.conf' file exists, read it in.
 	if (-r $anvil->data->{path}{configs}{'anvil.conf'})
 	{
-		$anvil->Storage->read_config({file => $anvil->data->{path}{configs}{'anvil.conf'}});
+		$anvil->Storage->read_config({debug => $debug, file => $anvil->data->{path}{configs}{'anvil.conf'}});
+		
+		### TODO: Should anvil.conf override parameters?
+		# Let parameters override config file values.
+		if ($parameter->{log_level})
+		{
+			$anvil->Log->level({set => $parameter->{log_level}});
+		}
+		if ($parameter->{log_secure})
+		{
+			$anvil->Log->secure({set => $parameter->{log_secure}});
+		}
 	}
 	
 	# Get the local host UUID.
-	$anvil->Get->host_uuid;
+	$anvil->Get->host_uuid({debug => $debug});
 	
 	# Read in any command line switches.
-	$anvil->Get->switches;
+	$anvil->Get->switches({debug => $debug});
 	
 	# Read in the local Anvil! version.
 	#...
@@ -263,7 +288,7 @@ Data can be entered into or access by treating '$anvil->data' as a normal hash r
  # Set 'A thing' in 'foo'.
  $anvil->data->{foo} = "A thing";
 
-The C<$an> variable is set inside all modules and acts as shared storage for variables, values and references in all modules. It acts as the core storage for most applications using Anvil::Tools.
+The C<< $anvil >> variable is set inside all modules and acts as shared storage for variables, values and references in all modules. It acts as the core storage for most applications using Anvil::Tools.
 
 =cut
 sub data
@@ -717,6 +742,7 @@ sub _set_defaults
 			log_transactions		=>	0,
 			maximum_batch_size		=>	25000,
 			name				=>	"anvil",
+			test_table			=>	"hosts",
 			user				=>	"admin",
 		},
 		host_type			=>	"",
@@ -806,9 +832,6 @@ sub _set_defaults
 			server		=>	"",
 			tag		=>	"anvil",
 		},
-		sql		=>	{
-			test_table	=>	"hosts",
-		},
 		template	=>	{
 			html		=>	"alteeve",
 		},

Anvil/Tools/Database.pm

@@ -116,10 +116,25 @@ sub archive_database
 	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->archive_database()" }});
 	
+	# Is archiving disabled?
+	if (not $anvil->data->{sys}{database}{archive}{trigger})
+	{
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0189"});
+		return(1);
+	}
+	
+	# Only the root user can archive the database so that the archived files can be properly secured.
+	if (($< != 0) && ($> != 0))
+	{
+		# Not root
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0188"});
+		return(1);
+	}
+	
 	# If we don't have an array of tables, we have nothing to do.
 	if ((not exists $anvil->data->{sys}{database}{check_tables}) or (ref(@{$anvil->data->{sys}{database}{check_tables}} ne "ARRAY")))
 	{
-		return(0);
+		return(1);
 	}
 	
 	# We'll use the list of tables created for _find_behind_databases()'s 'sys::database::check_tables' 
@@ -196,11 +211,11 @@ sub check_lock_age
 
 =head2 configure_pgsql
 
-This configures the local database server. Specifically, it checks to make sure the daemon is running and starts it if not. It also checks the 'pg_hba.conf' configuration to make sure it is set properly to listen on this machine's IP addresses and interfaces.
+This configures the local database server. Specifically, it checks to make sure the daemon is running and starts it if not. It also checks the C<< pg_hba.conf >> configuration to make sure it is set properly to listen on this machine's IP addresses and interfaces.
 
 If the system is already configured, this method will do nothing, so it is safe to call it at any time.
 
-If there is a problem, C<< !!error!! >> is returned.
+If the method completes, C<< 0 >> is returned. If this method is called without C<< root >> access, it returns C<< 1 >> without doing anything. If there is a problem, C<< !!error!! >> is returned.
 
 =cut
 sub configure_pgsql
@@ -221,11 +236,11 @@ sub configure_pgsql
 		# This is a minor error as it will be hit by every unpriviledged program that connects to the
 		# database(s).
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, priority => "alert", key => "log_0113"});
-		return("!!error!!");
+		return(1);
 	}
 	
 	# First, is it running?
-	my $running = $anvil->System->check_daemon({daemon => "postgresql"});
+	my $running = $anvil->System->check_daemon({debug => $debug, daemon => "postgresql"});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { running => $running }});
 	
 	if (not $running)
@@ -234,7 +249,7 @@ sub configure_pgsql
 		if (not -e $anvil->data->{path}{configs}{'pg_hba.conf'})
 		{
 			# Initialize.
-			my $output = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{'postgresql-setup'}." initdb", source => $THIS_FILE, line => __LINE__});
+			my $output = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{'postgresql-setup'}." initdb", source => $THIS_FILE, line => __LINE__});
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output }});
 			
 			# Did it succeed?
@@ -250,14 +265,14 @@ sub configure_pgsql
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0055"});
 				
 				# Enable it on boot.
-				my $return_code = $anvil->System->enable_daemon({daemon => "postgresql"});
+				my $return_code = $anvil->System->enable_daemon({debug => $debug, daemon => "postgresql"});
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { return_code => $return_code }});
 			}
 		}
 	}
 	
 	# Setup postgresql.conf, if needed
-	my $postgresql_conf        = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'postgresql.conf'}});
+	my $postgresql_conf        = $anvil->Storage->read_file({debug => $debug, file => $anvil->data->{path}{configs}{'postgresql.conf'}});
 	my $update_postgresql_file = 1;
 	my $new_postgresql_conf    = "";
 	foreach my $line (split/\n/, $postgresql_conf)
@@ -288,6 +303,7 @@ sub configure_pgsql
 		if (not -e $postgresql_backup)
 		{
 			$anvil->Storage->copy_file({
+				debug       => $debug, 
 				source_file => $anvil->data->{path}{configs}{'postgresql.conf'}, 
 				target_file => $postgresql_backup,
 			});
@@ -295,6 +311,7 @@ sub configure_pgsql
 		
 		# Write the updated one.
 		$anvil->Storage->write_file({
+			debug     => $debug, 
 			file      => $anvil->data->{path}{configs}{'postgresql.conf'}, 
 			body      => $new_postgresql_conf,
 			user      => "postgres", 
@@ -306,7 +323,7 @@ sub configure_pgsql
 	}
 	
 	# Setup pg_hba.conf now, if needed.
-	my $pg_hba_conf        = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'pg_hba.conf'}});
+	my $pg_hba_conf        = $anvil->Storage->read_file({debug => $debug, file => $anvil->data->{path}{configs}{'pg_hba.conf'}});
 	my $update_pg_hba_file = 1;
 	my $new_pg_hba_conf    = "";
 	foreach my $line (split/\n/, $pg_hba_conf)
@@ -631,7 +648,7 @@ sub connect
 	my $source     = defined $parameter->{source}     ? $parameter->{source}     : "core";
 	my $sql_file   = defined $parameter->{sql_file}   ? $parameter->{sql_file}   : $anvil->data->{path}{sql}{'anvil.sql'};
 	my $tables     = defined $parameter->{tables}     ? $parameter->{tables}     : "";
-	my $test_table = defined $parameter->{test_table} ? $parameter->{test_table} : $anvil->data->{defaults}{sql}{test_table};
+	my $test_table = defined $parameter->{test_table} ? $parameter->{test_table} : $anvil->data->{sys}{database}{test_table};
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		source     => $source, 
 		sql_file   => $sql_file, 
@@ -730,6 +747,7 @@ sub connect
 		{
 			# Can I ping?
 			my ($pinged) = $anvil->System->ping({
+				debug   => $debug, 
 				ping    => $host, 
 				count   => 1,
 				timeout => $anvil->data->{database}{$uuid}{ping},
@@ -754,7 +772,7 @@ sub connect
 		}
 		
 		# Before we try to connect, see if this is a local database and, if so, make sure it's setup.
-		my $is_local = $anvil->System->is_local({host => $host});
+		my $is_local = $anvil->System->is_local({debug => $debug, host => $host});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { is_local => $is_local }});
 		if ($is_local)
 		{
@@ -762,7 +780,7 @@ sub connect
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::read_db_uuid" => $anvil->data->{sys}{read_db_uuid} }});
 			
 			# Set it up (or update it) if needed. This method just returns if nothing is needed.
-			$anvil->Database->configure_pgsql({uuid => $uuid});
+			$anvil->Database->configure_pgsql({debug => $debug, uuid => $uuid});
 		}
 		elsif (not $anvil->data->{sys}{read_db_uuid})
 		{
@@ -775,6 +793,7 @@ sub connect
 		if (not $is_local)
 		{
 			my $remote_version = $anvil->Get->anvil_version({
+				debug    => $debug, 
 				target   => $host,
 				password => $password,
 			});
@@ -783,7 +802,7 @@ sub connect
 				"anvil->_anvil_version" => $anvil->_anvil_version,
 			}});
 			
-			if ($remote_version ne $anvil->_anvil_version)
+			if ($remote_version ne $anvil->_anvil_version({debug => $debug}))
 			{
 				# Version doesn't match, 
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0145", variables => { 
@@ -886,13 +905,13 @@ sub connect
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'sys::use_db_fh' => $anvil->data->{sys}{use_db_fh} }});
 			}
 			
-			# If the '$test_table' isn't the same as 'defaults::sql::test_table', see if the core schema needs loading first.
+			# If the '$test_table' isn't the same as 'sys::database::test_table', see if the core schema needs loading first.
-			if ($test_table ne $anvil->data->{defaults}{sql}{test_table})
+			if ($test_table ne $anvil->data->{sys}{database}{test_table})
 			{
 				my $query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename=".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{defaults}{sql}{test_table})." AND schemaname='public';";
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
 				
-				my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
+				my $count = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
 				
 				if ($count < 1)
@@ -900,7 +919,7 @@ sub connect
 					### TODO: Create a version file/flag and don't sync with peers unless
 					###       they are the same version. Back-port this to v2.
 					# Need to load the database.
-					$anvil->Database->initialize({uuid => $uuid, sql_file => $anvil->data->{path}{sql}{'anvil.sql'}});
+					$anvil->Database->initialize({debug => $debug, uuid => $uuid, sql_file => $anvil->data->{path}{sql}{'anvil.sql'}});
 				}
 			}
 			
@@ -908,13 +927,13 @@ sub connect
 			my $query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename=".$anvil->data->{sys}{use_db_fh}->quote($test_table)." AND schemaname='public';";
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
 			
-			my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
+			my $count = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
 			
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
 			if ($count < 1)
 			{
 				# Need to load the database.
-				$anvil->Database->initialize({uuid => $uuid, sql_file => $sql_file});
+				$anvil->Database->initialize({debug => $debug, uuid => $uuid, sql_file => $sql_file});
 			}
 			
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
@@ -952,7 +971,7 @@ sub connect
 				my $query = "SELECT cast(now() AS timestamp with time zone)::timestamptz(0);";
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
 				
-				$anvil->data->{sys}{db_timestamp} = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
+				$anvil->data->{sys}{db_timestamp} = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::db_timestamp" => $anvil->data->{sys}{db_timestamp} }});
 			}
 			
@@ -1002,6 +1021,7 @@ sub connect
 		
 		# If I've not sent an alert about this DB loss before, send one now.
 		my $set = $anvil->Alert->check_alert_sent({
+			debug          => $debug, 
 			type           => "set",
 			set_by         => $THIS_FILE,
 			record_locator => $uuid,
@@ -1025,11 +1045,12 @@ sub connect
 				
 				# These are warning level alerts.
 				$anvil->Alert->register_alert({
-					alert_level		=>	"warning", 
+					debug                   => $debug, 
-					alert_set_by		=>	$THIS_FILE,
+					alert_level             => "warning", 
-					alert_title_key		=>	"alert_title_0003",
+					alert_set_by            => $THIS_FILE,
-					alert_message_key	=>	$message_key,
+					alert_title_key         => "alert_title_0003",
-					alert_message_variables	=>	$message_variables,
+					alert_message_key       => $message_key,
+					alert_message_variables => $message_variables,
 				});
 			}
 		}
@@ -1055,29 +1076,31 @@ sub connect
 # 		my $query = "SELECT COUNT(*) FROM hosts WHERE host_name = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{database}{$uuid}{host}).";";
 # 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
 # 
-# 		my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
+# 		my $count = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
 # 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
 # 		
 # 		if ($count > 0)
 # 		{
 			my $cleared = $anvil->Alert->check_alert_sent({
-				type		=>	"clear",
+				debug          => $debug, 
-				set_by		=>	$THIS_FILE,
+				type           => "clear",
-				record_locator	=>	$uuid,
+				set_by         => $THIS_FILE,
-				name		=>	"connect_to_db",
+				record_locator => $uuid,
-				modified_date	=>	$anvil->data->{sys}{db_timestamp},
+				name           => "connect_to_db",
+				modified_date  => $anvil->data->{sys}{db_timestamp},
 			});
 			if ($cleared)
 			{
 				$anvil->Alert->register_alert({
-					level			=>	"warning", 
+					debug             => $debug, 
-					agent_name		=>	"Anvil!",
+					level             => "warning", 
-					title_key		=>	"an_title_0006",
+					agent_name        => "Anvil!",
-					message_key		=>	"cleared_log_0055",
+					title_key         => "an_title_0006",
-					message_variables	=>	{
+					message_key       => "cleared_log_0055",
-						name			=>	$database_name,
+					message_variables => {
-						host			=>	$anvil->data->{database}{$uuid}{host},
+						name => $database_name,
-						port			=>	defined $anvil->data->{database}{$uuid}{port} ? $anvil->data->{database}{$uuid}{port} : 5432,
+						host => $anvil->data->{database}{$uuid}{host},
+						port => defined $anvil->data->{database}{$uuid}{port} ? $anvil->data->{database}{$uuid}{port} : 5432,
 					},
 				});
 			}
@@ -1091,7 +1114,7 @@ sub connect
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0103"});
 		
 		# Disconnect and set the connection count to '0'.
-		$anvil->Database->disconnect();
+		$anvil->Database->disconnect({debug => $debug});
 		$connections = 0;
 	}
 	
@@ -1100,25 +1123,29 @@ sub connect
 	if ($connections > 1)
 	{
 		$anvil->Database->_find_behind_databases({
+			debug  => $debug, 
 			source => $source, 
 			tables => $tables,
 		});
 	}
 	
 	# Hold if a lock has been requested.
-	$anvil->Database->locking();
+	$anvil->Database->locking({debug => $debug});
 	
 	# Mark that we're not active.
-	$anvil->Database->mark_active({set => 1});
+	$anvil->Database->mark_active({debug => $debug, set => 1});
 	
 	# Archive old data.
-	$anvil->Database->archive_database({});
+	$anvil->Database->archive_database({debug => $debug});
 	
 	# Sync the database, if needed.
-	$anvil->Database->resync_databases;
+	$anvil->Database->resync_databases({debug => $debug});
 	
 	# Add ourselves to the database, if needed.
-	$anvil->Database->insert_or_update_hosts;
+	$anvil->Database->insert_or_update_hosts({debug => $debug});
+	
+	# We'll store the number of connections in the hash.
+	$anvil->data->{sys}{db_connections} = $connections;
 	
 	return($connections);
 }
@@ -1159,6 +1186,9 @@ sub disconnect
 	delete $anvil->data->{sys}{use_db_fh};
 	delete $anvil->data->{sys}{read_db_uuid};
 	
+	# Set the connection count to 0.
+	$anvil->data->{sys}{db_connections} = 0;
+	
 	return(0);
 }
 
@@ -5160,20 +5190,6 @@ sub _archive_table
 		return("!!error!!");
 	}
 	
-	# Has the user disabled archiving?
-	my $trigger = $anvil->data->{sys}{database}{archive}{trigger};
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { trigger => $trigger }});
-	if ((exists $anvil->data->{sys}{database}{archive}{tables}{$table}{division}) && ($anvil->data->{sys}{database}{archive}{tables}{$table}{division} =~ /^\d+$/))
-	{
-		$trigger = $anvil->data->{sys}{database}{archive}{tables}{$table}{division};
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { trigger => $trigger }});
-	}
-	if ($trigger)
-	{
-		# Archiving is disabled.
-		return(0);
-	}
-	
 	# First, if this table doesn't have a history schema, exit.
 	my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'history' AND table_name = ".$anvil->data->{sys}{use_db_fh}->quote($table).";";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
@@ -5191,8 +5207,11 @@ sub _archive_table
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
 	
 	$count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-	if ($count <= $trigger)
+		"s1:count"                           => $count,
+		"s2:sys::database::archive::trigger" => $anvil->data->{sys}{database}{archive}{trigger},
+	}});
+	if ($count <= $anvil->data->{sys}{database}{archive}{trigger})
 	{
 		# History table doesn't exist, we're done.
 		return(0);

Anvil/Tools/Get.pm

@@ -462,25 +462,17 @@ sub host_uuid
 	if ($set)
 	{
 		$anvil->data->{HOST}{UUID} = $set;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "HOST::UUID" => $anvil->data->{HOST}{UUID} }});
 	}
 	elsif (not $anvil->data->{HOST}{UUID})
 	{
-		# Read dmidecode if I am root, and the cache if not.
+		# Read dmidecode if I am root, otherwise, read the cache.
 		my $uuid = "";
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { '$<' => $<, '$>' => $> }});
 		if (($< == 0) or ($> == 0))
 		{
-			my $shell_call = $anvil->data->{path}{exe}{dmidecode}." --string system-uuid";
+			$uuid = lc($anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{dmidecode}." --string system-uuid"}));
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
-			open(my $file_handle, $shell_call." 2>&1 |") or warn $THIS_FILE." ".__LINE__."; [ Warning ] - Failed to call: [".$shell_call."], the error was: $!\n";
-			while(<$file_handle>)
-			{
-				# This should never be hit...
-				chomp;
-				$uuid = lc($_);
-				#print $THIS_FILE." ".__LINE__."; [ Debug ] - UUID: [$uuid]\n";
-			}
-			close $file_handle;
 		}
 		else
 		{
@@ -495,7 +487,7 @@ sub host_uuid
 			}
 			else
 			{
-				$uuid = $anvil->Storage->read_file({ file => $anvil->data->{path}{data}{host_uuid} });
+				$uuid = $anvil->Storage->read_file({debug => $debug, file => $anvil->data->{path}{data}{host_uuid}});
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
 			}
 		}
@@ -509,6 +501,7 @@ sub host_uuid
 				# Apache run scripts can't call the system UUID, so we'll write it to a text
 				# file.
 				$anvil->Storage->write_file({
+					debug     => $debug, 
 					file      => $anvil->data->{path}{data}{host_uuid}, 
 					body      => $uuid,
 					user      => "apache", 
@@ -524,6 +517,7 @@ sub host_uuid
 			# Bad UUID.
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0134", variables => { uuid => $uuid }});
 			$anvil->data->{HOST}{UUID} = "";
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "HOST::UUID" => $anvil->data->{HOST}{UUID} }});
 		}
 	}
 	
@@ -534,6 +528,7 @@ sub host_uuid
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::host_uuid" => $anvil->data->{sys}{host_uuid} }});
 	}
 	
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "HOST::UUID" => $anvil->data->{HOST}{UUID} }});
 	return($anvil->data->{HOST}{UUID});
 }
 

Anvil/Tools/Log.pm

@@ -173,7 +173,6 @@ This is the 'verbose' log level. It will generally generate a significant amount
 
 This is the highest log level, and it will generate a tremendous amount of log entries. This is generally used is loops or recursive functions where the output is significant, but the usefulness of the output is not.
 
-
 =head3 line (optional)
 
 When set, the string is prepended to the log entry, after 'C<< file >> if set, and should be set to C<< __LINE__ >>. It is used to show where in 'C<< file >>' the log entry was made and can assist with debugging.
@@ -317,8 +316,10 @@ sub entry
 			my $log_file           = $anvil->data->{sys}{log_file} =~ /^\// ? $anvil->data->{sys}{log_file} : "/var/log/".$anvil->data->{sys}{log_file};
 			my ($directory, $file) = ($log_file =~ /^(\/.*)\/(.*)$/);
 			
+			### WARNING: We MUST set the debug level really high, or else we'll go into a deep 
+			###          recursion!
 			# Make sure the log directory exists.
-			$anvil->Storage->make_directory({directory => $directory, group => 755});
+			$anvil->Storage->make_directory({debug => 99, directory => $directory, mode => 755});
 			
 			# Now open the log
 			my $shell_call = $log_file;

Anvil/Tools/System.pm

@@ -9,6 +9,7 @@ use Data::Dumper;
 use Net::SSH2;
 use Scalar::Util qw(weaken isweak);
 use Time::HiRes qw(gettimeofday tv_interval);
+use Proc::Simple;
 
 our $VERSION  = "3.0.0";
 my $THIS_FILE = "System.pm";
@@ -100,6 +101,10 @@ This method makes a system call and returns the output (with the last new-line r
 
 Parameters;
 
+=head3 background (optional, default '0')
+
+If set to C<< 1 >>, the program will be started in the background and the C<< Proc::Simple >> handle will be returned instead of the command's output.
+
 =head3 line (optional)
 
 This is the line number of the source file that called this method. Useful for logging and debugging.
@@ -120,6 +125,18 @@ This is the shell command to call.
 
 This is the name of the source file calling this method. Useful for logging and debugging.
 
+=head3 stderr_file (optional)
+
+B<NOTE>: This is only used when C<< background >> is set to C<< 1 >>.
+
+If set, the C<< STDERR >> output will be sent to the corresponding file. If this isn't a full path, the file will be placed under C<< /tmp/ >>.
+
+=head3 stdout_file (optional)
+
+B<NOTE>: This is only used when C<< background >> is set to C<< 1 >>.
+
+If set, the C<< STDOUT >> output will be sent to the corresponding file. If this isn't a full path, the file will be placed under C<< /tmp/ >>.
+
 =cut
 sub call
 {
@@ -128,16 +145,22 @@ sub call
 	my $anvil     = $self->parent;
 	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
 	
+	my $background      = defined $parameter->{background}      ? $parameter->{background}      : 0;
 	my $line            = defined $parameter->{line}            ? $parameter->{line}            : __LINE__;
 	my $redirect_stderr = defined $parameter->{redirect_stderr} ? $parameter->{redirect_stderr} : 1;
 	my $shell_call      = defined $parameter->{shell_call}      ? $parameter->{shell_call}      : "";
 	my $secure          = defined $parameter->{secure}          ? $parameter->{secure}          : 0;
 	my $source          = defined $parameter->{source}          ? $parameter->{source}          : $THIS_FILE;
+	my $stderr_file     = defined $parameter->{stderr_file}     ? $parameter->{stderr_file}     : "";
+	my $stdout_file     = defined $parameter->{stdout_file}     ? $parameter->{stdout_file}     : "";
 	my $redirect        = $redirect_stderr ? " 2>&1" : "";
 	$anvil->Log->variables({source => $source, line => $line, level => $debug, secure => $secure, list => { 
+		background      => $background, 
 		shell_call      => $shell_call,
 		redirect        => $redirect, 
 		redirect_stderr => $redirect_stderr, 
+		stderr_file     => $stderr_file, 
+		stdout_file     => $stdout_file, 
 	}});
 	
 	my $output = "#!error!#";
@@ -175,21 +198,74 @@ sub call
 		if ($found)
 		{
 			# Make the system call
-			$output = "";
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, key => "log_0011", variables => { shell_call => $shell_call }});
-			open (my $file_handle, $shell_call.$redirect." |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => $secure, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }});
+			### TODO: We should split the arguments off, which the below does, to pass arguments 
-			while(<$file_handle>)
+			###       to the shell without having the shell expand the arguments
+			my $program   = $shell_call;
+			my $arguments = "";
+			if ($shell_call =~ /^(\/.*?) (.*)$/)
 			{
-				chomp;
+				$program   = $1;
-				my $line = $_;
+				$arguments = $2;
-				$line =~ s/\n$//;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { 
-				$line =~ s/\r$//;
+					"s1:program"   => $program,
-				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, key => "log_0017", variables => { line => $line }});
+					"s2:arguments" => $arguments, 
-				$output .= $line."\n";
+				}});
+			}
+			if ($background)
+			{
+				# Prepend '/tmp/' to STDOUT and/or STDERR output files, if needed.
+				if (($stderr_file) && ($stderr_file !~ /^\//))
+				{
+					$stderr_file = "/tmp/".$stderr_file;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { stderr_file => $stderr_file }});
+				}
+				if (($stdout_file) && ($stdout_file !~ /^\//))
+				{
+					$stdout_file = "/tmp/".$stdout_file;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { stdout_file => $stdout_file }});
+				}
+				my $process = Proc::Simple->new();
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { process => $process }});
+				
+				# Setup output files
+				if (($stderr_file) && ($stdout_file))
+				{
+					$process->($stdout_file, $stderr_file);
+				}
+				elsif ($stdout_file)
+				{
+					$process->($stdout_file, undef);
+				}
+				elsif ($stderr_file)
+				{
+					$process->(undef, $stderr_file);
+				}
+				
+				my $status = $process->start($shell_call);
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { status => $status }});
+				
+				# We'll return the handle instead of output.
+				$output = $process;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { output => $output }});
+			}
+			else
+			{
+				$output = "";
+				open (my $file_handle, $shell_call.$redirect." |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => $secure, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }});
+				while(<$file_handle>)
+				{
+					chomp;
+					my $line = $_;
+					$line =~ s/\n$//;
+					$line =~ s/\r$//;
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, key => "log_0017", variables => { line => $line }});
+					$output .= $line."\n";
+				}
+				close $file_handle;
+				chomp($output);
+				$output =~ s/\n$//s;
 			}
-			close $file_handle;
-			chomp($output);
-			$output =~ s/\n$//s;
 		}
 	}
 	

anvil.conf

@@ -64,15 +64,11 @@
 # 
 # NOTE: If the archive directory doesn't exist, Anvil! will create it 
 #       automatically the first time it is needed.
-sys::database::archive::compress		=	1
+#sys::database::archive::compress		=	1
-sys::database::archive::count			=	50000
+#sys::database::archive::count			=	50000
-sys::database::archive::directory		=	/usr/local/anvil/archives/
+#sys::database::archive::directory		=	/usr/local/anvil/archives/
-sys::database::archive::division		=	60000
+#sys::database::archive::division		=	60000
-sys::database::archive::trigger			=	100000
+#sys::database::archive::trigger		=	100000
-
-
-# This is the schema for the Anvil! database.
-sys::database::schema			=	/usr/share/anvil/anvil.sql
 
 # This puts a limit on how many queries (writes, generally) to make in a single batch transaction. This is 
 # useful when doing very large transacions, like resync'ing a large table, by limiting how long a given 

cgi-bin/striker

@@ -24,13 +24,11 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 	$running_directory =~ s/^\./$ENV{PWD}/;
 }
 
-my $anvil = Anvil::Tools->new();
+my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
 
 ### NOTE: We'll print the headers only when we need to. If we print them hear, it will block cookies being set.
 
 # Set the log level to 2. Setting 3 slows he program down a LOT.
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
 
 # Read the config and then connect to the database.
 $anvil->Storage->read_config();
@@ -53,12 +51,12 @@ if (not -e $anvil->data->{path}{data}{host_uuid})
 	print_and_exit($anvil);
 }
 
-my $connections = $anvil->Database->connect({
+$anvil->Database->connect({
 	sql_file   => $anvil->data->{sys}{database}{schema},
 	test_table => "network_interfaces", 
 });
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132", variables => { connections => $connections }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
-if (not $connections)
+if (not $anvil->data->{sys}{db_connections})
 {
 	# No databases, exit.
 	print $anvil->Words->string({key => "error_0003"});
@@ -575,6 +573,7 @@ sub add_sync_peer
 		if ($anvil->data->{cgi}{confirm}{value})
 		{
 			# OK, save!
+			
 		}
 		else
 		{

html/index.html

@@ -1,7 +1,7 @@
 <head>
 	<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 	<title>Striker</title>
-	<META HTTP-EQUIV="Refresh" CONTENT="0; URL=cgi-bin/home">
+	<META HTTP-EQUIV="Refresh" CONTENT="0; URL=cgi-bin/striker">
 </head>
 <body>
 	Hi, I'll be right with you...

rpm/SPECS/anvil.spec

@@ -39,6 +39,7 @@ Requires:       perl-JSON
 Requires:       perl-Log-Journald 
 Requires:       perl-Net-SSH2 
 Requires:       perl-NetAddr-IP 
+Requires:       perl-Proc-Simple;
 Requires:       perl-Sys-Syslog
 Requires:       perl-Time-HiRes
 Requires:       perl-XML-Simple 

share/words.xml

@@ -286,6 +286,8 @@ The database connection error was:
 		<key name="log_0185"><![CDATA[<unknown>]]></key>	<!-- Used in some cases when a variable isn't known -->
 		<key name="log_0186"><![CDATA[<suppressed>]]></key>	<!-- Used in some cases when a variable isn't known -->
 		<key name="log_0187">UUID cache file: [#!data!path::data::host_uuid!#] doesn't exists and we're not running as root. Unable to proceed.</key>
+		<key name="log_0188">Database archive check skipped, not running as root.</key>
+		<key name="log_0189">Database archiving is disabled, skipping archive checks.</key>
 		
 		<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
 		<key name="t_0000">Test</key>
@@ -419,6 +421,7 @@ Here we will inject 't_0006', which injects 't_0001' which has a variable: [#!st
 		<key name="striker_warning_0004">Accessing the peer over SSH worked, but a test connection to the database failed.</key>
 		<key name="striker_warning_0005">There was a problem reading the peer's UUID. Read: [#!variable!uuid!#], which appears to be invalid.</key>
 		<key name="striker_warning_0006">An SSH connection was established to: [#!variable!target!#], but we failed to establish a channel. The last error was: [#!variable!error!#].</key>
+		<key name="striker_warning_0007">The job: [#!variable!command!#] was picked up by: [#!variable!pid!#], but that process is not running and it appears to only be: [#!variable!percent!# %] complete. Restarting the job.</key>
 		
 		<!-- Errors -->
 		<key name="error_0001">There are not enough network interfaces on this machine. You have: [#!variable!interface_count!#] interface(s), and you need at least: [#!variable!required_interfaces_for_single!#] interfaces to connect to the requested networks (one for Back-Channel and one for each Internet-Facing network).</key>

tools/anvil-change-password

@@ -29,9 +29,7 @@ $| = 1;
 $< = $>;
 $( = $);
 
-my $anvil = Anvil::Tools->new();
+my $anvil = Anvil::Tools->new({log_level => 1, log_secure => 0});
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
 
 # Read switches
 $anvil->Get->switches;
@@ -49,9 +47,9 @@ if (($< != 0) && ($> != 0))
 }
 
 # Connect
-my $connections = $anvil->Database->connect();
+$anvil->Database->connect();
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { connections => $connections }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
-if (not $connections)
+if (not $anvil->data->{sys}{db_connections})
 {
 	# No databases, exit.
 	print $anvil->Words->string({key => "error_0003"})."\n";

tools/anvil-configure-striker

@@ -26,9 +26,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 # Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
 $| = 1;
 
-my $anvil = Anvil::Tools->new();
+my $anvil = Anvil::Tools->new({log_level => 1, log_secure => 1});
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
 
 # Read switches
 $anvil->data->{switches}{'no-reboot'} = 0;
@@ -47,14 +45,10 @@ if (($< != 0) && ($> != 0))
 }
 
 # Connect
-my $connections = $anvil->Database->connect({
+$anvil->Database->connect;
-	sql_file   => $anvil->data->{sys}{database}{schema},
-	test_table => "network_interfaces", 
-});
-
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0031"});
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132", variables => { connections => $connections }});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
-if (not $connections)
+if (not $anvil->data->{sys}{db_connections})
 {
 	# No databases, exit.
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "error_0003"});

tools/anvil-daemon

@@ -13,6 +13,7 @@
 use strict;
 use warnings;
 use Anvil::Tools;
+use Proc::Simple;
 
 my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
 my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
@@ -24,8 +25,12 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 # Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
 $| = 1;
 
-my $anvil = Anvil::Tools->new();
+my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
-$anvil->Log->level({set => 1});
+
+# Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks
+# is to setup the database server.
+$anvil->Database->connect();
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
 
 # There are some things we only want to run on (re)start and don't need to always run.
 run_once($anvil);
@@ -33,6 +38,10 @@ run_once($anvil);
 # Calculate my sum so that we can exit if it changes later.
 $anvil->Storage->record_md5sums;
 
+# Disconnect. We'll reconnect inside the loop
+$anvil->Database->disconnect;
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
+
 # These are the things we always want running.
 while(1)
 {
@@ -62,7 +71,8 @@ $anvil->nice_exit({code => 0});
 # Functions                                                                                                 #
 #############################################################################################################
 
-# These are tools that don't need to constantly run.
+# These are tools that don't need to constantly run. They'll typically run when the server starts up or the
+# daemon is restarted or reloaded.
 sub run_once
 {
 	my ($anvil) = @_;
@@ -84,12 +94,157 @@ sub keep_running
 {
 	my ($anvil) = @_;
 	
+	# Check for jobs that were running and now exited.
+	if (exists $anvil->data->{processes})
+	{
+		foreach my $job_uuid (%{$anvil->data->{jobs}{handles}})
+		{
+			# If it's not a handle, delete it.
+			my $running = $anvil->data->{jobs}{handles}{$job_uuid}->poll();
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 
+				"jobs::handles::${job_uuid}" => $anvil->data->{jobs}{handles}{$job_uuid},
+				running                      => $running, 
+			}});
+			
+			# If it's not running, update the table to clear the 'job_picked_up_by' column.
+			if (not $running)
+			{
+				my $exit_status = $anvil->data->{jobs}{handles}{$job_uuid}->exit_status();
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { exit_status => $exit_status }});
+				
+				# Free up memory
+				$anvil->data->{jobs}{handles}{$job_uuid}->cleanup();
+				
+				clear_job($anvil, $job_uuid);
+			}
+		}
+	}
+	
 	# Update hardware state files.
 	update_state_file($anvil);
 	
 	# Run any pending jobs by calling 'anvil-jobs' with the 'job_uuid' as a background process.
+	run_jobs($anvil);
+	
+	return(0);
+}
+
+# This clears the 'job_picked_up_by'.
+sub clear_job
+{
+	my ($anvil, $job_uuid) = @_;
+	
+	my $query = "
+UPDATE 
+    jobs 
+SET 
+    job_picked_up_by = '0', 
+    modified_date    = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{sys}{db_timestamp})." 
+WHERE 
+    job_uuid         = ".$anvil->data->{sys}{use_db_fh}->quote($job_uuid)." 
+";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+	$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
 	
-	# Check the local network and running services and tune the firewall as things change.
+	return(0);
+}
+
+# This will check for any jobs that aren't at 100%. For each found, if 'picked_up_by' is set, a check is made
+# to see if the PID is still alive. If it isn't, or if 'picked_up_by' is not set, the appropriate tool is
+# invoked to handle it.
+sub run_jobs
+{
+	my ($anvil) = @_;
+	
+	# Get a list of pending or incomplete jobs.
+	my $query = "
+SELECT 
+    job_uuid, 
+    job_command, 
+    job_data, 
+    job_picked_up_by, 
+    job_picked_up_at, 
+    job_updated, 
+    job_progress 
+FROM 
+    jobs 
+WHERE 
+    job_progress != 100 
+AND 
+    job_host_uuid = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->Get->host_uuid)."
+LIMIT 1;";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+	
+	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
+	my $count   = @{$results};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		results => $results, 
+		count   => $count,
+	}});
+	foreach my $row (@{$results})
+	{
+		my $job_uuid         =         $row->[0];
+		my $job_command      =         $row->[1];
+		my $job_data         = defined $row->[2] ? $row->[2] : "";
+		my $job_picked_up_by =         $row->[3];
+		my $job_picked_up_at =         $row->[4]; 
+		my $job_updated      =         $row->[5];
+		my $job_progress     =         $row->[6];
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			job_uuid         => $job_uuid,
+			job_command      => $job_command,
+			job_data         => $job_data,
+			job_picked_up_by => $job_picked_up_by,
+			job_picked_up_at => $job_picked_up_at,
+			job_updated      => $job_updated,
+			job_progress     => $job_progress,
+		}});
+		
+		# See if the job was picked up by another running instance.
+		if ($job_picked_up_by)
+		{
+			# Check if the PID is still active.
+			$anvil->System->pids({ignore_me => 1});
+			
+			### TODO: Add a check to verify the job isn't hung.
+			# Skip if this job is in progress.
+			next if exists $anvil->data->{pids}{$job_picked_up_by};
+
+			# The previous job is gone, but the job isn't finished. Start it again.
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "striker_warning_0007", variables => { 
+				command => $job_command,
+				pid     => $job_picked_up_by,
+				percent => $job_progress, 
+			}});
+			clear_job($anvil, $job_uuid);
+		}
+		
+		# Start the job, appending '--job-uuid' to the command.
+		$anvil->data->{jobs}{handles}{$job_uuid} = $anvil->System->call({
+			debug       => 2,
+			background  => 1, 
+			stdout_file => "/tmp/anvil.job.".$job_uuid.".stdout",
+			stderr_file => "/tmp/anvil.job.".$job_uuid.".stderr", 
+			shell_call  => $job_command." --job-uuid ".$job_uuid, 
+			source      => $THIS_FILE, 
+			line        => __LINE__});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "jobs::handles::${job_uuid}" => $anvil->data->{jobs}{handles}{$job_uuid} }});
+		
+		# Record the PID
+		my $pid = $anvil->data->{jobs}{handles}{$job_uuid}->pid();
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { pid => $pid }});
+		my $query = "
+UPDATE 
+    jobs 
+SET 
+    job_picked_up_by = ".$anvil->data->{sys}{use_db_fh}->quote($pid).", 
+    modified_date    = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{sys}{db_timestamp})." 
+WHERE 
+    job_uuid         = ".$anvil->data->{sys}{use_db_fh}->quote($job_uuid)." 
+";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+		$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
+	}
 	
 	return(0);
 }

tools/anvil-jobs

@@ -23,8 +23,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 # Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
 $| = 1;
 
-my $anvil = Anvil::Tools->new();
+my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
-$anvil->Log->level({set => 2});
 
 # I should have been passed a 'job_uuid'. Read in the details.
 

tools/anvil-prep-database

@@ -26,9 +26,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 # Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
 $| = 1;
 
-my $anvil = Anvil::Tools->new();
+my $anvil = Anvil::Tools->new({log_level => 1, log_secure => 0});
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 0});
 
 # Read switches
 $anvil->Get->switches;

tools/anvil-scan-network

@@ -28,8 +28,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 # Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
 $| = 1;
 
-my $anvil = Anvil::Tools->new();
+my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
-$anvil->Log->level({set => 2});
 
 scan($anvil);
 

tools/anvil-update-states

@@ -18,18 +18,13 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 	$running_directory =~ s/^\./$ENV{PWD}/;
 }
 
-my $anvil = Anvil::Tools->new();
+my $anvil = Anvil::Tools->new({log_level => 1, log_secure => 1});
-$anvil->Log->level({set => 2});
 
 $anvil->Storage->read_config({file => "/etc/anvil/anvil.conf"});
-my $connections = $anvil->Database->connect({
+$anvil->Database->connect;
-	sql_file   => $anvil->data->{sys}{database}{schema},
-	test_table => "network_interfaces", 
-});
 
-print $THIS_FILE." ".__LINE__."; connections: [".$connections."]\n";
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0132", variables => { connections => $connections }});
+if (not $anvil->data->{sys}{db_connections})
-if (not $connections)
 {
 	# No databases, exit.
 	print $anvil->Words->string({key => "error_0003"})."\n";