Merge pull request #660 from ClusterLabs/net-config

Net config

Anvil/Tools.pm

@@ -428,6 +428,9 @@ sub nice_exit
 		$anvil->data->{HANDLE}{'log'}{main} = "";
 	}
 	
+	# Call a disk sync.
+	system($anvil->data->{path}{exe}{sync});
+	
 	#print "Exiting with RC: [".$exit_code."]\n";
 	exit($exit_code);
 }
@@ -1320,6 +1323,7 @@ sub _set_paths
 				su				=>	"/usr/bin/su",
 				'subscription-manager'		=>	"/usr/sbin/subscription-manager",
 				swapon				=>	"/usr/sbin/swapon",
+				sync				=>	"/usr/bin/sync",
 				sysctl				=>	"/usr/sbin/sysctl",
 				systemctl			=>	"/usr/bin/systemctl",
 				tail				=>	"/usr/bin/tail",

Anvil/Tools/Cluster.pm

@@ -1251,7 +1251,7 @@ sub check_stonith_config
 				pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command,
 			}});
 			
-			# If there's an entry in the CIB, so if it's different somehow
+			# If there's an entry in the CIB, see if it's different somehow
 			if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name})
 			{
 				foreach my $argument (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name}{argument}})

Anvil/Tools/Database.pm

@@ -19,6 +19,7 @@ my $THIS_FILE = "Database.pm";
 # archive_database
 # backup_database
 # check_file_locations
+# check_hosts
 # check_lock_age
 # check_for_schema
 # configure_pgsql
@@ -439,6 +440,115 @@ sub check_file_locations
 	return(0);
 }
 
+=head2 check_hosts
+
+This checks to see if there's an entry in the C<< hosts >> table on each database. This is meant to avoid an INSERT on a table with a record already, wich can happen when programs start before initial sync.
+
+This method takes no parameters.
+
+=cut
+sub check_hosts
+{
+	my $self      = shift;
+	my $parameter = shift;
+	my $anvil     = $self->parent;
+	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->check_hosts()" }});
+
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		"sys::database::connections" => $anvil->data->{sys}{database}{connections},
+	}});
+	if ($anvil->data->{sys}{database}{connections} < 1)
+	{
+		# Nothing to do.
+		return(0);
+	}
+	
+	# If we're starting with a new database, which is not yet in the hosts table, we can hit a case where
+	# this tries to insert into both DBs when it's only missing from one. So to habdle that, we'll 
+	# manually check each DB to see if all hosts are there and, if not, INSERT only into the needed DB.
+	foreach my $db_uuid (sort {$a cmp $b} keys %{$anvil->data->{database}})
+	{
+		# Are we connected?
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			"db_status::${db_uuid}::access" => $anvil->data->{db_status}{$db_uuid}{access},
+		}});
+		next if not $anvil->data->{db_status}{$db_uuid}{access};
+		
+		# Get the host information from the host.
+		my $query = "
+SELECT 
+    host_ipmi, 
+    host_name, 
+    host_type, 
+    host_key, 
+    host_status 
+FROM 
+    hosts 
+WHERE 
+    host_uuid = ".$anvil->Database->quote($db_uuid)."
+;";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
+		
+		my $results = $anvil->Database->query({uuid => $db_uuid, query => $query, source => $THIS_FILE, line => __LINE__});
+		my $count   = @{$results};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			results => $results, 
+			count   => $count,
+		}});
+		next if not $count;
+		my $db_host_ipmi   = $results->[0]->[0];
+		my $db_host_name   = $results->[0]->[1];
+		my $db_host_type   = $results->[0]->[2];
+		my $db_host_key    = $results->[0]->[3];
+		my $db_host_status = $results->[0]->[4];
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
+			db_host_ipmi   => $db_host_ipmi =~ /passw/ ? $anvil->Log->is_secure($db_host_ipmi) : $db_host_ipmi,
+			db_host_name   => $db_host_name, 
+			db_host_type   => $db_host_type, 
+			db_host_key    => $db_host_key, 
+			db_host_status => $db_host_status, 
+		}});
+		
+		# Is this host in all DBs?
+		foreach my $check_uuid (sort {$a cmp $b} keys %{$anvil->data->{database}})
+		{
+			# Are we connected?
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				"db_status::${check_uuid}::access" => $anvil->data->{db_status}{$check_uuid}{access},
+			}});
+			next if not $anvil->data->{db_status}{$check_uuid}{access};
+			
+			my $query = "SELECT COUNT(*) FROM hosts WHERE host_uuid = ".$anvil->Database->quote($check_uuid).";";
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
+			
+			my $results = $anvil->Database->query({uuid => $check_uuid, query => $query, source => $THIS_FILE, line => __LINE__});
+			my $count   = @{$results};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				results => $results, 
+				count   => $count,
+			}});
+			
+			if (not $count)
+			{
+				# INSERT it!
+				$anvil->Database->insert_or_update_hosts({
+					debug       => 2,
+					uuid        => $check_uuid, 
+					host_ipmi   => $db_host_ipmi, 
+					host_key    => $db_host_key, 
+					host_name   => $db_host_name, 
+					host_type   => $db_host_type, 
+					host_uuid   => $db_uuid, 
+					host_status => $db_host_status, 
+				});
+			}
+		}
+	}
+	
+	return(0);
+}
+
 
 =head2 check_lock_age
 
@@ -1361,8 +1471,8 @@ sub connect
 	}
 	
 	# If we're a Striker, see if we're configured.
-	my $local_host_type = $anvil->Get->host_type();
+	my $local_host_type = $anvil->Get->host_type({debug => $debug});
-	my $local_host_uuid = $anvil->Get->host_uuid();
+	my $local_host_uuid = $anvil->Get->host_uuid({debug => $debug});
 	my $db_count        = keys %{$anvil->data->{database}};
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		local_host_type     => $local_host_type, 
@@ -2143,6 +2253,12 @@ sub connect
 		$anvil->Database->disconnect({debug => $debug});
 	}
 	
+	if ($local_host_type eq "striker")
+	{
+		# More sure any configured databases are in the hosts file.
+		$anvil->Database->check_hosts({debug => $debug});
+	}
+	
 	# If this is a time sensitive call, end here.
 	if ($sensitive)
 	{
@@ -2153,17 +2269,19 @@ sub connect
 		return($anvil->data->{sys}{database}{connections});
 	}
 	
-	# If 'check_for_resync' is set to '2', then only check if we're primary.
+	# If 'check_for_resync' is set to '2', and the uptime is over two hours, only check if we're primary.
+	my $uptime = $anvil->Get->uptime();
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
 		"sys::database::primary_db" => $anvil->data->{sys}{database}{primary_db},
 		"sys::host_uuid"            => $anvil->data->{sys}{host_uuid},
 		check_for_resync            => $check_for_resync, 
+		uptime                      => $uptime,
 	}});
 	if ($check_for_resync == 2)
 	{
-		if ($anvil->data->{sys}{database}{primary_db} eq $anvil->data->{sys}{host_uuid})
+		if (($uptime < 7200) or ($anvil->data->{sys}{database}{primary_db} eq $anvil->data->{sys}{host_uuid}))
 		{
-			# We're primary.
+			# We're primary or the uptime is low.
 			$check_for_resync = 1;
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { check_for_resync => $check_for_resync }});
 		}
@@ -6473,12 +6591,14 @@ INSERT INTO
     storage_group_member_storage_group_uuid, 
     storage_group_member_host_uuid, 
     storage_group_member_vg_uuid, 
+    storage_group_member_note, 
     modified_date 
 ) VALUES (
     ".$anvil->Database->quote($storage_group_member_uuid).", 
     ".$anvil->Database->quote($storage_group_uuid).", 
     ".$anvil->Database->quote($this_host_uuid).", 
     ".$anvil->Database->quote($closest_scan_lvm_vg_uuid).", 
+    'auto-created',
     ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
 );";
 						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
@@ -9853,9 +9973,9 @@ INSERT INTO
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query =~ /passw/ ? $anvil->Log->is_secure($query) : $query }});
 		$anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
 	}
-	elsif (($old_host_name   ne $host_name)   or 
+	elsif (($old_host_name   ne $host_name) or 
-	       ($old_host_type   ne $host_type)   or 
+	       ($old_host_type   ne $host_type) or 
-	       ($old_host_key    ne $host_key)    or 
+	       ($old_host_key    ne $host_key)  or 
 	       ($old_host_status ne $host_status))
 	{
 		# Clear the stop data.
@@ -10612,6 +10732,7 @@ AND
 			{
 				$found = 1;
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { found => $found }});
+				last;
 			}
 		}
 		if (not $found)
@@ -10656,7 +10777,7 @@ INSERT INTO
 );
 ";
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
-		$anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
+		$anvil->Database->write({debug => $debug, uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
 	}
 	else
 	{
@@ -17248,7 +17369,7 @@ sub purge_data
 
 This performs a query and returns an array reference of array references (from C<< DBO->fetchall_arrayref >>). The first array contains all the returned rows and each row is an array reference of columns in that row.
 
-If an error occurs, C<< !!error!! >> will be returned.
+If an error occurs, an empty array reference is returned.
 
 For example, given the query;
 
@@ -17287,7 +17408,7 @@ Parameters;
 
 By default, the local database will be queried (if run on a machine with a database). Otherwise, the first database successfully connected to will be used for queries (as stored in C<< $anvil->data->{sys}{database}{read_uuid} >>).
 
-If you want to read from a specific database, though, you can set this parameter to the ID of the database (C<< database::<id>::host). If you specify a read from a database that isn't available, C<< !!error!! >> will be returned.
+If you want to read from a specific database, though, you can set this parameter to the ID of the database (C<< database::<id>::host). If you specify a read from a database that isn't available, An empty array reference will be returned.
 
 =head3 line (optional)
 
@@ -17307,6 +17428,12 @@ If set, the query will be treated as containing sensitive data and will only be
 
 To help with logging the source of a query, C<< source >> can be set to the name of the script that requested the query. It is generally used along side C<< line >>.
 
+=head3 timeout (optional, default 30)
+
+This sets a timeout on the execution of the query. If the query doesn't return in the set time, the query will be aborted and An empty array reference will be returned. 
+
+Set to C<< 0 >> to set no / infinite timeout.
+
 =cut
 sub query
 {
@@ -17316,20 +17443,35 @@ sub query
 	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->query()" }});
 	
-	my $uuid   = $parameter->{uuid}   ? $parameter->{uuid}   : $anvil->data->{sys}{database}{read_uuid};
+	my $uuid    =         $parameter->{uuid}    ? $parameter->{uuid}    : "";
-	my $line   = $parameter->{line}   ? $parameter->{line}   : __LINE__;
+	my $line    =         $parameter->{line}    ? $parameter->{line}    : __LINE__;
-	my $query  = $parameter->{query}  ? $parameter->{query}  : "";
+	my $query   =         $parameter->{query}   ? $parameter->{query}   : "";
-	my $secure = $parameter->{secure} ? $parameter->{secure} : 0;
+	my $secure  =         $parameter->{secure}  ? $parameter->{secure}  : 0;
-	my $source = $parameter->{source} ? $parameter->{source} : $THIS_FILE;
+	my $source  =         $parameter->{source}  ? $parameter->{source}  : $THIS_FILE;
+	my $timeout = defined $parameter->{timeout} ? $parameter->{timeout} : 30;
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		uuid                              => $uuid, 
-		"cache::database_handle::${uuid}" => $anvil->data->{cache}{database_handle}{$uuid}, 
+		"cache::database_handle::${uuid}" => $uuid ? $anvil->data->{cache}{database_handle}{$uuid} : "", 
 		line                              => $line, 
 		query                             => (not $secure) ? $query : $anvil->Log->is_secure($query), 
 		secure                            => $secure, 
 		source                            => $source, 
+		timeout                           => $timeout, 
 	}});
 	
+	# Use the default read_uuid if a specific UUID wasn't specified.
+	my $used_read_uuid = 0;
+	if (not $uuid)
+	{
+		$uuid           = $anvil->data->{sys}{database}{read_uuid};
+		$used_read_uuid = 1;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			"s1:uuid"                            => $uuid, 
+			"s2:used_read_uuid"                  => $used_read_uuid, 
+			"s3:cache::database_handle::${uuid}" => $anvil->data->{cache}{database_handle}{$uuid}, 
+		}});
+	}
+	
 	# Make logging code a little cleaner
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		"s1:database::${uuid}::name"  => $anvil->data->{database}{$uuid}{name}, 
@@ -17345,6 +17487,9 @@ sub query
 		"s2:say_server"    => $say_server, 
 	}});
 	
+	my $failed_array_ref = [];
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed_array_ref => $failed_array_ref }});
+	
 	if (not $uuid)
 	{
 		# No database to talk to...
@@ -17353,7 +17498,9 @@ sub query
 			source => $source, 
 			line   => $line, 
 		}});
-		return("!!error!!");
+		
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed_array_ref => $failed_array_ref }});
+		return($failed_array_ref);
 	}
 	elsif (not defined $anvil->data->{cache}{database_handle}{$uuid})
 	{
@@ -17367,7 +17514,8 @@ sub query
 		if (not defined $anvil->data->{cache}{database_handle}{$uuid})
 		{
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0073", variables => { uuid => $uuid }});
-			return("!!error!!");
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed_array_ref => $failed_array_ref }});
+			return($failed_array_ref);
 		}
 		else
 		{
@@ -17381,14 +17529,40 @@ sub query
 	if (not $query)
 	{
 		# No query
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0084", variables => { 
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "warn", key => "log_0084", variables => { server => $say_server }});
-			server => $say_server,
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed_array_ref => $failed_array_ref }});
-		}});
+		return($failed_array_ref);
-		return("!!error!!");
 	}
 	
 	# Test access to the DB before we do the actual query
-	$anvil->Database->_test_access({debug => $debug, uuid => $uuid});
+	my $problem = $anvil->Database->_test_access({debug => $debug, uuid => $uuid});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }});
+	if ($problem)
+	{
+		if ($used_read_uuid)
+		{
+			# Switch to the new read_uuid, if possible,
+			if ($anvil->data->{sys}{database}{read_uuid})
+			{
+				$uuid = $anvil->data->{sys}{database}{read_uuid};
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
+			}
+			else
+			{
+				# No usable databases are available.
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "warn", key => "warning_0181", variables => { server => $say_server }});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed_array_ref => $failed_array_ref }});
+				return($failed_array_ref);
+			}
+		}
+		else
+		{
+			# We were given a specific UUID, and we can't read from it. Return an error.
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "warn", key => "warning_0180", variables => { server => $say_server }});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed_array_ref => $failed_array_ref }});
+			return($failed_array_ref);
+		}
+	}
 	
 	# If I am still alive check if any locks need to be renewed.
 	$anvil->Database->check_lock_age({debug => $debug});
@@ -17404,6 +17578,15 @@ sub query
 			query => $query, 
 		}});
 	}
+
+	### TODO: Remove this before pr/660 is released.
+	# Trying to see how the handle changes if/when the handle is lost.
+	foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}})
+	{
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
+			"cache::database_handle::${uuid}" => $anvil->data->{cache}{database_handle}{$uuid},
+		}});
+	}
 	
 	# Do the query.
 	local $@;
@@ -17412,16 +17595,47 @@ sub query
 			server   => $say_server,
 			db_error => $DBI::errstr, 
 		}}); };
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'eval_error' => $@ }});
 	if ($@)
 	{
 		### TODO: Report back somehow that the handle is dead.
-		$anvil->Database->disconnect({debug => $debug});
+		my $connections = $anvil->Database->reconnect({
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0675", variables => { 
+			debug     => $debug,
-			query      => (not $secure) ? $query : $anvil->Log->is_secure($query), 
+			lost_uuid => $uuid, 
-			server     => $say_server,
+		});
-			eval_error => $@, 
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { connections => $connections }});
-		}});
+		if ($connections)
-		return("!!error!!");
+		{
+			# Try the prepare again
+			$DBreq = eval { $anvil->data->{cache}{database_handle}{$uuid}->prepare($query) or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0075", variables => {
+				query    => (not $secure) ? $query : $anvil->Log->is_secure($query),
+				server   => $say_server,
+				db_error => $DBI::errstr,
+			}}); };
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'eval_error' => $@ }});
+			if ($@)
+			{
+				# No luck, we're dead
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0675", variables => {
+					query      => (not $secure) ? $query : $anvil->Log->is_secure($query),
+					server     => $say_server,
+					eval_error => $@,
+				}});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed_array_ref => $failed_array_ref }});
+				return($failed_array_ref);
+			}
+		}
+		else
+		{
+			# No luck, we're dead
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0675", variables => {
+				query      => (not $secure) ? $query : $anvil->Log->is_secure($query),
+				server     => $say_server,
+				eval_error => $@,
+			}});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed_array_ref => $failed_array_ref }});
+			return($failed_array_ref);
+		}
 	}
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		uuid       => $uuid, 
@@ -17431,11 +17645,39 @@ sub query
 	}});
 	
 	# Execute on the query
-	$DBreq->execute() or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0076", variables => { 
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { timeout => $timeout }});
+	alarm($timeout);
+	eval {
+		$DBreq->execute() or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0076", variables => { 
 			query    => (not $secure) ? $query : $anvil->Log->is_secure($query), 
 			server   => $say_server,
 			db_error => $DBI::errstr, 
-		}});
+		}}); 
+	};
+	alarm(0);
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'alarm $@' => $@ }});
+	if ($@)
+	{
+		if (($@ =~ /time/i) && ($@ =~ /out/i))
+		{
+			# Timed out 
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "warning_0175", variables => { 
+				query   => (not $secure) ? $query : $anvil->Log->is_secure($query),
+				timeout => $timeout, 
+				error   => $@,
+			}});
+		}
+		else
+		{
+			# Other error
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "warning_0175", variables => { 
+				query => (not $secure) ? $query : $anvil->Log->is_secure($query),
+				error => $@,
+			}});
+		}
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed_array_ref => $failed_array_ref }});
+		return($failed_array_ref);
+	}
 	
 	# Return the array
 	return($DBreq->fetchall_arrayref());
@@ -17446,6 +17688,8 @@ sub query
 
 This quotes a string for safe use in database queries/writes. It operates exactly as C<< DBI >>'s C<< quote >> method. This method is simply a wrapper that uses the C<< DBI >> handle set as the currently active read database.
 
+If there is a problem, an empty string will be returned and an error will be logged and printed to STDOUT.
+
 Example;
 
  $anvil->Database->quote("foo");
@@ -17461,8 +17705,22 @@ sub quote
 	my $string = shift;
 	my $anvil  = $self->parent;
 	
-	   $string = "" if not defined $string;
+	$string = "" if not defined $string;
-	my $quoted = $anvil->Database->read->quote($string);
+	   
+	# Make sure we're using an active handle.
+	my $quoted = eval {$anvil->Database->read->quote($string); };
+	if ($@)
+	{
+		$quoted = "" if not defined $quoted;
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "warning_0177", variables => { 
+			string => $string,
+			error  => $@,
+		}});
+		
+		# Given this might be about to get used in a DB query, return nothing. That should cause 
+		# whatever query this was called for to error safely.
+		return("");
+	}
 	
 	return($quoted);
 }
@@ -17764,6 +18022,58 @@ AND
 }
 
 
+=head2 reconnect
+
+This method disconnects from any connected databases, re-reads the config, and then tries to reconnect to any databases again. The number of connected datbaases is returned.
+
+Parameters;
+
+=head3 lost_uuid (optional)
+
+If set to a database UUID, then the database handle is deleted before the disconnect method is called, preventing an attempt to update locks and state information on a dead DB connection.
+
+=cut
+sub reconnect
+{
+	my $self      = shift;
+	my $parameter = shift;
+	my $anvil     = $self->parent;
+	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->reconnect()" }});
+	
+	my $lost_uuid = defined $parameter->{lost_uuid} ? $parameter->{lost_uuid} : "";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		lost_uuid => $lost_uuid, 
+	}});
+	
+	if (($lost_uuid) && ($anvil->data->{cache}{database_handle}{$lost_uuid}))
+	{
+		$anvil->data->{cache}{database_handle}{$lost_uuid} = "";
+		$anvil->data->{sys}{database}{connections}-- if $anvil->data->{sys}{database}{connections} > 0;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			"s1:cache::database_handle::${lost_uuid}" => $anvil->data->{cache}{database_handle}{$lost_uuid}, 
+			"s2:sys::database::connections"           => $anvil->data->{sys}{database}{connections}, 
+		}});
+	}
+
+	# Disconnect from all databases and then stop the daemon, then reconnect.
+	$anvil->Database->disconnect({debug => $debug});
+	sleep 2;
+
+	# Refresh configs.
+	$anvil->refresh();
+
+	# Reconnect.
+	$anvil->Database->connect({debug => $debug});
+
+	# Log our connection count.
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
+		"sys::database::connections" => $anvil->data->{sys}{database}{connections},
+	}});
+	return($anvil->data->{sys}{database}{connections});
+}
+
+
 =head2 refresh_timestamp
 
 This refreshes C<< sys::database::timestamp >>. It returns C<< sys::database::timestamp >> as well.
@@ -19111,7 +19421,16 @@ sub write
 		if (not $initializing)
 		{
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
-			$anvil->Database->_test_access({debug => $debug, uuid => $uuid});
+			
+			my $problem = $anvil->Database->_test_access({debug => $debug, uuid => $uuid});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }});
+			
+			if ($problem)
+			{
+				# We can't use this DB. 
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "warn", key => "warning_0182", variables => { uuid => $uuid }});
+				next;
+			}
 		}
 		
 		# Do the actual query(ies)
@@ -19140,14 +19459,15 @@ sub write
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0089", variables => { uuid => $uuid }});
 				next;
 			}
-			
+
 			# Do the do. Do it in an eval block though so that if it fails, we can do something 
 			# useful.
 			my $test = eval { $anvil->data->{cache}{database_handle}{$uuid}->do($query); };
 			   $test = "" if not defined $test;
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-				's1:test' => $test,
+				's1:test'  => $test,
-				's2:$@'   => $@,
+				's2:$@'    => $@,
+				's3:query' => $query, 
 			}});
 			
 			if (not $test)
@@ -20417,11 +20737,11 @@ sub _mark_database_as_behind
 
 =head2 _test_access
 
-This method takes a database UUID and tests the connection to it using the DBD 'ping' method. If it fails, open references to the database are removed or replaced, then an attempt to reconnect is made.
+This method takes a database UUID and tests the connection to it using the DBD 'ping' method. If it fails, the database connections will be refreshed. If after this there is still no connection, C<< 1 >> is returned. If the connection is up (immediately or after reconnect), C<< 0 >> is returned.
 
 This exists to handle the loss of a database mid-run where a normal query, which isn't wrapped in a query, could hang indefinately.
 
-B<< Note >>: If there is no active handle, this returns 0 immediately.
+B<< Note >>: If there is no active handle, this returns C<< 1 >> immediately without trying to reconnect.
 
 =cut
 sub _test_access
@@ -20439,9 +20759,11 @@ sub _test_access
 	}});
 	
 	# If the handle is down, return 0.
+	my $problem = 1;
 	if ((not exists $anvil->data->{cache}{database_handle}{$uuid}) or (not $anvil->data->{cache}{database_handle}{$uuid}))
 	{
-		return(0);
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }});
+		return($problem);
 	}
 	
 	# Make logging code a little cleaner
@@ -20451,28 +20773,6 @@ sub _test_access
 	# Log our test
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0087", variables => { server => $say_server }});
 	
-	# TODO: Is there a use for this anymore?
-	if (0)
-	{
-		# Ping works. Try a quick test query.
-		my $query = "SELECT 1";
-		my $DBreq = $anvil->data->{cache}{database_handle}{$uuid}->prepare($query) or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0075", variables => { 
-				query    => $query, 
-				server   => $say_server,
-				db_error => $DBI::errstr, 
-			}});
-		
-		# Give the test query a few seconds to respond, just in case we have some latency to a remote DB.
-		alarm(10);
-		$DBreq->execute() or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0076", variables => { 
-				query    => $query, 
-				server   => $say_server,
-				db_error => $DBI::errstr, 
-			}});
-		# If we're here, we made contact.
-		alarm(0);
-	}
-	
 	# Check using ping. Returns '1' on success, '0' on fail.
 	alarm(120);
 	my $connected = $anvil->data->{cache}{database_handle}{$uuid}->ping();
@@ -20480,107 +20780,78 @@ sub _test_access
 	alarm(0);
 	if (not $connected)
 	{
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0192", variables => { server => $say_server }});
+		$anvil->data->{sys}{in_test_access} = 0 if not defined $anvil->data->{sys}{in_test_access};
-		
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::in_test_access" => $anvil->data->{sys}{in_test_access} }});
-		# Try to reconnect.
+		if (not $anvil->data->{sys}{in_test_access})
-		$anvil->data->{sys}{database}{connections}--;
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::connections" => $anvil->data->{sys}{database}{connections} }});
-		
-		# If this was the DB we were reading from or that the use_db_handle matches, and another DB 
-		# appears to still be up, switch to one of the others.
-		if ($anvil->data->{sys}{database}{connections})
 		{
+			# This prevents deep recursion
+			$anvil->data->{sys}{in_test_access} = 1;
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-				"sys::database::use_handle"       => $anvil->Database->read,
+				"sys::in_test_access" => $anvil->data->{sys}{in_test_access},
+			}});
+			
+			# Tell the user we're going to try to reconnect
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0192", variables => { server => $say_server }});
+			
+			# Try to reconnect.
+			$anvil->Database->reconnect({
+				debug     => $debug,
+				lost_uuid => $uuid, 
+			});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				"sys::database::connections"      => $anvil->data->{sys}{database}{connections},
 				"cache::database_handle::${uuid}" => $anvil->data->{cache}{database_handle}{$uuid},
 			}});
-			if ($anvil->Database->read eq $anvil->data->{cache}{database_handle}{$uuid})
+			
-			{
+			$anvil->data->{sys}{in_test_access} = 0;
-				foreach my $this_uuid (keys %{$anvil->data->{cache}{database_handle}})
-				{
-					# We don't test this connection because, if it's down, we'll know 
-					# when it is tested.
-					my $database_name = defined $anvil->data->{database}{$this_uuid}{name} ? $anvil->data->{database}{$this_uuid}{name} : "anvil";
-					my $say_server    = $anvil->data->{database}{$this_uuid}{host}.":".$anvil->data->{database}{$this_uuid}{port}." -> ".$database_name;
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0193", variables => { server => $say_server }});
-
-					$anvil->Database->read({set => $anvil->data->{cache}{database_handle}{$this_uuid}});
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'anvil->Database->read' => $anvil->Database->read }});
-					last;
-				}
-			}
-
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-				uuid                       => $uuid,
+				"s1:sys::in_test_access"             => $anvil->data->{sys}{in_test_access}, 
-				"sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid},
+				"s2:cache::database_handle::${uuid}" => $anvil->data->{cache}{database_handle}{$uuid}, 
 			}});
-			if ($uuid eq $anvil->data->{sys}{database}{read_uuid})
+			
+			if ($anvil->data->{cache}{database_handle}{$uuid})
 			{
-				# We were reading from this DB, switch.
+				alarm(120);
-				foreach my $this_uuid (keys %{$anvil->data->{cache}{database_handle}})
+				my $connected = $anvil->data->{cache}{database_handle}{$uuid}->ping();
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { connected => $connected }});
+				alarm(0);
+				
+				if ($connected)
 				{
-					# We don't test this connection because, if it's down, we'll know 
+					# We reconnected.
-					# when it is tested.
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0854", variables => { server => $say_server }});
-					my $database_name = defined $anvil->data->{database}{$this_uuid}{name} ? $anvil->data->{database}{$this_uuid}{name} : "anvil";
+					$problem = 0;
-					my $say_server    = $anvil->data->{database}{$this_uuid}{host}.":".$anvil->data->{database}{$this_uuid}{port}." -> ".$database_name;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }});
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0194", variables => { server => $say_server }});
+					return($problem);
-
+				}
-					$anvil->data->{sys}{database}{read_uuid} = $this_uuid;
+				else
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid} }});
+				{
-					last;
+					# The tartget DB is gone.
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "warning_0179", variables => { server => $say_server }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }});
+					return($problem);
 				}
 			}
-			
+			else
+			{
+				# The tartget DB is gone.
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "warning_0179", variables => { server => $say_server }});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }});
+				return($problem);
+			}
 		}
 		else
 		{
-			# We're in trouble if we don't reconnect...
+			# No luck.
-			$anvil->Database->read({set => "delete"});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "warning_0179", variables => { server => $say_server }});
-			$anvil->data->{sys}{database}{read_uuid} = "";
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }});
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			return($problem);
-				'anvil->Database->read'    => $anvil->Database->read, 
-				"sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid},
-			}});
-			
-		}
-		
-		# Delete the old handle and then try to reconnect. If the reconnect succeeds, and this is the
-		# local database, this database will be re-selected as default for reads.
-		delete $anvil->data->{cache}{database_handle}{$uuid};
-		
-		my $delay = 5;
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0195", variables => { 
-			delay  => $delay,
-			server => $say_server,
-		}});
-		sleep $delay;
-		$anvil->Database->connect({debug => $debug, db_uuid => $uuid});
-		
-		# If we're down to '0' databases, error out.
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::connections" => $anvil->data->{sys}{database}{connections} }});
-		if (not $anvil->data->{sys}{database}{connections})
-		{
-			# No connections are left.
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0366"});
-			
-			# It's possible the network was just reconfigured, and they were trying to updated a 
-			# job in the database. If so, this failure can be hit. To handle this, we'll check 
-			# if 'sys::reboot' is set. If so, we'll reboot now.
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::reboot" => $anvil->data->{sys}{reboot} }});
-			if ($anvil->data->{sys}{reboot})
-			{
-				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0196"});
-				my $shell_call = $anvil->data->{path}{exe}{systemctl}." reboot";
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
-				my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
-			}
-			return(1);
 		}
 	}
 	
 	# Success!
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0088"});
-	
+	$problem = 0;
-	return(0);
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }});
+	return($problem);
 }

Anvil/Tools/Get.pm

@@ -1454,7 +1454,7 @@ AND
     b.ip_address_address =  ".$anvil->Database->quote($ip_address)."
 ;";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
-	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
+	my $results = $anvil->Database->query({debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__});
 	my $count   = @{$results};
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		results => $results, 

Anvil/Tools/Log.pm

@@ -544,6 +544,10 @@ sub entry
 			
 			# The handle has to be wrapped in a block to make 'print' happy as it doesn't like non-scalars for file handles
 			print { $anvil->data->{HANDLE}{'log'}{alert} } $log_to_alert;
+			
+			### NOTE: uncheck this is you have reason to think kernel buffering is preventing all
+			###       logs being flushed to disk. Obviously, this adds overhead.
+			#system('/usr/bin/sync');
 		}
 		$anvil->data->{loop}{count} = 0;
 	}

Anvil/Tools/Network.pm

@@ -564,6 +564,10 @@ Parameters;
 
 If this is set to C<< 1 >>, any connetions found to be down and not referencing any devices will be assigned the unroutable IP C<< 169.0.0.x >>, where C<< x >> is a sequential number. This should bring up unconfigured devices. 
 
+=head3 up (optional, default '0')
+
+If this is set to C<< 1 >>, any configured interfaces (determined by checking for C<< match.interface-name >>) that are down will be started, if possible.
+
 =cut
 sub collect_data
 {
@@ -574,8 +578,10 @@ sub collect_data
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Network->check_internet()" }});
 	
 	my $start = defined $parameter->{start} ? $parameter->{start} : "";
+	my $up    = defined $parameter->{up}    ? $parameter->{up}    : "";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		start => $start, 
+		up    => $up, 
 	}});
 	
 	if (exists $anvil->data->{nmcli})
@@ -665,6 +671,18 @@ sub collect_data
 					"nmcli::uuid::${uuid}::${variable}" => $anvil->data->{nmcli}{uuid}{$uuid}{$variable},
 				}});
 				
+				if ($variable eq "match.interface-name")
+				{
+					# Make sure we can look up the nmcli UUID by any of the names.
+					foreach my $interface (split/,/, $value)
+					{
+						$anvil->data->{nmcli}{interface}{$interface}{uuid} = $uuid;
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+							"nmcli::interface::${interface}::uuid" => $anvil->data->{nmcli}{interface}{$interface}{uuid},
+						}});
+					}
+				}
+				
 				if ($variable =~ /IP(\d).ADDRESS\[(\d+)\]/)
 				{
 					my $ip_type  = $1;
@@ -1109,6 +1127,31 @@ sub collect_data
 		}
 	}
 	
+	# Should we bring up interfaces?
+	if ($up)
+	{
+		foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{nmcli}{uuid}})
+		{
+			$anvil->data->{nmcli}{uuid}{$uuid}{'match.interface-name'} = "" if not defined $anvil->data->{nmcli}{uuid}{$uuid}{'match.interface-name'};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				"nmcli::uuid::${uuid}::active"               => $anvil->data->{nmcli}{uuid}{$uuid}{active},
+				"nmcli::uuid::${uuid}::match.interface-name" => $anvil->data->{nmcli}{uuid}{$uuid}{'match.interface-name'},
+			}});
+			if ((not $anvil->data->{nmcli}{uuid}{$uuid}{active}) && ($anvil->data->{nmcli}{uuid}{$uuid}{'match.interface-name'}))
+			{
+				my $shell_call = $anvil->data->{path}{exe}{nmcli}." connection up ".$uuid;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+				my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					output      => $output,
+					return_code => $return_code, 
+				}});
+				# NM seems to have a race issue, so we sleep a second after nmcli calls.
+				sleep 1;
+			}
+		}
+	}
+	
 	# Should we start interfaces?
 	if ($start)
 	{
@@ -1120,8 +1163,8 @@ sub collect_data
 		foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{nmcli}{uuid}})
 		{
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-				"nmcli::uuid::${uuid}::active" => $anvil->data->{nmcli}{uuid}{$uuid}{active},
+				"nmcli::uuid::${uuid}::active"                    => $anvil->data->{nmcli}{uuid}{$uuid}{active},
-				"nmcli::uuid::${uuid}::active" => $anvil->data->{nmcli}{uuid}{$uuid}{'connection.interface-name'},
+				"nmcli::uuid::${uuid}::connection.interface-name" => $anvil->data->{nmcli}{uuid}{$uuid}{'connection.interface-name'},
 			}});
 			if ((not $anvil->data->{nmcli}{uuid}{$uuid}{active}) && (not $anvil->data->{nmcli}{uuid}{$uuid}{'connection.interface-name'}))
 			{
@@ -1156,6 +1199,8 @@ sub collect_data
 					output      => $output,
 					return_code => $return_code, 
 				}});
+				# NM seems to have a race issue, so we sleep a second after nmcli calls.
+				sleep 1;
 				
 				$rescan = 1;
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { rescan => $rescan }});
@@ -4066,6 +4111,8 @@ sub modify_connection
 		output      => $output,
 		return_code => $return_code, 
 	}});
+	# NM seems to have a race issue, so we sleep a second after nmcli calls.
+	sleep 1;
 	
 	return($output, $return_code);
 }
@@ -4548,6 +4595,8 @@ sub reset_connection
 		output      => $output,
 		return_code => $return_code, 
 	}});
+	# NM seems to have a race issue, so we sleep a second after nmcli calls.
+	sleep 1;
 	
 	$shell_call = $anvil->data->{path}{exe}{nmcli}." connection up ".$uuid;
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
@@ -4556,6 +4605,8 @@ sub reset_connection
 		output      => $output,
 		return_code => $return_code, 
 	}});
+	# NM seems to have a race issue, so we sleep a second after nmcli calls.
+	sleep 1;
 	
 	return($output, $return_code);
 }
@@ -4744,7 +4795,7 @@ sub wait_for_network
 				's4:state'          => $state, 
 			}});
 			
-			if (($state eq "activated") or ($state == 1))
+			if (($state eq "activated") or ($state eq "1"))
 			{
 				$anvil->data->{network}{watch}{$interface_name}{ready} = 1;
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 

Anvil/Tools/ScanCore.pm

@@ -2528,12 +2528,28 @@ LIMIT 1;";
 					foreach my $order (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{order}})
 					{
 						my $method = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{order}{$order}{devices};
-						my $agent  = $anvil->data->{cib}{parsed}{data}{stonith}{primitive_id}{$method}{agent};
 						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 							's1:order'  => $order,
 							's2:method' => $method, 
-							's3:agent'  => $agent 
 						}});
+						my $agent = $anvil->data->{cib}{parsed}{data}{stonith}{primitive_id}{$method}{agent};
+						if ((not defined $agent) && ($method =~ /,/))
+						{
+							# Break up the method name to find the agent.
+							$agent = "";
+							foreach my $sub_method (split/,/, $method)
+							{
+								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { sub_method => $sub_method }});
+								if ((exists $anvil->data->{cib}{parsed}{data}{stonith}{primitive_id}{$sub_method}) && 
+								    (defined $anvil->data->{cib}{parsed}{data}{stonith}{primitive_id}{$sub_method}{agent}))
+								{
+									$agent = $anvil->data->{cib}{parsed}{data}{stonith}{primitive_id}{$sub_method}{agent};
+									$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { agent => $agent }});
+									last;
+								}
+							}
+						}
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { agent => $agent }});
 						
 						# We can't trust a PDU's output, so skip them. We also can't use the fake 'fence_delay' agent.
 						next if $agent =~ /pdu/;

Anvil/Tools/Striker.pm

@@ -312,9 +312,9 @@ This is a comma-separated list of DNS servers to use.
 
 This is the domain name to use for this Anvil! node.
 
-=head3 manifest_uuid (optional)
+=head3 manifest_uuid (required)
 
-This allows updating an existing mannifest, or specifying the manifest UUID to use for the new manifest.
+This allows updating an existing manifest, or specifying the manifest UUID to use for the new manifest. When creating a new manifest, set this to C<< new >>.
 
 =head3 mtu (optional)
 
@@ -338,16 +338,16 @@ sub generate_manifest
 	my $self      = shift;
 	my $parameter = shift;
 	my $anvil     = $self->parent;
-	my $debug     = $parameter->{debug} // 3;
+	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Striker->generate_manifest()" }});
 
-	my $network_dns     = $parameter->{dns}           // $anvil->data->{cgi}{dns}{value};
+	my $network_dns     = defined $parameter->{dns}           ? $parameter->{dns}           : "";
-	my $domain          = $parameter->{domain}        // $anvil->data->{cgi}{domain}{value};
+	my $domain          = defined $parameter->{domain}        ? $parameter->{domain}        : "";
-	my $manifest_uuid   = $parameter->{manifest_uuid} // $anvil->data->{cgi}{manifest_uuid}{value};
+	my $manifest_uuid   = defined $parameter->{manifest_uuid} ? $parameter->{manifest_uuid} : "";
-	my $network_mtu     = $parameter->{mtu}           // $anvil->data->{cgi}{mtu}{value};
+	my $network_mtu     = defined $parameter->{mtu}           ? $parameter->{mtu}           : "";
-	my $network_ntp     = $parameter->{ntp}           // $anvil->data->{cgi}{ntp}{value};
+	my $network_ntp     = defined $parameter->{ntp}           ? $parameter->{ntp}           : "";
-	my $name_prefix     = $parameter->{prefix}        // $anvil->data->{cgi}{prefix}{value};
+	my $name_prefix     = defined $parameter->{prefix}        ? $parameter->{prefix}        : "";
-	my $padded_sequence = $parameter->{sequence}      // $anvil->data->{cgi}{sequence}{value};
+	my $padded_sequence = defined $parameter->{sequence}      ? $parameter->{sequence}      : "";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		network_dns     => $network_dns,
 		domain          => $domain, 
@@ -379,6 +379,13 @@ sub generate_manifest
 	$anvil->Database->get_upses({debug => $debug});
 	$anvil->Database->get_fences({debug => $debug});
 	
+	if (not $manifest_uuid)
+	{
+		# Don't proceed, we'd get an invalid manifest.
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "warning_0041"});
+		return('!!error!!');
+	}
+	
 	if ($manifest_uuid eq "new")
 	{
 		$manifest_uuid = "";
@@ -390,7 +397,7 @@ sub generate_manifest
 		$padded_sequence = sprintf("%02d", $padded_sequence);
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { padded_sequence => $padded_sequence }});
 	}
-
+	
 	my $anvil_name = $name_prefix."-anvil-".$padded_sequence;
 	my $node1_name = $name_prefix."-a".$padded_sequence."n01";
 	my $node2_name = $name_prefix."-a".$padded_sequence."n02";
@@ -419,6 +426,14 @@ sub generate_manifest
 			my $subnet_value  = $parameter->{$subnet_key} // $anvil->data->{cgi}{$subnet_key}{value};
 			my $gateway_key   = $network_name."_gateway";
 			my $gateway_value = $parameter->{$gateway_key} // $anvil->data->{cgi}{$gateway_key}{value};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				's1:network_key'   => $network_key,
+				's2:network_value' => $network_value, 
+				's3:subnet_key'    => $subnet_key, 
+				's4:subnet_value'  => $subnet_value, 
+				's5:gateway_key'   => $gateway_key, 
+				's6:gateway_value' => $gateway_value, 
+			}});
 
 			$manifest_xml .= '		<network name="'.$network_name.'" network="'.$network_value.'" subnet="'.$subnet_value.'" gateway="'.$gateway_value.'" />'."\n";
 			
@@ -428,6 +443,10 @@ sub generate_manifest
 				# Record the network
 				my $ip_key   = $machine."_".$network_name."_ip";
 				my $ip_value = ($parameter->{$ip_key} // $anvil->data->{cgi}{$ip_key}{value}) // "";
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					's1:ip_key'   => $ip_key,
+					's2:ip_value' => $ip_value, 
+				}});
 
 				$machines->{$machine}{network}{$network_name} = $ip_value;
 				
@@ -437,6 +456,10 @@ sub generate_manifest
 					# Get the IP.
 					my $ipmi_ip_key   = $machine."_ipmi_ip";
 					my $ipmi_ip_value = ($parameter->{$ipmi_ip_key} // $anvil->data->{cgi}{$ipmi_ip_key}{value}) // "";
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						's1:ipmi_ip_key'   => $ipmi_ip_key,
+						's2:ipmi_ip_value' => $ipmi_ip_value, 
+					}});
 
 					$machines->{$machine}{ipmi_ip} = $ipmi_ip_value;
 					
@@ -445,6 +468,10 @@ sub generate_manifest
 					{
 						my $ups_key                              = $machine."_ups_".$ups_name;
 						my $ups_value = ($parameter->{$ups_key} // $anvil->data->{cgi}{$ups_key}{value}) // "";
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+							's1:ups_key'   => $ups_key,
+							's2:ups_value' => $ups_value, 
+						}});
 
 						$machines->{$machine}{ups}{$ups_name} = $ups_value ? "1" : "0";
 					}
@@ -454,6 +481,10 @@ sub generate_manifest
 					{
 						my $fence_key   = $machine."_fence_".$fence_name;
 						my $fence_value = ($parameter->{$fence_key} // $anvil->data->{cgi}{$fence_key}{value}) // "";
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+							's1:fence_key'   => $fence_key,
+							's2:fence_value' => $fence_value, 
+						}});
 
 						$machines->{$machine}{fence}{$fence_name} = $fence_value;
 					}
@@ -531,6 +562,7 @@ sub generate_manifest
 		manifest_name => $anvil_name, 
 		manifest_xml  => $manifest_xml, 
 	});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { manifest_uuid => $manifest_uuid }});
 	
 	return($manifest_uuid, $anvil_name);
 }

Anvil/Tools/System.pm

@@ -2302,7 +2302,9 @@ LIMIT 1
 	# use, but we'll scan 1..9 + 0.
 	foreach my $i (1..9, 0)
 	{
-		my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{ipmitool}." lan print ".$i});
+		my $shell_call = $anvil->data->{path}{exe}{ipmitool}." lan print ".$i;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
+		my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			output      => $output, 
 			return_code => $return_code,
@@ -2483,7 +2485,10 @@ LIMIT 1
 	while ($waiting)
 	{
 		my $debug = 2;
-		my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel});
+		my $shell_call = $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
+		
+		my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			output      => $output, 
 			return_code => $return_code,
@@ -4990,17 +4995,18 @@ sub test_ipmi
 			# Build the shell call.
 			$shell_call = $anvil->data->{path}{directories}{fence_agents}."/fence_ipmilan ".$lanplus_switch." --ip ".$ipmi_target." --username ".$ipmi_user." --password \"".$test_password."\" --action status";
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => 1, list => { shell_call => $shell_call }});
-	
+			
+			# HPs can take over 10 seconds to respond.
+			my $timeout     = 30;
 			my $output      = "";
 			my $return_code = "";
 			if ($target)
 			{
 				### Remote call
-				# HPs can take over 10 seconds to respond, so we set the timeout higher to account for this.
 				($output, my $error, $return_code) = $anvil->Remote->call({
 					debug       => $debug, 
 					secure      => 1,
-					timeout     => 20,
+					timeout     => $timeout,
 					shell_call  => $shell_call, 
 					target      => $target,
 					password    => $password,
@@ -5017,7 +5023,7 @@ sub test_ipmi
 				($output, $return_code) = $anvil->System->call({
 					debug       => $debug, 
 					secure      => 1,
-					timeout     => 2,
+					timeout     => $timeout,
 					shell_call  => $shell_call, 
 				});
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 

anvil.spec.in

@@ -263,10 +263,10 @@ setenforce 0
 ### TODO: check it if was disabled (if it existed before) and, if so, leave it disabled.
 systemctl enable --now chronyd.service
 systemctl enable --now anvil-daemon.service
-systemctl enable --now anvil-monitor-daemons.service
 systemctl enable --now anvil-monitor-network.service
-systemctl enable --now anvil-monitor-performance.service
 systemctl enable --now scancore.service
+systemctl disable --now anvil-monitor-daemons.service
+systemctl disable --now anvil-monitor-performance.service
 
 %pre striker
 getent passwd %{suiapi} >/dev/null \

man/anvil-manage-files.8

@@ -1,8 +1,11 @@
-.\" Manpage for the Anvil! server removal tool
+.\" Manpage for the Anvil! file management tool
 .\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
 .TH anvil-manage-files "8" "August 02 2022" "Anvil! Intelligent Availability™ Platform"
 .SH NAME
 anvil-manage-files \- This program manages the files sync'ed across machines in the Anvil! cluster
+.TP
+.B Note: 
+This tool is not yet complete.
 .SH SYNOPSIS
 .B anvil-manage-files 
 \fI\,<command> \/\fR[\fI\,options\/\fR]
@@ -21,18 +24,31 @@ When logging, record sensitive data, like passwords.
 Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
 .SS "Commands:"
 .TP
+\fB\-\-add\fR 
+(not implemented yet)
+.TP
+\fB\-\-check\fR 
+This runs a check for files that need to be processed (added, deleted, etc).
+.TP
 \fB\-\-delete\fR 
 This will delete the \fB\-\-file\fR </path/to/file> from the entire Anvil! cluster. 
 .TP
-This action is permanent!
+\fB\-\-download\fR 
+(not implemented yet)
+.TP
+\fB\-\-file\fR <file>
+This is the file being worked on.
+.TP
+\fB\-\-is\-script\fR 
+When set, the file is marked as a script or program. Files marked this way can be used in different places in the Anvil! cluster.
+.TP
+\fB\-\-rename\fR 
+The changes the name of a file. Use \fB\-\-file\fR <old name> and \fB\-\-to\fR <new name>.
 .TP
-\fB\-\-job\-uuid\fR <name>
+\fB\-\-to\fR <new name>
-The program is normally run as a job, with data on how to configure the host defined in the job. This switch allows the running of a specific job. If this is not set, the program will search for a job that has not yet been picked up by another process. If found, that job UUID is used automatically.
+The is the new file name that's being changed. 
 .IP
 .SH AUTHOR
 Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
 .SH "REPORTING BUGS"
 Report bugs to users@clusterlabs.org
-
-
-", "download", "everywhere", "file", "is-script", "job-uuid", "rename", "to

man/anvil-manage-firewall.8

@@ -0,0 +1,36 @@
+.\" Manpage for the Anvil! firewall management tool
+.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
+.TH anvil-manage-firewall "8" "August 02 2022" "Anvil! Intelligent Availability™ Platform"
+.SH NAME
+anvil-manage-firewall \- This program manages the firewall rules on the host.
+.TP
+.B Note: 
+This tool is not yet complete.
+.SH SYNOPSIS
+.B anvil-manage-firewall 
+\fI\,<command> \/\fR[\fI\,options\/\fR]
+.SH DESCRIPTION
+This handles dynamically opening and closing ports on the firewall of the various machines in an Anvil! cluster.
+.TP
+.SH OPTIONS
+.TP
+\-?, \-h, \fB\-\-help\fR
+Show this man page.
+.TP
+\fB\-\-log-secure\fR
+When logging, record sensitive data, like passwords.
+.TP
+\-v, \-vv, \-vvv
+Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
+.SS "Commands:"
+.TP
+\fB\-\-job\-uuid\fR 
+This is the UUID for a job being run.
+.TP
+\fB\-\-server\fR <name or uuid>
+This opens or closes the VNC and DRBD ports related to a server
+.IP
+.SH AUTHOR
+Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
+.SH "REPORTING BUGS"
+Report bugs to users@clusterlabs.org

man/anvil-manage-host.8

@@ -1,13 +1,13 @@
 .\" Manpage for the Anvil! server boot program
 .\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
-.TH anvil-manage-host "8" "Octobober 12 2023" "Anvil! Intelligent Availability™ Platform"
+.TH anvil-manage-host "8" "October 12 2023" "Anvil! Intelligent Availability™ Platform"
 .SH NAME
 anvil-manage-host \- Tool used to check or set various configuration options for a host.
 .SH SYNOPSIS
 .B anvil-manage-host 
 \fI\,<command> \/\fR[\fI\,options\/\fR]
 .SH DESCRIPTION
-
+This tool provides a way to perform actions on a host that normally are handled by other processes, like marking a system as configure, etc. This will likely grow constantly over time.
 .TP
 .SH OPTIONS
 .TP
@@ -22,7 +22,7 @@ Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a s
 .SS "Commands:"
 .TP
 \fB\-\-age\-out\-database\fR
-This requests the database check for records that are too old and purge them.
+This searches the database(s) for data that is old enough that it's no longer needed and deletes it from the database. This helps with the performance of resync operations in particular.
 .TP
 \fB\-\-auto\-grow\-pv\fR
 This looks at LVM physical volumes on the local host. For each one that is found, 'parted' is called to check if there's more that 1 GiB of free space available after it. If so, it will extend the PV partition to use the free space.
@@ -30,25 +30,28 @@ This looks at LVM physical volumes on the local host. For each one that is found
 If you deleted the default '/home' partition during the install of a subnode or DR host, this should give you that space back.
 .TP
 \fB\-\-check\-configured\fR
-Check to see if the host is marked as configured or yet.
+This reports if the host is marked as configured yet or not.
 .TP
 \fB\-\-check\-database\fR
-This checks to see if the database is enabled or not.
+This reports if the database on the local Striker is active or inactive.
 .TP
 \fB\-\-confirm\fR
 This confirms actions that would normally prompt the user to confirm before proceeding. 
 .TP
 \fB\-\-database\-active\fR
-This enables the database on the local Striker dashboard.
+This marks the local database to be active.
 .TP
 \fB\-\-database\-inactive\fR
-This disables the database on the local Striker dashboard.
+This marks the local database to be inactive.
 .TP
 \fB\-\-mark\-configured\fR
-This marks the host as having been configured.
+This marks the local system as being configured. This should be used carefully.
+.TP
+\fB\-\-mark\-unconfigured\fR
+This marks the local system as being unconfigured. This should be used carefully.
 .TP
 \fB\-\-resync\-database\fR
-This forces a database resync if two or more strikers are online.
+This forces the databases to be resync'ed, even if they seem synced.
 .IP
 .SH AUTHOR
 Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.

ocf/alteeve/server

@@ -1991,10 +1991,17 @@ sub validate_ram
 {
 	my ($anvil) = @_;
 	
+	### TODO: During a migration, we need to check the memory in migrate_to.
+	if ($anvil->data->{switches}{migrate_to})
+	{
+		return(0);
+	}
+	
 	# How mcuh RAM does the server need and how much do we have free?
 	my $local_host       = $anvil->Get->short_host_name();
 	my $server           = $anvil->data->{environment}{OCF_RESKEY_name};
 	my $server_ram_bytes = $anvil->data->{server}{$local_host}{$server}{from_disk}{memory};
+	# In migrations, this checks local RAM when validating if it can run on the peer, which makes no sense. Disabling.
 	my $available        = $anvil->Get->free_memory({debug => 3});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 		server_ram_bytes => $anvil->Convert->add_commas({number => $server_ram_bytes})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_bytes}).")", 
@@ -2007,7 +2014,7 @@ sub validate_ram
 			name                => $server, 
 			ram                 => $anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_bytes}), 
 			ram_bytes           => $anvil->Convert->add_commas({number => $server_ram_bytes}), 
-			available_ram       => $anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_bytes}), 
+			available_ram       => $anvil->Convert->bytes_to_human_readable({'bytes' => $available}), 
 			available_ram_bytes => $anvil->Convert->add_commas({number => $available}), 
 		}});
 		$anvil->nice_exit({exit_code => 1});

scancore-agents/scan-network/scan-network

@@ -536,7 +536,18 @@ sub collect_data
 	my ($anvil) = @_;
 	
 	# Read in data from Network Manager
-	$anvil->Network->collect_data({debug => 2});
+	my $uptime = $anvil->Get->uptime;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uptime => $uptime }});
+	if ($uptime < 600)
+	{
+		# Check for / start down but configured interfaces.
+		$anvil->Network->collect_data({up => 1, debug => 2});
+	}
+	else
+	{
+		# Just collect data.
+		$anvil->Network->collect_data({debug => 2});
+	}
 	
 	### TODO: We can get the IPs from collect_data, we should phase this out.
 	# The 'local_host' is needed to pull data recorded by Network->get_ips();

share/words.xml

@@ -1991,7 +1991,7 @@ The database connection error was:
 		<key name="log_0189">Database archiving is disabled, skipping archive checks.</key>
 		<key name="log_0190">Peer: [#!variable!peer!#], database: [#!variable!name!#], password: [#!variable!password!#], host UUID: [#!variable!uuid!#]</key>
 		<key name="log_0191">Connection only to: [#!variable!db_uuid!#], skipping: [#!variable!uuid!#].</key>
-		<key name="log_0192">The connection to the database: [#!variable!server!#] has failed. Will attempt to reconnect.</key>
+		<key name="log_0192">The connection to the database: [#!variable!server!#] has failed. Will attempt to reconnect to databases-.</key>
 		<key name="log_0193">Switching the default database handle to use the database: [#!variable!server!#] prior to reconnect attempt.</key>
 		<key name="log_0194">Switching the default database to read from to the database: [#!variable!server!#] prior to reconnect attempt.</key>
 		<key name="log_0195">Ready to try to reconnect to: [#!variable!server!#], but delaying for: [#!variable!delay!#] seconds to give the database a chance to come back online in case this is a transient issue.</key>
@@ -2739,6 +2739,9 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
 The key in the database has been updated, so we're going to swap the key. The change is;
 old key: [#!variable!old_key!#]
 new key: [#!variable!new_key!#]</key>
+		<key name="log_0852">Finished configuring bonds.</key>
+		<key name="log_0853">Now configuring bridges.</key>
+		<key name="log_0854">The connection to the database: [#!variable!server!#] has been restored!</key>
 
 		<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
 		<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>
@@ -4197,6 +4200,14 @@ We will try to proceed anyway.</key>
 		<key name="warning_0172">[ Warning ] - The line: [#!variable!line!#] that was going to be added to the hosts file is invalid, removing it.</key>
 		<key name="warning_0173">[ Warning ] - Failed to convert: [#!variable!source_file!#] to: [#!variable!new_file!#] (format: [#!variable!format!#]! Return code was: [#!variable!return_code!#], expected '0'.</key>
 		<key name="warning_0174">[ Warning ] - The fence method: [#!variable!method!#] already existed, proceeding.</key>
+		<key name="warning_0175">[ Warning ] - The DB query: [#!variable!query!#] timed out! It was given: [#!variable!timeout!#] seconds, and alarmed with: [#!variable!error!#].</key>
+		<key name="warning_0176">[ Warning ] - The DB query: [#!variable!query!#] failed with the error: [#!variable!error!#].</key>
+		<key name="warning_0177">[ Warning ] - SQL quoting string: [#!variable!string!#] failed with the error: [#!variable!error!#].</key>
+		<key name="warning_0178">[ Warning ] - About to run 'anvil-configure-host'. This is likely going to take the network down, so we will hold here until this job is complete.</key>
+		<key name="warning_0179">[ Warning ] - The connection to the database: [#!variable!server!#] was not restored, unable to this database.</key>
+		<key name="warning_0180">[ Warning ] - We were asked to read from the database: [#!variable!server!#], but it is not or is no longer available.</key>
+		<key name="warning_0181">[ Warning ] - We were asked to read from the database: [#!variable!server!#], but it is no longer available, and no alternative databases are available.</key>
+		<key name="warning_0182">[ Warning ] - We appear to have lost access to the database: [#!variable!uuid!#], during a write. Skipping this database.</key>
 		
 	</language>
 	<!-- 日本語 -->

tools/anvil-change-password

@@ -36,7 +36,7 @@ my $anvil = Anvil::Tools->new();
 if (($< != 0) && ($> != 0))
 {
 	# Not root
-	print $anvil->Words->string({key => "error_0005"})."\n";
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "error_0005"});
 	$anvil->nice_exit({exit_code => 1});
 }
 
@@ -50,7 +50,7 @@ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure =
 if (not $anvil->data->{sys}{database}{connections})
 {
 	# No databases, exit.
-	print $anvil->Words->string({key => "error_0003"})."\n";
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "error_0003"});
 	$anvil->nice_exit({exit_code => 2});
 }
 
@@ -71,7 +71,7 @@ if ($anvil->data->{switches}{'password-file'})
 	else
 	{
 		# The file doesn't exist.
-		print $anvil->Words->string({key => "error_0008", variables => { file => $anvil->data->{switches}{'password-file'} }})."\n";
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "error_0008", variables => { file => $anvil->data->{switches}{'password-file'} }});
 		$anvil->nice_exit({exit_code => 4});
 	}
 }
@@ -110,7 +110,7 @@ elsif (not $anvil->data->{switches}{'new-password'})
 	}
 	else
 	{
-		print $anvil->Words->string({key => "error_0007"})."\n";
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "error_0007"});
 		$anvil->nice_exit({exit_code => 3});
 	}
 }
@@ -130,7 +130,7 @@ else
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { "switches::new-password" => $anvil->data->{switches}{'new-password'} }});
 	if (($anvil->data->{switches}{y}) or ($anvil->data->{switches}{yes}))
 	{
-		print $anvil->Words->string({key => "message_0023"})."\n";
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, secure => 0, key => "message_0023"});
 		update_local_passwords($anvil);
 	}
 	else
@@ -151,9 +151,10 @@ else
 	}
 }
 
-
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0750"});
 $anvil->nice_exit({exit_code => 0});
 
+
 #############################################################################################################
 # Functions                                                                                                 #
 #############################################################################################################
@@ -165,7 +166,7 @@ sub update_local_passwords
 	
 	# Update the 'admin' user password in the database.
 	my $user = "admin";
-	print $anvil->Words->string({key => "message_0024", variables => { user => $user }});
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "message_0024", variables => { user => $user }});
 	
 	my $user_uuid = $anvil->Database->insert_or_update_users({
 		debug               => 2,
@@ -177,7 +178,7 @@ sub update_local_passwords
 	});
 	# Log out any Striker sessions.
 	$anvil->Account->logout({host_uuid => "all"});
-	print $anvil->Words->string({key => "message_0025"})."\n";
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "message_0025"});
 	
 	# Validate
 	my $valid = $anvil->Account->validate_password({
@@ -215,25 +216,25 @@ sub update_local_passwords
 	my $owner_name    = $results->[0]->[1];
 	foreach my $user ("postgres", $owner_name)
 	{
-		print $anvil->Words->string({key => "message_0026", variables => { user => $user }});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "message_0026", variables => { user => $user }});
 		my $query = "ALTER ROLE ".$user." WITH PASSWORD ".$dbh->quote($anvil->data->{switches}{'new-password'});
 		$dbh->do($query) or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0090", variables => { 
 				query    => $anvil->Log->is_secure($query), 
 				server   => "localhost",
 				db_error => $DBI::errstr, 
 			}});
-		print $anvil->Words->string({key => "message_0025"})."\n";
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "message_0025"});
 	}
 	
 	# Update our database password in anvil.conf
-	print $anvil->Words->string({key => "message_0027", variables => { file => $anvil->data->{switches}{'new-password'} }});
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "message_0027", variables => { file => $anvil->data->{switches}{'new-password'} }});
 	$anvil->Storage->update_config({
 		debug    => 2,
 		secure   => 1, 
 		variable => "database::${host_uuid}::password", 
 		value => $anvil->data->{switches}{'new-password'},
 	});
-	print $anvil->Words->string({key => "message_0025"})."\n";
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "message_0025"});
 		
 	### TODO: Loop through any other dashboards and nodes we know about and call the above with 'target'
 	###       (and password, port and remote_user) set.
@@ -241,13 +242,13 @@ sub update_local_passwords
 	# Update the local users.
 	foreach my $user ("admin", "root")
 	{
-		print $anvil->Words->string({key => "message_0028", variables => { user => $user }});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "message_0028", variables => { user => $user }});
 		$anvil->System->change_shell_user_password({debug => 2, user => $user, new_password => $anvil->data->{switches}{'new-password'}});
-		print $anvil->Words->string({key => "message_0025"})."\n";
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "message_0025"});
 	}
 	
 	# All done!
-	print $anvil->Words->string({key => "message_0029"})."\n";
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "message_0029"});
 	
 	return(0);
 }

tools/anvil-configure-host

@@ -245,8 +245,14 @@ sub update_passwords
 				my $shell_call = $anvil->data->{path}{exe}{'anvil-change-password'}." -y --password-file ".$temp_file.$anvil->Log->switches;
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 				
-				my ($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $shell_call });
+				my ($output, $return_code) = $anvil->System->call({
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
+					debug      => 2, 
+					timeout    => 30, 
+					shell_call => $shell_call });
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					output      => $output, 
+					return_code => $return_code,
+				}});
 				
 				if ($return_code)
 				{
@@ -271,7 +277,9 @@ sub update_passwords
 			}
 			
 			# Unlink the temp file.
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0263", variables => { file => $temp_file }});
 			unlink $temp_file;
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0283"});
 		}
 	}
 	
@@ -507,7 +515,7 @@ ORDER BY
 	$anvil->data->{sys}{reboot} = 0;
 	
 	# Read the existing network data 
-	$anvil->Network->collect_data({start => 1});
+	$anvil->Network->collect_data({debug => 2, start => 1});
 	
 	# This will be set to '1' if we make a change.
 	my $changes        = 0;
@@ -534,23 +542,40 @@ ORDER BY
 			file      => $THIS_FILE, 
 			line      => __LINE__,
 		});
-		$anvil->Database->disconnect();
+		$anvil->Database->disconnect({debug => 2});
 	}
 	
 	# These can brake the connection.
+	$anvil->data->{sys}{restart_nm} = 0;
 	reconfigure_bonds($anvil);
 	reconfigure_bridges($anvil);
 	reconfigure_ip_addresses($anvil);
 	
+	# Sleep a few seconds, and then restart NetworkManager.service.
+	my $nm_running = $anvil->System->check_daemon({debug => 2, daemon => "NetworkManager.service"});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		nm_running        => $nm_running,
+		'sys::restart_nm' => $anvil->data->{sys}{restart_nm},
+	}});
+	if (($anvil->data->{sys}{restart_nm}) or (not $nm_running))
+	{
+		$anvil->System->stop_daemon({debug => 2, daemon => "NetworkManager.service"});
+		$anvil->System->start_daemon({debug => 2, daemon => "NetworkManager.service"});
+		sleep 5;
+	}
+	
 	# Reconnect!
 	if ($host_type ne "striker")
 	{
-		my $time_now   = time;
+		my $time_now        = time;
-		my $wait_until = $time_now + 120;
+		my $restart_nm_time = $time_now + 30;
-		my $waiting    = 1;
+		my $wait_until      = $time_now + 60;
+		my $nm_restarted    = 0;
+		my $waiting         = 1;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-			time_now   => $time_now, 
+			's1:time_now'        => $time_now, 
-			wait_until => $wait_until,
+			's2:restart_nm_time' => $restart_nm_time, 
+			's3:wait_until'      => $wait_until,
 		}});
 		while ($waiting)
 		{
@@ -575,6 +600,17 @@ ORDER BY
 				}});
 				if ($time_left > 0)
 				{
+					if (($now_time > $restart_nm_time) && (not $nm_restarted))
+					{
+						# Kick the network
+						$nm_restarted = 1;
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nm_restarted => $nm_restarted }});
+						
+						# Restart the daemon
+						$anvil->System->stop_daemon({debug => 2, daemon => "NetworkManager.service"});
+						$anvil->System->start_daemon({debug => 2, daemon => "NetworkManager.service"});
+					}
+					
 					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0829", variables => { time_left => $time_left }});
 					sleep 5;
 				}
@@ -582,7 +618,14 @@ ORDER BY
 				{
 					# Give up and reboot.
 					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "warning_0169"});
-					do_reboot($anvil);
+					
+					# Don't use 'do_reboot()' as we've got no DB connection.
+					my $shell_call = $anvil->data->{path}{exe}{systemctl}." reboot";
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { shell_call => $shell_call }});
+					
+					# We should be dead by now...
+					my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
 				}
 			}
 		}
@@ -691,6 +734,7 @@ sub reconfigure_bridges
 {
 	my ($anvil) = @_;
 	
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0853"});
 	my $local_host = $anvil->Get->short_host_name();
 	my $bcn_count  = $anvil->data->{config}{bcn_count};
 	my $ifn_count  = $anvil->data->{config}{ifn_count};
@@ -793,6 +837,10 @@ sub reconfigure_bridges
 					return_code => $return_code, 
 				}});
 				
+				# NM seems to have a race issue, so we sleep a second after nmcli calls.
+				$anvil->data->{sys}{restart_nm} = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::restart_nm' => $anvil->data->{sys}{restart_nm} }});
+				
 				if ($return_code)
 				{
 					# Failed to add the bridge
@@ -841,6 +889,10 @@ sub reconfigure_bridges
 						output      => $output,
 						return_code => $return_code, 
 					}});
+					
+					# NM seems to have a race issue, so we sleep a second after nmcli calls.
+					$anvil->data->{sys}{restart_nm} = 1;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::restart_nm' => $anvil->data->{sys}{restart_nm} }});
 				}
 				
 				# Rescan.
@@ -962,6 +1014,10 @@ sub reconfigure_bridges
 				return_code => $return_code, 
 			}});
 			
+			# NM seems to have a race issue, so we sleep a second after nmcli calls.
+			$anvil->data->{sys}{restart_nm} = 1;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::restart_nm' => $anvil->data->{sys}{restart_nm} }});
+			
 			if ($return_code)
 			{
 				# Failed to connect the device to the bridge.
@@ -1189,6 +1245,11 @@ sub reconfigure_bonds
 				my $bond_uuid = ($output =~ /\((.*?)\) successfully added/)[0];
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bond_uuid => $bond_uuid }});
 				
+				# NM seems to have a race issue, so we sleep a second after nmcli calls.
+				$anvil->data->{sys}{restart_nm} = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::restart_nm' => $anvil->data->{sys}{restart_nm} }});
+				sleep 5;
+				
 				if ($bond_uuid)
 				{
 					# Disabling DHCP on the new bond device
@@ -1215,6 +1276,11 @@ sub reconfigure_bonds
 						output      => $output,
 						return_code => $return_code, 
 					}});
+					
+					# NM seems to have a race issue, so we sleep a second after nmcli calls.
+					$anvil->data->{sys}{restart_nm} = 1;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::restart_nm' => $anvil->data->{sys}{restart_nm} }});
+					sleep 5;
 				}
 				
 				# Done! Rescanning the network config
@@ -1349,6 +1415,11 @@ sub reconfigure_bonds
 					return_code => $return_code, 
 				}});
 				
+				# NM seems to have a race issue, so we sleep a second after nmcli calls.
+				$anvil->data->{sys}{restart_nm} = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::restart_nm' => $anvil->data->{sys}{restart_nm} }});
+				sleep 5;
+				
 				if ($return_code)
 				{
 					# Adding the link failed.
@@ -1396,14 +1467,61 @@ sub reconfigure_bonds
 					line      => __LINE__,
 					variables => { },
 				});
-				$anvil->Network->collect_data();
+				$anvil->Network->collect_data({debug => 2});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nm_uuid => $nm_uuid }});
 			}
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { i => $i }});
 		}
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_type => $network_type }});
 	}
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0852"});
 	
 	return(0);
 }
 
+sub wait_for_nm
+{
+	my ($anvil, $bond_name) = @_;
+	
+	my $found      = 0;
+	my $waiting    = 1;
+	my $wait_until = time + 30;
+	while ($waiting)
+	{
+		my $shell_call = $anvil->data->{path}{exe}{nmcli}." connection show | ".$anvil->data->{path}{exe}{'grep'}." -c ".$bond_name;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			output      => $output,
+			return_code => $return_code, 
+		}});
+		
+		if ($output eq "0")
+		{
+			if (time > $wait_until)
+			{
+				# Give up.
+				$waiting = 0;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
+			}
+			# Not found yet.
+			sleep 1;
+		}
+		else
+		{
+			# Found it.
+			$found   = 0;
+			$waiting = 0;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				found   => $found, 
+				waiting => $waiting,
+			}});
+		}
+	}
+	
+	return($found);
+}
+
 sub reconfigure_ip_addresses
 {
 	my ($anvil) = @_;
@@ -1729,6 +1847,11 @@ sub reconfigure_ip_addresses
 					},
 				});
 				($output, $return_code) = $anvil->Network->reset_connection({uuid => $old_uuid});
+				
+				# NM seems to have a race issue, so we sleep a second after nmcli calls.
+				$anvil->data->{sys}{restart_nm} = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::restart_nm' => $anvil->data->{sys}{restart_nm} }});
+				sleep 5;
 			}
 			
 			# Now assign the IP.
@@ -1752,6 +1875,11 @@ sub reconfigure_ip_addresses
 				return_code => $return_code, 
 			}});
 			
+			# NM seems to have a race issue, so we sleep a second after nmcli calls.
+			$anvil->data->{sys}{restart_nm} = 1;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::restart_nm' => $anvil->data->{sys}{restart_nm} }});
+			sleep 5;
+			
 			# Restart the interface
 			$anvil->Job->update_progress({
 				progress  => $anvil->Job->bump_progress({steps => 1}), 
@@ -1779,7 +1907,7 @@ sub reconfigure_ip_addresses
 				line      => __LINE__,
 				variables => { },
 			});
-			$anvil->Network->collect_data();
+			$anvil->Network->collect_data({debug => 2});
 		}
 	}
 	
@@ -1831,6 +1959,8 @@ sub reconfigure_interfaces
 					output      => $output, 
 					return_code => $return_code, 
 				}});
+				# NM seems to have a race issue, so we sleep a second after nmcli calls.
+				sleep 1;
 				
 				# Rescan.
 				$anvil->Job->update_progress({
@@ -1843,7 +1973,7 @@ sub reconfigure_interfaces
 					line      => __LINE__,
 					variables => { },
 				});
-				$anvil->Network->collect_data();
+				$anvil->Network->collect_data({debug => 2});
 			}
 		}
 	}
@@ -1925,11 +2055,20 @@ sub reconfigure_interfaces
 				foreach my $interface (split/,/, $match_interface_name)
 				{
 					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { interface => $interface }});
+					
+					# In case the GENERAL.IP-IFACE or GENERAL.NAME name didn't update, 
+					# map this interface name to 'nmcli::interface::<name>::uuid'.
+					if (not exists $anvil->data->{nmcli}{interface}{$interface})
+					{
+						$anvil->data->{nmcli}{interface}{$interface}{uuid} = $link1_nm_uuid;
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+							"nmcli::interface::${interface}::uuid" => $anvil->data->{nmcli}{interface}{$interface}{uuid},
+						}});
+					}
 					if ($interface eq $wanted_link1_name)
 					{
 						$found = 1;
 						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
-						last;
 					}
 				}
 				if (not $found)
@@ -2211,6 +2350,8 @@ sub rename_interface
 			output      => $output, 
 			return_code => $return_code, 
 		}});
+		# NM seems to have a race issue, so we sleep a second after nmcli calls.
+		sleep 1;
 		
 		$shell_call = $anvil->data->{path}{exe}{nmcli}." --get-values connection.interface-name connection show ".$nm_uuid;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
@@ -2282,6 +2423,8 @@ sub rename_interface
 		output      => $output, 
 		return_code => $return_code, 
 	}});
+	# NM seems to have a race issue, so we sleep a second after nmcli calls.
+	sleep 1;
 	
 	# Read it back
 	$shell_call = $anvil->data->{path}{exe}{nmcli}." --get-values match.interface-name connection show ".$nm_uuid;
@@ -2331,6 +2474,8 @@ sub rename_interface
 		output      => $output, 
 		return_code => $return_code, 
 	}});
+	# NM seems to have a race issue, so we sleep a second after nmcli calls.
+	sleep 1;
 	
 	# Read it back
 	$shell_call = $anvil->data->{path}{exe}{nmcli}." --get-values connection.id connection show ".$nm_uuid;
@@ -2352,7 +2497,7 @@ sub rename_interface
 		line      => __LINE__,
 		variables => { },
 	});
-	$anvil->Network->collect_data();
+	$anvil->Network->collect_data({debug => 2});
 	
 	# Set the reboot flag.
 	$anvil->data->{sys}{reboot_needed} = 1;

tools/anvil-daemon

@@ -173,21 +173,25 @@ my $delay = set_delay($anvil);
 # Once a day, we'll refresh an Install Target's RPM repository (has no effect on non-Striker dashboards).
 $anvil->data->{timing}{minute_checks}         = 60;
 $anvil->data->{timing}{ten_minute_checks}     = 600;
+$anvil->data->{timing}{hourly_checks}         = 3600;
 $anvil->data->{timing}{daily_checks}          = 86400;
 $anvil->data->{timing}{repo_update_interval}  = 86400;
 $anvil->data->{timing}{next_minute_check}     = $now_time - 1;
+$anvil->data->{timing}{next_hourly_check}     = $now_time - 1;
 $anvil->data->{timing}{next_ten_minute_check} = $now_time - 1;
 $anvil->data->{timing}{next_daily_check}      = ($now_time + $delay) - 1;
 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-	"s1:timing::minute_checks"         => $anvil->data->{timing}{minute_checks}, 
+	"s01:timing::minute_checks"         => $anvil->data->{timing}{minute_checks}, 
-	"s2:timing::ten_minute_checks"     => $anvil->data->{timing}{ten_minute_checks}, 
+	"s02:timing::ten_minute_checks"     => $anvil->data->{timing}{ten_minute_checks}, 
-	"s3:timing::daily_checks"          => $anvil->data->{timing}{daily_checks}, 
+	"s03:timing::hourly_checks"         => $anvil->data->{timing}{hourly_checks}, 
-	"s4:timing::repo_update_interval"  => $anvil->data->{timing}{repo_update_interval}, 
+	"s04:timing::daily_checks"          => $anvil->data->{timing}{daily_checks}, 
-	"s5:now_time"                      => $now_time, 
+	"s05:timing::repo_update_interval"  => $anvil->data->{timing}{repo_update_interval}, 
-	"s6:delay"                         => $delay, 
+	"s06:now_time"                      => $now_time, 
-	"s7:timing::next_minute_check"     => $anvil->data->{timing}{next_minute_check}, 
+	"s07:delay"                         => $delay, 
-	"s8:timing::next_ten_minute_check" => $anvil->data->{timing}{next_ten_minute_check}, 
+	"s08:timing::next_minute_check"     => $anvil->data->{timing}{next_minute_check}, 
-	"s9:timing::next_daily_check"      => $anvil->data->{timing}{next_daily_check}, 
+	"s09:timing::next_ten_minute_check" => $anvil->data->{timing}{next_ten_minute_check}, 
+	"s10:timing::next_hourly_check"     => $anvil->data->{timing}{next_hourly_check}, 
+	"s11:timing::next_daily_check"      => $anvil->data->{timing}{next_daily_check}, 
 }});
 
 # Disconnect. We'll reconnect inside the loop
@@ -231,9 +235,6 @@ while(1)
 		$anvil->nice_exit({exit_code => 0});
 	}
 	
-	# Check how much RAM we're using.
-	check_ram($anvil);
-	
 	# Disconnect from the database(s) and sleep now.
 	$anvil->Database->disconnect();
 	sleep(2);
@@ -443,8 +444,9 @@ sub handle_periodic_tasks
 		"s1:now_time"                      => $now_time, 
 		"s2:timing::next_minute_check"     => $anvil->data->{timing}{next_minute_check}, 
 		"s3:timing::next_ten_minute_check" => $anvil->data->{timing}{next_ten_minute_check}, 
-		"s4:timing::next_daily_check"      => $anvil->data->{timing}{next_daily_check},
+		"s4:timing::next_hourly_check"     => $anvil->data->{timing}{next_hourly_check}, 
-		"s5:host_type"                     => $host_type,
+		"s5:timing::next_daily_check"      => $anvil->data->{timing}{next_daily_check},
+		"s6:host_type"                     => $host_type,
 	}});
 	
 	# Time to run once per minute tasks.
@@ -618,6 +620,20 @@ sub handle_periodic_tasks
 		}});
 	}
 	
+	# Now check to see if it's time to run hourly tasks.
+	if ($now_time >= $anvil->data->{timing}{next_hourly_check})
+	{
+		# Check how much RAM we're using.
+		check_ram($anvil);
+		
+		# Update the next check time.
+		$anvil->data->{timing}{next_hourly_check} = $now_time + $anvil->data->{timing}{hourly_checks};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			"s1:timing::hourly_checks"     => $anvil->data->{timing}{hourly_checks},
+			"s2:timing::next_hourly_check" => $anvil->data->{timing}{next_hourly_check}, 
+		}});
+	}
+	
 	# Now check to see if it's time to run daily tasks.
 	if ($now_time >= $anvil->data->{timing}{next_daily_check})
 	{
@@ -1381,6 +1397,8 @@ AND
 		$anvil->Database->get_hosts({debug => 2});
 		my $host_uuid = $anvil->Get->host_uuid({debug => 2});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_uuid => $host_uuid }});
+		
+		# Now update that we're online.
 		$anvil->Database->insert_or_update_hosts({
 			debug       => 2,
 			host_ipmi   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}, 
@@ -1567,10 +1585,10 @@ sub run_jobs
 			time_since_start => $time_since_start,
 			start_time       => $start_time,
 		}});
-		if ($time_since_start < 60)
+		if ($time_since_start < 30)
 		{
 			# Log that we'll start jobs in X seconds.
-			my $will_start_in = 60 - $time_since_start;
+			my $will_start_in = 30 - $time_since_start;
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "message_0326", variables => { will_start_in => $will_start_in }});
 			return(0);
 		}
@@ -1750,6 +1768,18 @@ sub run_jobs
 					# The job is running.
 					$anvil->data->{jobs_started}{$short_command} = $job_uuid;
 					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "jobs_started::${short_command}" => $anvil->data->{jobs_started}{$short_command} }});
+					
+					if ($short_command =~ /anvil-configure-host/)
+					{
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+							"pids::${job_picked_up_by}::cpu"         => $anvil->data->{pids}{$job_picked_up_by}{cpu}, 
+							"pids::${job_picked_up_by}::memory"      => $anvil->data->{pids}{$job_picked_up_by}{memory}, 
+							"pids::${job_picked_up_by}::state_codes" => $anvil->data->{pids}{$job_picked_up_by}{state_codes}, 
+							"pids::${job_picked_up_by}::start_time"  => $anvil->data->{pids}{$job_picked_up_by}{start_time}, 
+							"pids::${job_picked_up_by}::time"        => $anvil->data->{pids}{$job_picked_up_by}{'time'}, 
+							"pids::${job_picked_up_by}::command"     => $anvil->data->{pids}{$job_picked_up_by}{command}, 
+						}});
+					}
 				}
 			}
 			
@@ -1881,31 +1911,55 @@ sub run_jobs
 					}
 				}
 				
-				# Start the job, appending '--job-uuid' to the command.
+				### Start the job, appending '--job-uuid' to the command.
-				($anvil->data->{jobs}{handles}{$job_uuid}, my $return_code) = $anvil->System->call({
+				# If the job is 'anvil-configure-host', DO NOT background it! We need to call it and wait for it to return.
-					background  => 1, 
+				if ($job_command =~ /anvil-configure-host/)
-					stdout_file => "/tmp/anvil.job.".$job_uuid.".stdout",
+				{
-					stderr_file => "/tmp/anvil.job.".$job_uuid.".stderr", 
+					# Log that we're going to run this and wait. Our DB connection is 
-					shell_call  => $command, 
+					# likely going to break, so disconnect now.
-					source      => $THIS_FILE, 
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "warning_0178"});
-					line        => __LINE__,
+					$anvil->Database->disconnect();
-				});
+					my ($output, $return_code) = $anvil->System->call({
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+						debug      => 2,
-					"jobs::handles::${job_uuid}" => $anvil->data->{jobs}{handles}{$job_uuid}, 
+						shell_call => $command, 
-					return_code                  => $return_code,
+					});
-				}});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-				
+						output      => $output, 
-				# Log the PID (the job should update the database).
+						return_code => $return_code,
-				my $pid = $anvil->data->{jobs}{handles}{$job_uuid}->pid();
+					}});
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { pid => $pid }});
+					
-				
+					# Refresh and reconnect
-				# Record that we've tried to start this job, so that we don't try to restart it for any reason for at least a minute.
+					$anvil->refresh({debug => 2});
-				$anvil->data->{jobs}{$job_uuid}{started} = time;
+					$anvil->Database->connect({debug => 2, check_for_resync => 2});
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "jobs::${job_uuid}::started" => $anvil->data->{jobs}{$job_uuid}{started} }});
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0132"});
-				
+				}
-				# Record that a job with this command has started 
+				else
-				$anvil->data->{jobs_started}{$short_command} = $job_uuid;
+				{
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "jobs_started::${short_command}" => $anvil->data->{jobs_started}{$short_command} }});
+					($anvil->data->{jobs}{handles}{$job_uuid}, my $return_code) = $anvil->System->call({
+						background  => 1, 
+						stdout_file => "/tmp/anvil.job.".$job_uuid.".stdout",
+						stderr_file => "/tmp/anvil.job.".$job_uuid.".stderr", 
+						shell_call  => $command, 
+						source      => $THIS_FILE, 
+						line        => __LINE__,
+					});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+						"jobs::handles::${job_uuid}" => $anvil->data->{jobs}{handles}{$job_uuid}, 
+						return_code                  => $return_code,
+					}});
+					
+					# Log the PID (the job should update the database).
+					my $pid = $anvil->data->{jobs}{handles}{$job_uuid}->pid();
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { pid => $pid }});
+					
+					# Record that we've tried to start this job, so that we don't try to restart it for any reason for at least a minute.
+					$anvil->data->{jobs}{$job_uuid}{started} = time;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "jobs::${job_uuid}::started" => $anvil->data->{jobs}{$job_uuid}{started} }});
+					
+					# Record that a job with this command has started 
+					$anvil->data->{jobs_started}{$short_command} = $job_uuid;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "jobs_started::${short_command}" => $anvil->data->{jobs_started}{$short_command} }});
+				}
 			}
 		}
 	}

tools/anvil-join-anvil

@@ -2841,14 +2841,20 @@ sub check_local_network
 			output      => $output,
 			return_code => $return_code, 
 		}});
+		# NM seems to have a race issue, so we sleep a second after nmcli calls.
+		sleep 1;
 		
 		foreach my $interface_name (sort {$a cmp $b} @{$restart_interfaces})
 		{
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0462", variables => { interface => $interface_name }});
 			$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection down \"".$interface_name."\""});
+			# NM seems to have a race issue, so we sleep a second after nmcli calls.
+			sleep 1;
 			
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0464", variables => { interface => $interface_name }});
 			$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection up \"".$interface_name."\""});
+			# NM seems to have a race issue, so we sleep a second after nmcli calls.
+			sleep 1;
 		}
 		
 		# Wait for a DB connection. We'll wait up to 130 seconds (updelay is 120 seconds, plus a small buffer).

tools/fence_pacemaker

@@ -37,6 +37,8 @@
 # - 7   = Fence succeeded
 #
 # TODO: 
+# - Move Cluster->_set_server_constraint() to call here at the same time as we ban the peer.
+#  
 # - Read the CIB; 'pcs status xml' or '/usr/sbin/cibadmin --local --query' ?
 # -- Map the peer's name in pacemaker.
 # -- Verify that stonith is enabled:
@@ -44,7 +46,7 @@
 # -- Verify that we're quorate (a-la pacemaker):
 # - Verify that the resource is 'resource-and-stonith'
 # - Verify that the resource is 'UpToDate' (if not, should we suicide to give the other node priority, regardless of fence delay? what if both nodes have resources that are not UpToDate?)
-# - 
+# 
 
 ### NOTE: This doesn't use Anvil::Tools on purpose. We want to be quick and depend on as few things as 
 ###       possible.

tools/striker-auto-initialize-all

@@ -861,6 +861,7 @@ sub create_manifest
 	$anvil->Database->get_fences({debug => 2});
 	
 	# This is common for all Anvil! systems we might be building
+	### CGI is legacy
 	$anvil->data->{cgi}{prefix}{value} = $anvil->data->{base}{prefix};
 	$anvil->data->{cgi}{domain}{value} = $anvil->data->{base}{domain};
 	$anvil->data->{cgi}{mtu}{value}    = $anvil->data->{base}{mtu};
@@ -874,6 +875,19 @@ sub create_manifest
 		"cgi::ntp::value"    => $anvil->data->{cgi}{ntp}{value}, 
 	}});
 	
+	my $network_dns = $anvil->data->{base}{dns};
+	my $domain      = $anvil->data->{base}{domain};
+	my $network_mtu = $anvil->data->{base}{mtu};
+	my $network_ntp = $anvil->data->{base}{ntp};
+	my $name_prefix = $anvil->data->{base}{prefix};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		network_dns => $network_dns, 
+		domain      => $domain, 
+		network_mtu => $network_mtu, 
+		network_ntp => $network_ntp, 
+		name_prefix => $name_prefix, 
+	}});
+	
 	foreach my $anvil_number (sort {$a cmp $b} keys %{$anvil->data->{anvil}})
 	{
 		$anvil->data->{cgi}{sequence}{value} = $anvil_number;
@@ -1055,8 +1069,18 @@ sub create_manifest
 				}
 			}
 		}
+		
 		# Now generate the manifest.
-		my ($manifest_uuid, $manifest_name) = $anvil->Striker->generate_manifest({debug => 2});
+		my ($manifest_uuid, $manifest_name) = $anvil->Striker->generate_manifest({
+			debug         => 2,
+			manifest_uuid => "new",
+			dns           => $network_dns,
+			domain        => $domain, 
+			mtu           => $network_mtu, 
+			ntp           => $network_ntp, 
+			prefix        => $name_prefix, 
+			sequence      => $padded_sequence, 
+		});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			manifest_uuid => $manifest_uuid, 
 			manifest_name => $manifest_name,