* Several bugs fixed and enhancements;

* DRBD is now configured to a ping-timeout of 3 seconds.
* Created Log->switches() that returnes the command line switches used by Anvil! tool command line calls based on the active log levels / secure logging. Appended this to all invocations of our tools.
* Updated Database->resync_databases() to now only skip 'jobs' and 'variables' tables with less than 10 record differences. All other differences will trigger a resync.
* Created System->_check_anvil_conf() that, as you might guess, checks in anvil.conf exists and created it (using defaults), if not. It also checks to see if the 'admin' group and user exists and creates them, if not.
* Updated anvil-daemon to check anvil.conf on start up and in each loop. Created the function check_journald() that checks (and sets, if needed) that journald logging is persistent.
* Made striker-manage-peers to check_if_configured on the Database->connect() when updating anvil.conf and the target UUID is the local machine. Also created a loop to make the reconnection a lot more robust.

Signed-off-by: Digimer <digimer@alteeve.ca>

Anvil/Tools.pm

@@ -1031,6 +1031,7 @@ sub _set_paths
 				hosts				=>	"/etc/hosts", 
 				'httpd.conf'			=>	"/etc/httpd/conf/httpd.conf", 
 				'journald_anvil'		=>	"/etc/systemd/journald.conf.d/anvil.conf",
+				'journald.conf'			=>	"/etc/systemd/journald.conf", 
 				'lvm.conf'			=>	"/etc/lvm/lvm.conf",
 				'pg_hba.conf'			=>	"/var/lib/pgsql/data/pg_hba.conf",
 				'postgresql.conf'		=>	"/var/lib/pgsql/data/postgresql.conf",
@@ -1069,6 +1070,7 @@ sub _set_paths
 				firewalld_zones			=>	"/usr/lib/firewalld/zones",
 				html				=>	"/var/www/html",
 				ifcfg				=>	"/etc/sysconfig/network-scripts",
+				journald			=>	"/var/log/journal", 
 				resource_status			=>	"/sys/kernel/debug/drbd/resources",
 				scan_agents			=>	"/usr/sbin/scancore-agents",
 				shared				=>	{
@@ -1145,6 +1147,7 @@ sub _set_paths
 				getent				=>	"/usr/bin/getent",
 				gethostip			=>	"/usr/bin/gethostip",
 				'grep'				=>	"/usr/bin/grep", 
+				groupadd			=>	"/usr/sbin/groupadd",
 				head				=>	"/usr/bin/head",
 				hostnamectl			=>	"/usr/bin/hostnamectl",
 				hpacucli			=>	"/usr/sbin/hpacucli",
@@ -1223,6 +1226,7 @@ sub _set_paths
 				'tr'				=>	"/usr/bin/tr",
 				uname				=>	"/usr/bin/uname",
 				unzip				=>	"/usr/bin/unzip",
+				useradd				=>	"/usr/sbin/useradd",
 				usermod				=>	"/usr/sbin/usermod",
 				uuidgen				=>	"/usr/bin/uuidgen",
 				virsh				=>	"/usr/bin/virsh",

Anvil/Tools/DRBD.pm

@@ -2167,6 +2167,7 @@ sub update_global_common
 	my $after_sb_1pri_seen       = 0;
 	my $after_sb_2pri_seen       = 0;
 	my $timeout_seen             = 0;
+	my $ping_timeout_seen        = 0;
 	my $wfc_timeout_seen         = 0;
 	
 	my $in_global   = 0;
@@ -2395,6 +2396,18 @@ sub update_global_common
 						$new_global_common .= $new_line."\n";
 						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0518", variables => { file => $anvil->data->{path}{configs}{'global-common.conf'}, line => $line }});
 					}
+					
+					if (not $ping_timeout_seen)
+					{
+						   $update   = 1;
+						my $new_line = "\t\tping-timeout ".$say_ping_timeout.";";
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+							's1:update'   => $update,
+							's2:new_line' => $new_line, 
+						}});
+						$new_global_common .= $new_line."\n";
+						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0518", variables => { file => $anvil->data->{path}{configs}{'global-common.conf'}, line => $line }});
+					}
 				}
 				else
 				{
@@ -2768,7 +2781,7 @@ sub update_global_common
 					's2:middle_space' => $middle_space, 
 					's3:value'        => $value, 
 					's4:right_side'   => $right_side,
-					's5:after_sb_2pri_seen' => $after_sb_2pri_seen, 
+					's5:timeout_seen' => $timeout_seen, 
 				}});
 				
 				if ($value ne $say_timeout)
@@ -2785,6 +2798,36 @@ sub update_global_common
 					next;
 				}
 			}
+			
+			if ($line =~ /(\s*)ping-timeout(\s+)(.*?)(;.*)$/)
+			{
+				my $left_space        = $1;
+				my $middle_space      = $2;
+				my $value             = $3;
+				my $right_side        = $4;
+				   $ping_timeout_seen = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					's1:left_space'        => $left_space,
+					's2:middle_space'      => $middle_space, 
+					's3:value'             => $value, 
+					's4:right_side'        => $right_side,
+					's5:ping_timeout_seen' => $ping_timeout_seen, 
+				}});
+				
+				if ($value ne $say_ping_timeout)
+				{
+					   $update   = 1;
+					my $new_line = $left_space."ping-timeout".$middle_space.$say_ping_timeout.$right_side;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						's1:update'   => $update,
+						's2:new_line' => $new_line, 
+					}});
+					
+					$new_global_common .= $new_line.$comment."\n";
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0518", variables => { file => $anvil->data->{path}{configs}{'global-common.conf'}, line => $line }});
+					next;
+				}
+			}
 		}
 		
 		# Add this line (will have 'next'ed if the line was modified before getting here).

Anvil/Tools/Database.pm

@@ -1383,6 +1383,7 @@ sub connect
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			db_connect_string => $db_connect_string, 
 			user              => $user, 
+			password          => $anvil->Log->is_secure($password),
 		}});
 		local $@;
 		my $test = eval { $dbh = DBI->connect($db_connect_string, $user, $password, {
@@ -1390,6 +1391,10 @@ sub connect
 			AutoCommit     => 1,
 			pg_enable_utf8 => 1
 		}); };
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			's1:test' => $test,
+			's2:$@'   => $@,
+		}});
 		if (not $test)
 		{
 			# Something went wrong...
@@ -1793,6 +1798,7 @@ sub disconnect
 		
 		$anvil->data->{cache}{database_handle}{$uuid}->disconnect;
 		delete $anvil->data->{cache}{database_handle}{$uuid};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
 	}
 	
 	# Delete the stored DB-related values.
@@ -14707,7 +14713,7 @@ sub resync_databases
 		# If the 'schema' is 'public', there is no table in the history schema. If there is a host 
 		# column, the resync will be restricted to entries from this host uuid.
 		my $schema      = $anvil->data->{sys}{database}{table}{$table}{schema}      ? $anvil->data->{sys}{database}{table}{$table}{schema}      : "public";
-		my $host_column = $anvil->data->{sys}{database}{table}{$table}{host_column};
+		my $host_column = $anvil->data->{sys}{database}{table}{$table}{host_column} ? $anvil->data->{sys}{database}{table}{$table}{host_column} : "";
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			table       => $table, 
 			schema      => $schema, 
@@ -16110,13 +16116,13 @@ ORDER BY
 					"s3:sys::database::table::${table}::uuid::${uuid}::row_count" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count}, 
 				}});
 				
-				# To avoid resyncs triggered by the differences that might occur if the row 
+				if ((($table eq "jobs") or ($table eq "variables")) && ($difference < 10))
-				# count changed slightly between counting both/all DBs, we won't resync 
-				# until there's at least 10 rows different. The exception is the hosts file,
-				# as it needs to resync on a single line difference when adding peer Striker
-				# dashboards.
-				if (($table eq "hosts") or ($difference > 10))
 				{
+					# These often fall out of sync and trigger resyncs when in fact it 
+					# was just a change that happened between counting columns. 
+					next;
+				}
+				
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0219", variables => { 
 					missing => $difference, 
 					table   => $table, 
@@ -16129,7 +16135,6 @@ ORDER BY
 				last;
 			}
 		}
-		}
 		last if $anvil->data->{sys}{database}{resync_needed};
 	}
 	

Anvil/Tools/Log.pm

@@ -19,6 +19,7 @@ my $THIS_FILE = "Log.pm";
 # language
 # level
 # secure
+# switches
 # variables
 # _adjust_log_level
 
@@ -744,6 +745,59 @@ sub secure
 	return($anvil->data->{defaults}{'log'}{secure});
 }
 
+
+=head2 switches
+
+This method returns switches to append to Alteeve tools to pass the active log level (and log secure) to our tools we call as shell calls.
+
+Examples;
+
+If the active log level is 1, and we're not logging secure messages;
+
+ my $switches = $anvil->Log->switches();
+ 
+In this case, C<< $switches >> would contain C<< -v >>.
+
+If the active log level is 2, and we are logging secure messages;
+
+ my $switches = $anvil->Log->switches();
+ 
+In this case, C<< $switches >> would contain C<< -vv --log-secure >>.
+
+B<< Note >>: The string returned is padded with a leading space so that this method can be called directly after the executable. Example;
+
+ my $shell_call = $anvil->data->{path}{exe}{'striker-prep-database'}.$anvil->Log->switches();
+
+This method takes no parameters.
+
+=cut
+sub switches
+{
+	my $self      = shift;
+	my $parameter = shift;
+	my $anvil     = $self->parent;
+	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 0;
+	
+	my $log_level  = $anvil->Log->level;
+	my $log_secure = $anvil->Log->secure;
+	my $switches   = "";
+	if ($log_level)
+	{
+		$switches .= " -";
+		for (1..$log_level)
+		{
+			$switches .= "v";
+		}
+	}
+	if ($log_secure)
+	{
+		$switches .= " --log-secure";
+	}
+	
+	return($switches);
+}
+
+
 =head2 variables
 
 This is a special method used in testing and debugging for logging a certain number of variables. It takes a hash reference via the 'C<< variables >>' parameter and creates a raw log entry showing the variables as 'C<< variable: [value] >>' pairs.

Anvil/Tools/ScanCore.pm

@@ -1413,7 +1413,7 @@ sub post_scan_analysis_node
 				$anvil->Email->send_alerts();
 				
 				# Shutdown using 'anvil-safe-stop' and set the reason to 'power'
-				my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason power --power-off";
+				my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason power --power-off".$anvil->Log->switches;
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 				$anvil->System->call({shell_call => $shell_call});
 				
@@ -1446,7 +1446,7 @@ sub post_scan_analysis_node
 				$anvil->Email->send_alerts();
 				
 				# Shutdown using 'anvil-safe-stop' and set the reason to 'thermal'
-				my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason thermal --power-off";
+				my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason thermal --power-off".$anvil->Log->switches;
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 				$anvil->System->call({shell_call => $shell_call});
 				
@@ -1474,7 +1474,7 @@ sub post_scan_analysis_node
 				$anvil->Alert->register({alert_level => "notice", message => "warning_0087", set_by => "ScanCore"});
 				$anvil->Email->send_alerts();
 				
-				my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason thermal --power-off";
+				my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason thermal --power-off".$anvil->Log->switches;
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 				$anvil->System->call({shell_call => $shell_call});
 				
@@ -1489,7 +1489,7 @@ sub post_scan_analysis_node
 				$anvil->Alert->register({alert_level => "notice", message => "warning_0088", set_by => "ScanCore"});
 				$anvil->Email->send_alerts();
 				
-				my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason power --power-off";
+				my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason power --power-off".$anvil->Log->switches;
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 				$anvil->System->call({shell_call => $shell_call});
 				
@@ -1561,7 +1561,7 @@ sub post_scan_analysis_node
 						$anvil->Email->send_alerts();
 						
 						# Shutdown using 'anvil-safe-stop' and set the reason to 'power_off'
-						my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason ".$power_off." --power-off";
+						my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason ".$power_off." --power-off".$anvil->Log->switches;
 						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 						$anvil->System->call({shell_call => $shell_call});
 						
@@ -1601,7 +1601,7 @@ sub post_scan_analysis_node
 					$anvil->Email->send_alerts();
 					
 					# Pull the server.
-					my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all";
+					my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all".$anvil->Log->switches;
 					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 					$anvil->System->call({shell_call => $shell_call});
 					
@@ -1632,7 +1632,7 @@ sub post_scan_analysis_node
 						$anvil->Alert->register({alert_level => "notice", message => "warning_0097", set_by => "ScanCore"});
 						$anvil->Email->send_alerts();
 						
-						my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason power --power-off";
+						my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason power --power-off".$anvil->Log->switches;
 						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 						$anvil->System->call({shell_call => $shell_call});
 						
@@ -1647,7 +1647,7 @@ sub post_scan_analysis_node
 						$anvil->Alert->register({alert_level => "notice", message => "warning_0096", set_by => "ScanCore"});
 						$anvil->Email->send_alerts();
 						
-						my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason power --power-off";
+						my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason power --power-off".$anvil->Log->switches;
 						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 						$anvil->System->call({shell_call => $shell_call});
 						
@@ -1675,7 +1675,7 @@ sub post_scan_analysis_node
 						$anvil->Alert->register({alert_level => "notice", message => "warning_0099", set_by => "ScanCore"});
 						$anvil->Email->send_alerts();
 						
-						my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason thermal --power-off";
+						my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason thermal --power-off".$anvil->Log->switches;
 						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 						$anvil->System->call({shell_call => $shell_call});
 						
@@ -1690,7 +1690,7 @@ sub post_scan_analysis_node
 						$anvil->Alert->register({alert_level => "notice", message => "warning_0100", set_by => "ScanCore"});
 						$anvil->Email->send_alerts();
 						
-						my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason thermal --power-off";
+						my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason thermal --power-off".$anvil->Log->switches;
 						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 						$anvil->System->call({shell_call => $shell_call});
 						
@@ -1725,7 +1725,7 @@ sub post_scan_analysis_node
 					$anvil->Email->send_alerts();
 					
 					# Pull the server.
-					my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all";
+					my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all".$anvil->Log->switches;
 					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 					$anvil->System->call({shell_call => $shell_call});
 					
@@ -1750,7 +1750,7 @@ sub post_scan_analysis_node
 					$anvil->Alert->register({alert_level => "notice", message => "warning_0106", set_by => "ScanCore"});
 					$anvil->Email->send_alerts();
 					
-					my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason thermal --power-off";
+					my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason thermal --power-off".$anvil->Log->switches;
 					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 					$anvil->System->call({shell_call => $shell_call});
 					
@@ -1767,7 +1767,7 @@ sub post_scan_analysis_node
 						$anvil->Alert->register({alert_level => "notice", message => "warning_0104", set_by => "ScanCore"});
 						$anvil->Email->send_alerts();
 						
-						my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason thermal --power-off";
+						my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason thermal --power-off".$anvil->Log->switches;
 						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 						$anvil->System->call({shell_call => $shell_call});
 						
@@ -1812,7 +1812,7 @@ sub post_scan_analysis_node
 							$anvil->Alert->register({alert_level => "warning", message => "warning_0084", set_by => "ScanCore", variables => $variables});
 							$anvil->Email->send_alerts();
 							
-							my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all";
+							my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all".$anvil->Log->switches;
 							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 							$anvil->System->call({shell_call => $shell_call});
 							
@@ -1868,7 +1868,7 @@ sub post_scan_analysis_node
 					}
 					$anvil->Email->send_alerts();
 					
-					my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all";
+					my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all".$anvil->Log->switches;
 					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 					$anvil->System->call({shell_call => $shell_call});
 					
@@ -1901,7 +1901,7 @@ sub post_scan_analysis_node
 					}
 					$anvil->Email->send_alerts();
 					
-					my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason ".$load_shed." --power-off";
+					my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --stop-reason ".$load_shed." --power-off".$anvil->Log->switches;
 					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 					$anvil->System->call({shell_call => $shell_call});
 					
@@ -1931,7 +1931,7 @@ sub post_scan_analysis_node
 						}
 						$anvil->Email->send_alerts();
 						
-						my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all";
+						my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all".$anvil->Log->switches;
 						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 						$anvil->System->call({shell_call => $shell_call});
 						
@@ -1966,7 +1966,7 @@ sub post_scan_analysis_node
 						}
 						$anvil->Email->send_alerts();
 						
-						my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all";
+						my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all".$anvil->Log->switches;
 						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 						$anvil->System->call({shell_call => $shell_call});
 						
@@ -2018,7 +2018,7 @@ sub post_scan_analysis_node
 						}
 						$anvil->Email->send_alerts();
 						
-						my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all";
+						my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all".$anvil->Log->switches;
 						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }});
 						$anvil->System->call({shell_call => $shell_call});
 						

Anvil/Tools/System.pm

@@ -46,6 +46,7 @@ my $THIS_FILE = "System.pm";
 # stop_daemon
 # stty_echo
 # update_hosts
+# _check_anvil_conf
 # _load_firewalld_zones
 # _load_specific_firewalld_zone
 # _match_port_to_service
@@ -528,6 +529,7 @@ sub change_shell_user_password
 	return($return_code);
 }
 
+
 =head2 check_daemon
 
 This method checks to see if a daemon is running or not. If it is, it returns 'C<< 1 >>'. If the daemon isn't running, it returns 'C<< 0 >>'. If the daemon wasn't found, 'C<< 2 >>' is returned.
@@ -635,7 +637,7 @@ sub check_memory
 	
 	my $used_ram = 0;
 	
-	my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{'anvil-check-memory'}." --program $program_name"});
+	my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{'anvil-check-memory'}." --program $program_name".$anvil->Log->switches});
 	foreach my $line (split/\n/, $output)
 	{
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
@@ -5067,6 +5069,78 @@ sub update_hosts
 # Private functions                                                                                         #
 #############################################################################################################
 
+
+=head2 _check_anvil_conf
+
+This looks for anvil.conf and, if it's missing, regenerates it.
+
+=cut
+sub _check_anvil_conf
+{
+	my $self      = shift;
+	my $parameter = shift;
+	my $anvil     = $self->parent;
+	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "System->_check_anvil_conf" }});
+	
+	# Make sure the 'admin' user exists...
+	my $admin_uid = getpwnam('admin');
+	my $admin_gid = getgrnam('admin');
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		admin_uid => $admin_uid,
+		admin_gid => $admin_gid, 
+	}});
+	if (not $admin_gid)
+	{
+		# Create the admin group 
+		my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{groupadd}." --system admin"});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			output      => $output, 
+			return_code => $return_code, 
+		}});
+		
+		$admin_gid = getgrnam('admin');
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "warning_0118", variables => { gid => $admin_gid }});
+	}
+	if (not $admin_uid)
+	{
+		# Create the admin user
+		my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{useradd}." --create-home --gid admin --comment \"Anvil! user account\" admin"});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			output      => $output, 
+			return_code => $return_code, 
+		}});
+		
+		my $admin_uid = getpwnam('admin');
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "warning_0119", variables => { uid => $admin_gid }});
+	}
+	
+	# Does the file exist?
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "path::configs::anvil.conf" => $anvil->data->{path}{configs}{'anvil.conf'} }});
+	if (not -e $anvil->data->{path}{configs}{'anvil.conf'})
+	{
+		
+		# Nope! What the hell? Create it.
+		my $failed = $anvil->Storage->write_file({
+			debug     => $debug,
+			overwrite => 1, 
+			file      => $anvil->data->{path}{configs}{'anvil.conf'}, 
+			body      => $anvil->Words->string({key => "file_0002"}), 
+			user      => "admin", 
+			group     => "admin", 
+			mode      => "0644", 
+		});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }});
+		if (not $failed)
+		{
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "warning_0117", variables => { file => $anvil->data->{path}{configs}{'anvil.conf'} }});
+		}
+		return($failed);
+	}
+	
+	return(0);
+}
+
 =head2 _load_firewalld_zones
 
 This reads in the XML files for all of the firewalld zones.

anvil.conf

@@ -88,10 +88,9 @@ sys::database::maximum_batch_size				=	25000
 
 
 ### Apache stuff
-# By default, we try to determine the host type using the host name. The rules used for this can be seen in 
+# By default, we try to determine the host type which anvil RPM is installed. If, for some reason, you want
-# 'perldoc Anvil::Tools::System -> determine_host_type'. If you are using non-standard host names, or for some
+# to statically assign the host type, you can do so with this variable. Note that this sets the host type of
-# other reason want to statically assign the host type, you can do so with this variable. Note that this sets
+# this host only. You will need to set this appropriately on other hosts.
-# the host type of this host only. You will need to set this appropriately on other hosts.
 # 
 # Normally, you should not need to set this.
 #sys::host_type							=	node

cgi-bin/striker

@@ -1312,7 +1312,7 @@ sub process_keys
 			my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
 				file            => $THIS_FILE, 
 				line            => __LINE__, 
-				job_command     => $anvil->data->{path}{exe}{'anvil-manage-keys'}, 
+				job_command     => $anvil->data->{path}{exe}{'anvil-manage-keys'}.$anvil->Log->switches, 
 				job_data        => $job_data, 
 				job_name        => "manage::broken_keys", 
 				job_title       => "job_0056", 
@@ -1483,7 +1483,7 @@ WHERE
 			file            => $THIS_FILE, 
 			line            => __LINE__,
 			job_host_uuid   => "all", 
-			job_command     => $anvil->data->{path}{exe}{'anvil-sync-shared'}, 
+			job_command     => $anvil->data->{path}{exe}{'anvil-sync-shared'}.$anvil->Log->switches, 
 			job_data        => "file_uuid=".$file_uuid, 
 			job_name        => "storage::purge", 
 			job_title       => "job_0136", 
@@ -1525,7 +1525,7 @@ WHERE
 				file            => $THIS_FILE, 
 				line            => __LINE__,
 				job_host_uuid   => "all", 
-				job_command     => $anvil->data->{path}{exe}{'anvil-sync-shared'}, 
+				job_command     => $anvil->data->{path}{exe}{'anvil-sync-shared'}.$anvil->Log->switches, 
 				job_data        => "file_uuid=".$file_uuid."\nold_name=".$file_name."\nnew_name=".$anvil->data->{cgi}{new_file_name}{value}, 
 				job_name        => "storage::rename", 
 				job_title       => "job_0138", 
@@ -1563,7 +1563,7 @@ WHERE
 				file            => $THIS_FILE, 
 				line            => __LINE__,
 				job_host_uuid   => "all", 
-				job_command     => $anvil->data->{path}{exe}{'anvil-sync-shared'}, 
+				job_command     => $anvil->data->{path}{exe}{'anvil-sync-shared'}.$anvil->Log->switches, 
 				job_data        => "file_uuid=".$file_uuid, 
 				job_name        => "storage::check_mode", 
 				job_title       => "job_0143", 
@@ -1653,7 +1653,7 @@ WHERE
 							file            => $THIS_FILE, 
 							line            => __LINE__,
 							job_host_uuid   => $host_uuid, 
-							job_command     => $anvil->data->{path}{exe}{'anvil-sync-shared'}, 
+							job_command     => $anvil->data->{path}{exe}{'anvil-sync-shared'}.$anvil->Log->switches, 
 							job_data        => "file_uuid=".$file_uuid, 
 							job_name        => $job_type, 
 							job_title       => $job_title, 
@@ -2179,7 +2179,7 @@ sub run_manifest
 				file            => $THIS_FILE, 
 				line            => __LINE__,
 				job_host_uuid   => $node1_host_uuid, 
-				job_command     => $anvil->data->{path}{exe}{'anvil-join-anvil'}, 
+				job_command     => $anvil->data->{path}{exe}{'anvil-join-anvil'}.$anvil->Log->switches, 
 				job_data        => "as_machine=node1,manifest_uuid=".$manifest_uuid.",anvil_uuid=".$anvil_uuid, 
 				job_name        => "join_anvil::node1", 
 				job_title       => "job_0072", 
@@ -2193,7 +2193,7 @@ sub run_manifest
 				file            => $THIS_FILE, 
 				line            => __LINE__,
 				job_host_uuid   => $node2_host_uuid, 
-				job_command     => $anvil->data->{path}{exe}{'anvil-join-anvil'}, 
+				job_command     => $anvil->data->{path}{exe}{'anvil-join-anvil'}.$anvil->Log->switches, 
 				job_data        => "as_machine=node2,manifest_uuid=".$manifest_uuid.",anvil_uuid=".$anvil_uuid, 
 				job_name        => "join_anvil::node2", 
 				job_title       => "job_0072", 
@@ -2209,7 +2209,7 @@ sub run_manifest
 					file            => $THIS_FILE, 
 					line            => __LINE__,
 					job_host_uuid   => $dr1_host_uuid, 
-					job_command     => $anvil->data->{path}{exe}{'anvil-join-anvil'}, 
+					job_command     => $anvil->data->{path}{exe}{'anvil-join-anvil'}.$anvil->Log->switches, 
 					job_data        => "as_machine=dr1,manifest_uuid=".$manifest_uuid.",anvil_uuid=".$anvil_uuid, 
 					job_name        => "join_anvil::dr1", 
 					job_title       => "job_0072", 
@@ -5831,7 +5831,7 @@ sub process_prep_network
 				file            => $THIS_FILE, 
 				line            => __LINE__,
 				job_host_uuid   => $anvil->data->{cgi}{host_uuid}{value}, 
-				job_command     => $anvil->data->{path}{exe}{'anvil-configure-host'}, 
+				job_command     => $anvil->data->{path}{exe}{'anvil-configure-host'}.$anvil->Log->switches, 
 				job_data        => "form::config_step2", 
 				job_name        => "configure::network", 
 				job_title       => "job_0001", 
@@ -6083,7 +6083,7 @@ sub process_prep_host_page
 	if (($connect) && ($confirm))
 	{
 		# Save the job
-		my $job_command = $anvil->data->{path}{exe}{'striker-initialize-host'};
+		my $job_command = $anvil->data->{path}{exe}{'striker-initialize-host'}.$anvil->Log->switches;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			job_command => $job_command,
 			password    => $anvil->Log->is_secure($host_password),
@@ -6369,7 +6369,7 @@ sub process_install_target
 		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
 			file            => $THIS_FILE, 
 			line            => __LINE__, 
-			job_command     => $anvil->data->{path}{exe}{'striker-manage-install-target'}." --".$anvil->data->{cgi}{subtask}{value}, 
+			job_command     => $anvil->data->{path}{exe}{'striker-manage-install-target'}." --".$anvil->data->{cgi}{subtask}{value}.$anvil->Log->switches, 
 			job_data        => "", 
 			job_name        => "install-target::".$anvil->data->{cgi}{task}{value}, 
 			job_title       => "job_0015", 
@@ -6412,14 +6412,14 @@ sub process_power
 	if ($anvil->data->{cgi}{confirm}{value})
 	{
 		# Record the job!
-		my $job_command     = $anvil->data->{path}{exe}{'anvil-manage-power'}." --reboot -y";
+		my $job_command     = $anvil->data->{path}{exe}{'anvil-manage-power'}." --reboot -y".$anvil->Log->switches;
 		my $job_title       = "job_0009";
 		my $job_description = "job_0006";
 		my $say_title       = "#!string!job_0005!#";
 		my $say_description = "#!string!job_0006!#";
 		if ($task eq "poweroff")
 		{
-			$job_command     = $anvil->data->{path}{exe}{'anvil-manage-power'}." --poweroff -y";
+			$job_command     = $anvil->data->{path}{exe}{'anvil-manage-power'}." --poweroff -y".$anvil->Log->switches;
 			$job_title       = "job_0010";
 			$job_description = "job_0008";
 			$say_title       = "#!string!job_0007!#";
@@ -6478,7 +6478,7 @@ sub process_update
 			debug           => 1,
 			file            => $THIS_FILE, 
 			line            => __LINE__, 
-			job_command     => $anvil->data->{path}{exe}{'anvil-update-system'}, 
+			job_command     => $anvil->data->{path}{exe}{'anvil-update-system'}.$anvil->Log->switches, 
 			job_data        => "", 
 			job_name        => "update::system", 
 			job_title       => "job_0003", 
@@ -6740,7 +6740,7 @@ sub delete_sync_peer
 			debug           => 3,
 			file            => $THIS_FILE, 
 			line            => __LINE__, 
-			job_command     => $anvil->data->{path}{exe}{'striker-manage-peers'}." --remove --host-uuid ".$uuid, 
+			job_command     => $anvil->data->{path}{exe}{'striker-manage-peers'}." --remove --host-uuid ".$uuid.$anvil->Log->switches, 
 			job_data        => "", 
 			job_name        => "striker-peer::remove", 
 			job_title       => "job_0013", 
@@ -6955,7 +6955,7 @@ sub add_sync_peer
 		if ($anvil->data->{cgi}{confirm}{value})
 		{
 			# OK, save the job!
-			my $job_command = $anvil->data->{path}{exe}{'striker-manage-peers'}." --add --host-uuid ".$peer_host_uuid." --host ".$host." --port ".$pgsql_port." --ping ".$ping;
+			my $job_command = $anvil->data->{path}{exe}{'striker-manage-peers'}." --add --host-uuid ".$peer_host_uuid." --host ".$host." --port ".$pgsql_port." --ping ".$ping.$anvil->Log->switches;
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 				job_command => $job_command,
 				password    => $anvil->Log->is_secure($password),
@@ -6975,7 +6975,7 @@ sub add_sync_peer
 				# See which of our IPs match theirs. If the peer is a host name, first
 				my $host_uuid   =  $anvil->Get->host_uuid;
 				my $sql_port    =  $anvil->data->{database}{$host_uuid}{port};
-				my $job_command =  $anvil->data->{path}{exe}{'striker-manage-peers'}." --add --host-uuid ".$host_uuid." --host ".$use_ip." --port ".$sql_port." --ping ".$ping;
+				my $job_command =  $anvil->data->{path}{exe}{'striker-manage-peers'}." --add --host-uuid ".$host_uuid." --host ".$use_ip." --port ".$sql_port." --ping ".$ping.$anvil->Log->switches;
 				   $job_data    .= "\npeer_job_command=".$job_command;
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 					job_command => $job_command,
@@ -7096,7 +7096,7 @@ sub configure_striker
 			debug           => 3,
 			file            => $THIS_FILE, 
 			line            => __LINE__, 
-			job_command     => $anvil->data->{path}{exe}{'anvil-configure-host'}, 
+			job_command     => $anvil->data->{path}{exe}{'anvil-configure-host'}.$anvil->Log->switches, 
 			job_data        => "form::config_step2", 
 			job_name        => "configure::network", 
 			job_title       => "job_0001", 
@@ -8463,7 +8463,7 @@ sub get_network_details
 	
 	### TODO: Daemonize this or solve selinux issues
 	### Refresh the network.xml
-	#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{'anvil-update-states'}});
+	#$anvil->System->call({shell_call => $anvil->data->{path}{exe}{'anvil-update-states'}.$anvil->Log->switches});
 	
 	# Now read the network.xml
 	my $file = $anvil->data->{path}{directories}{html}."/status/network.xml";

scancore-agents/scan-cluster/scan-cluster

@@ -13,6 +13,9 @@
 # 2 = Not a cluster member
 # 
 # TODO: 
+# - When a node is lost, update the location constraints to keep the servers on the surviving node when the
+#   peer returns.
+# - Test that the fence delay favours the host that has all the servers.
 # 
 
 use strict;

share/words.xml

@@ -417,7 +417,7 @@ The attempt to start the servers appears to have failed. The return code '0' was
 		<key name="error_0303">The passed in host UUID: [#!variable!host_uuid!#] was not found in the database.</key>
 		
 		<!-- Files templates -->
-		<!-- NOTE: Translating these files requires an understanding of which likes are translatable -->
+		<!-- NOTE: Translating these files requires an understanding of which lines are translatable -->
 		<!--       comments and which are command lines that can't be changed! -->
 		<key name="file_0001"><![CDATA[# Resource for #!variable!server!#
 resource #!variable!server!# {
@@ -457,6 +457,203 @@ resource #!variable!server!# {
 		}
 	}
 }
+]]></key>
+		<!-- NOTE: If the core anvil.conf is updated, remember to update this as well. -->
+		<key name="file_0002"><![CDATA[### This is the main Anvil! configuration file.
+
+# To help understand how the Anvil! is used, some features will "call home" to record anonymous information
+# about a machine participating in an Anvil! system. An example is DRBD's 'usage-count' option. If you wish 
+# to not have this happen, set this to '1'.
+sys::privacy::strong						=	#!data!sys::privacy::strong!#
+
+### Features
+# Normally, if one node in the Anvil! is healthier than the other, it will pull the servers from the peer
+# on to it. This is a process called "preventative live migration". If you would like to disable this 
+# feature, set this to '1'.
+feature::scancore::disable::preventative-live-migration		=	0
+
+### Database
+# Database connections;
+# 
+# Each Anvil! database is defined below using an incrementing counter as the second variable. The value of 
+# the second variable is the local 'host_uuid' (via: dmidecode --string system-uuid | perl -ne 'print lc').
+# This ensures that entries can be moved and copied without causing conflicts.
+# 
+# There are six variables for each database definition;
+# host     = This is the (resolvable) host name or IP address of the peer database.
+# port     = This is the TCP port used to connect to the database. By default, it is 5432
+# password = This is the password used to connect to the database.
+#            NOTE: Do not directly change the database password. Please use 'anvil-update-password' so that
+#                  the WebUI, database, nodes and peers are all kept in sync.
+# ping     = If set, this will cause a ping to be made against the database server before the actual 
+#            connection is attempted. This can be useful when peer dashboards are offline for extended 
+#            periods of time. The value can be any real number and will be used as the timeout for the actual
+#            ping. If the peer dashboard is on the same subnet, a value of '0.25' (250ms) should be a good 
+#            balance between giving the peer a chance to reply and not waiting too long when it is offline. 
+#            If the peer is remote, you will want to set this to '1' (1000ms) or higher. 
+#            
+#            Setting this to '0' disables pinging before connecting entirely. In this case, if the peer is
+#            offline, it will take about 3 seconds on average for the connection attempt to timeout. This 
+#            might be necessary if the peer dashboard is behind a firewall/router or otherwise can't respond
+#            to ICMP pings.
+# NOTE: The database is called 'anvil' and the database owner is 'admin'.
+#database::eec27c2f-2308-4b4f-bd81-24118b53f8a3::host		=	localhost
+#database::eec27c2f-2308-4b4f-bd81-24118b53f8a3::port		=	5432
+#database::eec27c2f-2308-4b4f-bd81-24118b53f8a3::password	=	Initial1
+#database::eec27c2f-2308-4b4f-bd81-24118b53f8a3::ping		=	1
+
+# Below are the databases configured for use by this system. Please be careful manually editing them. They 
+# are updated by Striker and changes can be overwritten. Please do not alter or remove the 'start db list' 
+# and 'end db list' comments below.
+### start db list ###
+### end db list ###
+
+# To keep Anvil!'s database growth in check, an auto-archive mechanism is used by some agents where, at the 
+# end of each scan, the number of records in the history schema for a given table are counted (restricted to 
+# the agent's host, when appropriate). 
+# 
+# When the number exceeds the trigger, the number of records that are archived is approximately (number of 
+# records above trigger + 'count'). This is not an exact number because a representative timestamp will be 
+# selected from the hostory schema at this count, and then any record equal to or older than the time stamp 
+# is removed.
+# 
+# To protect against the potential of using too much disk space, archives are off by default. Under normal 
+# behaviour, old records are simple removed. To enable the archive function, set this to '1'.
+#sys::database::archive::save_to_disk				=	1
+# 
+# When archiving to disk is enabled, to protect against large memory use or long archive times in the case 
+# where the number of records to archive are particularly large, the 'division' value is used to break up the
+# archive job into "chunks". Generally speaking, the division should be greater than the count, and never be 
+# needed. However, if the archive process takes too long, or if the archive was triggered well above the 
+# trigger value, the division can help prevent using too much memory at once. If division is set to '0', 
+# archive jobs will never be divided.
+# 
+# The archives are all stored in the specified directory using the name format 
+# '<agent>.<table>.<timestamp>.bz2' and the archives are synced between dashboards for safe keeping. Archive 
+# files are never removed automatically.
+# 
+# To disable auto-archiving entirely, set 'trigger' to '0'.
+# 
+# NOTE: If the archive directory doesn't exist, Anvil! will create it 
+#       automatically the first time it is needed.
+sys::database::archive::compress				=	1
+sys::database::archive::trigger					=	50000
+sys::database::archive::count					=	25000
+sys::database::archive::division				=	30000
+sys::database::archive::directory				=	/usr/local/anvil/archives/
+
+# This puts a limit on how many queries (writes, generally) to make in a single batch transaction. This is 
+# useful when doing very large transacions, like resync'ing a large table, by limiting how long a given 
+# transaction can take and how much memory is used. 
+sys::database::maximum_batch_size				=	25000
+
+
+### Apache stuff
+# By default, we try to determine the host type which anvil RPM is installed. If, for some reason, you want
+# to statically assign the host type, you can do so with this variable. Note that this sets the host type of
+# this host only. You will need to set this appropriately on other hosts.
+# 
+# Normally, you should not need to set this.
+#sys::host_type							=	node
+# This configuration file provides a way to override Anvil::Tools' built-in defaults.
+
+# This controls the default language. The value is the ISO code of the country's language you want to use by
+# default. Note that the logging language is set with 'defaults::log::language' below.
+# NOTE: Be sure the language exists before changing it!
+defaults::languages::output					=	en_CA
+
+# This controls how many loops Anvil::Tools::Words is allow to make while processing a string. This acts as a 
+# mechanism to exit infinite loops, and generally should not need to be changed.
+defaults::limits::string_loops					=	1000
+
+
+### Logging options
+# This controls whether all database transactions are recorded or not. Genreally this should be left off 
+# unless you are debugging the program.
+# WARNING: This ignores 'secure', and will always be logged. Be careful about exposing sensitive data!
+sys::database::log_transactions					=	0
+
+# By default, if a configured database is not accessible, a log level 1 alert is registered. This can cause a
+# lot of log traffic. If you want to silence these log alerts, you can set the value below to be higher than
+# your current active log level (default is '1', so set to '2' or '3' to silence).
+# NOTE: It's important to only use this temporarily. 
+sys::database::failed_connection_log_level			=	1
+
+# This controls what log facility to use by default.
+# NOTE: This will always be 'authpriv' when a log entry is marked as secure.
+defaults::log::facility						=	local0
+
+# This controls what language logs are recorded in. Be sure that the language exists before changing it!
+defaults::log::language						=	en_CA
+
+# This controls the default log level. See 'perldoc Anvil::Tools::Logs' for details.
+defaults::log::level						=	1
+
+# This controls whether sensitive log entries are logged or not. Generally, this should be left disabled!
+defaults::log::secure						=	0
+
+# THis sets the default log server to send the log entries to. Leave it blank in most cases.
+#defaults::log::server						=	
+
+# This sets the default log tag used when logging an entry. Most programs will likely override this.
+defaults::log::tag						=	anvil
+
+
+### Templates
+# This sets the default template used when rendering HTML pages. It must be the same as the directory name
+# under /var/www/html/skins/
+defaults::template::html					=	alteeve
+
+
+### Install Target options
+# Note; Please see 'pxe.txt' for editable templates for 'dhcpd.conf', (tftpboot's BIOS) 'default' and the
+#       kickstart templates.
+# 
+# This section allows for adapting certain installations of systems via the Install Target feature. 
+# Generally, these don't need to be edited.
+# 
+# This controls the keyboard configuration. See: 
+# - https://docs.fedoraproject.org/en-US/fedora/f28/install-guide/appendixes/Kickstart_Syntax_Reference/#sect-kickstart-commands-keyboard
+#kickstart::keyboard						=	--vckeymap=us --xlayouts='us'
+#
+# This sets the default password of newly stage-1 built machines. Generally, this shouldn't be change. It is
+# recorded in plain text and it is used in the stage-2 configuration tools. 
+#kickstart::password						=	Initial1
+#
+# This is the system timezone to be set. Generally, it's recommended to leave the Anvil! machines to UTC, but
+# you might want to change this is if you spend time working directly on the various Anvil! cluster machines.
+#kickstart::timezone						=	Etc/GMT --isUtc
+
+# If this is set to '1', the packages used to build machines via the Install Target feature will not 
+# auto-update.
+install-manifest::refresh-packages				=	1
+
+# This controls how often the local RPM repository is checked for updates. The default is '86400' seconds 
+# (one day). If anything, you might want to increase this. Common values;
+# 86400   = Once per day
+# 604800  = Once per week
+# 2419200 = Once per month (well, 4 weeks)
+install-manifest::refresh-period				=	86400
+
+
+### This controls Striker-specific features
+# This can be set a a comma-separated list of packages to be added to Striker's RPM repos. Note that this is
+# only useful is you want to store EL8-specific packages needed outside the Anvil!.
+striker::repo::extra-packages					=	
+
+### System functions
+# The machines used in the Anvil! are treated as appliances, and thus fully under our control. As such, much 
+# of the system is monitored, managed and auto-repaired. This can frustrate sysadmins. As such, an admin may
+# use the 'system::*' options to retake control over some system behaviour.
+
+# Setting this to '0' will disable auto-management of the firewall.
+sys::manage::firewall						=	1
+
+
+### Server related options
+# This is the "short list" of servers shown when provisioning a new server. To see the full list of options,
+# run '/usr/bin/osinfo-query os' on any machine in the Anvil!. 
+#sys::servers::os_short_list					=	debian10,fedora32,freebsd12.1,gentoo,macosx10.7,msdos6.22,openbsd6.7,opensuse15.2,rhel5.11,rhel6.10,rhel7.9,rhel8.3,sles12sp5,solaris11,ubuntu20.04,win10,win2k16,win2k19
 ]]></key>
 		
 		<!-- Table headers -->
@@ -1593,7 +1790,7 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
 		<key name="message_0010">The SSH session to: [#!variable!target!#] was closed because 'no_cache' was set and there was an open SSH connection.</key>
 		<key name="message_0011">Wrote the system UUID to the file: [#!variable!file!#] to enable the web based tools to read this system's UUID.</key>
 		<key name="message_0012">Wrote the journald config file: [#!variable!file!#] to disable rate limiting to ensure high log levels are not lost.</key>
-		<key name="message_0013">#!free!#</key>
+		<key name="message_0013">Updated the journald config file: [#!variable!file!#] to enable persistent storage of logs to disk. Will restart the journald daemon now.</key>
 		<key name="message_0014">One or more files on disk have changed. Exiting to reload.</key>
 		<key name="message_0015">The reconfigure of the network has begun.</key>
 		<key name="message_0016">The host name: [#!variable!host_name!#] has been set.</key>
@@ -1922,6 +2119,7 @@ Are you sure that you want to delete the server: [#!variable!server_name!#]? [Ty
 		<key name="message_0245">Now purging: [#!variable!host_name!#] (host UUID: [#!variable!host_uuid!#]:</key>
 		<key name="message_0246">Purging the Anvil!: [#!variable!anvil_name!#] (UUID: [#!variable!anvil_uuid!#]:</key>
 		<key name="message_0247"><![CDATA[Please specify which server you want to manage with '--server <name_or_uuid>'. Available servers on this Anvil! system;]]></key>
+		<key name="message_0248">Created the journald directory: [#!variable!directory!#].</key>
 		
 		<!-- Success messages shown to the user --> 
 		<key name="ok_0001">Saved the mail server information successfully!</key>
@@ -2544,6 +2742,10 @@ Read UUID: .... [#!variable!read_uuid!#]
 		<key name="warning_0114">[ Warning ] - We're doing a load shed to reduce thermal loading, and the estimated migration time to pull the servers to us from our peer is shorter than the reverse. As such, we'll pull the peer's servers to here.</key>
 		<key name="warning_0115">[ Warning ] - We're doing a load shed to conserve UPS power, and by all measures, the time to migrate off either node is equal. We're node 1, so we will pull the servers to us now.</key>
 		<key name="warning_0116">[ Warning ] - We're doing a load shed to reduce thermal loading, and by all measures, the time to migrate off either node is equal. We're node 1, so we will pull the servers to us now.</key>
+		<key name="warning_0117">[ Warning ] - The core Anvil! configuration file: [#!variable!file!#] was missing! It's been recreated using default values. It is possible that the database connection information will need to be restored manually.</key>
+		<key name="warning_0118">[ Warning ] - The 'admin' group was created as a system group with the group ID: [#!variable!gid!#].</key>
+		<key name="warning_0119">[ Warning ] - The 'admin' user was created with the user ID: [#!variable!uid!#].</key>
+		<key name="warning_0120">[ Warning ] - Timed out waiting for the database: [#!variable!uuid!#] to become available.</key>
 
 		<!-- The entries below here are not sequential, but use a key to find the entry. -->
 		<!-- Run 'striker-parse-os-list to find new entries. --> 

tools/anvil-configure-host

@@ -139,7 +139,7 @@ sub update_passwords
 		}
 		else
 		{
-			my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{'anvil-change-password'}." -y --password-file ".$temp_file });
+			my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{'anvil-change-password'}." -y --password-file ".$temp_file.$anvil->Log->switches });
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, list => { output => $output, return_code => $return_code }});
 			foreach my $line (split/\n/, $output)
 			{

tools/anvil-daemon

@@ -86,6 +86,9 @@ if (($< != 0) && ($> != 0))
 	$anvil->nice_exit({exit_code => 1});
 }
 
+# If, so some reason, anvil.conf is lost, create it.
+$anvil->System->_check_anvil_conf();
+
 # Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks
 # is to setup the database server.
 $anvil->Database->connect({check_if_configured => 1});
@@ -100,9 +103,9 @@ if (not $anvil->data->{sys}{database}{connections})
 	{
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0201"});
 		prep_database($anvil);
-		sleep 1;
 		
 		# Try connecting again
+		$anvil->refresh();
 		$anvil->Database->connect();
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0132"});
 		if (not $anvil->data->{sys}{database}{connections})
@@ -195,6 +198,10 @@ while(1)
 {
 	# Reload defaults, re-read the config and then connect to the database(s)
 	$anvil->refresh();
+	
+	# If, so some reason, anvil.conf is lost, create it.
+	$anvil->System->_check_anvil_conf();
+	
 	$anvil->Database->connect({check_if_configured => $check_if_database_is_configured});
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0132"});
 	
@@ -402,7 +409,7 @@ sub handle_periodic_tasks
 		}});
 		
 		# Even when this runs, it should finish in under ten seconds so we don't need to background it.
-		my ($parse_output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{'anvil-parse-fence-agents'}, source => $THIS_FILE, line => __LINE__});
+		my ($parse_output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{'anvil-parse-fence-agents'}.$anvil->Log->switches, source => $THIS_FILE, line => __LINE__});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { parse_output => $parse_output }});
 		
 		# Scan the local network.
@@ -459,7 +466,7 @@ sub handle_periodic_tasks
 			my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
 				file            => $THIS_FILE, 
 				line            => __LINE__, 
-				job_command     => $anvil->data->{path}{exe}{'striker-manage-install-target'}." --refresh", 
+				job_command     => $anvil->data->{path}{exe}{'striker-manage-install-target'}." --refresh".$anvil->Log->switches, 
 				job_data        => "", 
 				job_name        => "install-target::refresh", 
 				job_title       => "job_0015", 
@@ -472,7 +479,7 @@ sub handle_periodic_tasks
 			($job_uuid) = $anvil->Database->insert_or_update_jobs({
 				file            => $THIS_FILE, 
 				line            => __LINE__, 
-				job_command     => $anvil->data->{path}{exe}{'striker-parse-oui'}, 
+				job_command     => $anvil->data->{path}{exe}{'striker-parse-oui'}.$anvil->Log->switches, 
 				job_data        => "", 
 				job_name        => "oui-data::refresh", 
 				job_title       => "job_0064", 
@@ -485,7 +492,7 @@ sub handle_periodic_tasks
 			($job_uuid) = $anvil->Database->insert_or_update_jobs({
 				file            => $THIS_FILE, 
 				line            => __LINE__, 
-				job_command     => $anvil->data->{path}{exe}{'striker-scan-network'}, 
+				job_command     => $anvil->data->{path}{exe}{'striker-scan-network'}.$anvil->Log->switches, 
 				job_data        => "", 
 				job_name        => "scan-network::refresh", 
 				job_title       => "job_0066", 
@@ -522,7 +529,7 @@ sub check_install_target
 	}
 	
 	my $status                 = "unavailable";
-	my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{'striker-manage-install-target'}." --status --check --no-refresh"});
+	my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{'striker-manage-install-target'}." --status --check --no-refresh".$anvil->Log->switches});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output }});
 	foreach my $line (split/\n/, $output)
 	{
@@ -579,6 +586,9 @@ sub run_once
 	# Check setuid wrappers
 	check_setuid_wrappers($anvil);
 	
+	# Check journald is configured for persistent storage.
+	check_journald($anvil);
+	
 	if ($anvil->data->{switches}{'startup-only'})
 	{
 		$anvil->nice_exit({exit_code => 0});
@@ -587,6 +597,98 @@ sub run_once
 	return(0);
 }
 
+sub check_journald
+{
+	my ($anvil) = @_;
+	
+	# Check the journald.conf to ensure logging in configured to be persistent.
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'path::configs::journald.conf' => $anvil->data->{path}{configs}{'journald.conf'} }});
+	my $peristent_seen    = 0;
+	my $change_storage    = 0;
+	my $old_journald_conf = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'journald.conf'}});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { old_journald_conf => $old_journald_conf }});
+	foreach my $line (split/\n/, $old_journald_conf)
+	{
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
+		if ($line =~ /^Storage=(.*)$/)
+		{
+			my $value = $1;
+			if ($value eq "persistent")
+			{
+				$peristent_seen = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { peristent_seen => $peristent_seen }});
+			}
+			else
+			{
+				$change_storage = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { change_storage => $change_storage }});
+			}
+		}
+	}
+	
+	# Make sure the journald directory 
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'path::directories::journald' => $anvil->data->{path}{directories}{journald} }});
+	if (not -d $anvil->data->{path}{directories}{journald})
+	{
+		$anvil->Storage->make_directory({
+			debug     => 2,
+			directory => $anvil->data->{path}{directories}{journald},
+		});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0248", variables => { directory => $anvil->data->{path}{directories}{journald} }});
+	}
+	
+	# Make sure the journald is configured for persistent storage.
+	if (not $peristent_seen)
+	{
+		my $storage_added     = 0;
+		my $new_journald_conf = "";
+		foreach my $line (split/\n/, $old_journald_conf)
+		{
+			if (($line =~ /^Storage=/) && ($change_storage))
+			{
+				if (not $storage_added)
+				{
+					$storage_added     =  1;
+					$new_journald_conf .= "Storage=persistent\n";
+				}
+				next;
+			}
+			if (($line =~ /^#Storage=/) && (not $storage_added))
+			{
+				$storage_added     =  1;
+				$new_journald_conf .= "Storage=persistent\n";
+			}
+			$new_journald_conf .= $line."\n";
+		}
+		if (not $storage_added)
+		{
+			$new_journald_conf .= "Storage=persistent\n";
+		}
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { new_journald_conf => $new_journald_conf }});
+		
+		$anvil->Storage->write_file({
+			debug     => 3,
+			secure    => 0, 
+			file      => $anvil->data->{path}{configs}{'journald.conf'}, 
+			body      => $new_journald_conf, 
+			mode      => "0644",
+			overwrite => 1,
+		});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0013", variables => { file => $anvil->data->{path}{configs}{'journald.conf'} }});
+		
+		# Restart the journald service.
+		my $shell_call = $anvil->data->{path}{exe}{systemctl}." restart systemd-journald.service";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+		my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			output      => $output, 
+			return_code => $return_code,
+		}});
+	}
+	
+	return(0);
+}
+
 # This creates, as needed, the setuid wrappers used by apache to make certain system calls.
 sub check_setuid_wrappers
 {
@@ -692,7 +794,7 @@ sub check_firewall
 	# Check the firewall needs to be updated.
 	if ($configured)
 	{
-		my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{'anvil-manage-firewall'}});
+		my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{'anvil-manage-firewall'}.$anvil->Log->switches});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output, return_code => $return_code }});
 	}
 	
@@ -826,19 +928,14 @@ sub prep_database
 	
 	# Only run this if we're a dashboard.
 	my $host_type = $anvil->Get->host_type();
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
 	if ($host_type eq "striker")
 	{
-		my ($database_output, $return_code) = $anvil->System->call({
+		my ($database_output, $return_code) = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{'striker-prep-database'}.$anvil->Log->switches, source => $THIS_FILE, line => __LINE__ });
-			debug      => 3, 
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
-			shell_call => $anvil->data->{path}{exe}{'striker-prep-database'}, 
+			database_output => $database_output,
-			source     => $THIS_FILE, 
+			return_code     => $return_code, 
-			line       => __LINE__,
+		}});
-		});
-		if ($database_output)
-		{
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_output => $database_output }});
-		}
 	}
 	
 	return(0);
@@ -1149,7 +1246,7 @@ sub update_state_file
 	
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0480"});
 	
-	my ($states_output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{'anvil-update-states'}." -v", source => $THIS_FILE, line => __LINE__});
+	my ($states_output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{'anvil-update-states'}.$anvil->Log->switches, source => $THIS_FILE, line => __LINE__});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 		states_output => $states_output,
 		return_code   => $return_code, 

tools/anvil-delete-server

@@ -368,7 +368,7 @@ sub remove_from_pacemaker
 	{
 		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
 			debug           => 2, 
-			job_command     => $anvil->data->{path}{exe}{'anvil-delete-server'}, 
+			job_command     => $anvil->data->{path}{exe}{'anvil-delete-server'}.$anvil->Log->switches, 
 			job_data        => "server_uuid=".$server_uuid."\npeer_mode=true", 
 			job_name        => "server::delete", 
 			job_title       => "job_0208", 
@@ -655,7 +655,7 @@ sub ask_for_server
 		
 		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
 			debug           => 2, 
-			job_command     => $anvil->data->{path}{exe}{'anvil-delete-server'}, 
+			job_command     => $anvil->data->{path}{exe}{'anvil-delete-server'}.$anvil->Log->switches, 
 			job_data        => "server_uuid=".$delete_uuid, 
 			job_name        => "server::delete", 
 			job_title       => "job_0208", 

tools/anvil-manage-files

@@ -458,7 +458,7 @@ AND
 				my $remote_size                    = 0;
 				my $remote_md5sum                  = "";
 				my ($output, $error, $return_code) = $anvil->Remote->call({
-					shell_call  => $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path." --with-md5sum", 
+					shell_call  => $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path." --with-md5sum".$anvil->Log->switches, 
 					remote_user => $remote_user, 
 					password    => $password,
 					target      => $ip, 

tools/anvil-provision-server

@@ -259,7 +259,7 @@ sub run_jobs
 		   $job_data   .= "drbd_tcp_port=".$anvil->data->{job}{drbd_tcp_port}."\n";
 		my ($job_uuid) =  $anvil->Database->insert_or_update_jobs({
 			debug           => 2, 
-			job_command     => $anvil->data->{path}{exe}{'anvil-provision-server'}, 
+			job_command     => $anvil->data->{path}{exe}{'anvil-provision-server'}.$anvil->Log->switches, 
 			job_data        => $job_data, 
 			job_name        => "server::provision", 
 			job_title       => "job_0147", 
@@ -2515,7 +2515,7 @@ server_uuid=".$anvil->data->{new_server}{name};
 		}
 		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
 			debug           => 2, 
-			job_command     => $anvil->data->{path}{exe}{'anvil-provision-server'}, 
+			job_command     => $anvil->data->{path}{exe}{'anvil-provision-server'}.$anvil->Log->switches, 
 			job_data        => $job_data, 
 			job_name        => "server::provision", 
 			job_title       => "job_0147", 

tools/anvil-safe-start

@@ -189,7 +189,7 @@ sub boot_servers
 	
 	# Call 'anvil-boot-server --server all' to boot the servers now.
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0614"});
-	my $shell_call = $anvil->data->{path}{exe}{'anvil-boot-server'}." --server all";
+	my $shell_call = $anvil->data->{path}{exe}{'anvil-boot-server'}." --server all".$anvil->Log->switches;
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 	my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 

tools/anvil-sync-shared

@@ -706,7 +706,7 @@ sub process_incoming_file
 			my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
 				file            => $THIS_FILE, 
 				line            => __LINE__, 
-				job_command     => $anvil->data->{path}{exe}{'anvil-sync-shared'}, 
+				job_command     => $anvil->data->{path}{exe}{'anvil-sync-shared'}.$anvil->Log->switches, 
 				job_data        => "file_uuid=".$file_uuid, 
 				job_name        => "storage::pull_file", 
 				job_title       => "job_0132", 

tools/scancore

@@ -355,7 +355,7 @@ sub startup_tasks
 		if ($uptime < 600)
 		{
 			# Run it as a background task
-			my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-start'};
+			my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-start'}.$anvil->Log->switches;
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0210", variables => { command => $shell_call }});
 			$anvil->System->call({shell_call => $shell_call, background => 1});
 		}

tools/striker-auto-initialize-all

@@ -327,7 +327,7 @@ sub configure_machine_networks
 				
 				my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
 					job_host_uuid   => $machine_host_uuid, 
-					job_command     => $anvil->data->{path}{exe}{'anvil-configure-host'}, 
+					job_command     => $anvil->data->{path}{exe}{'anvil-configure-host'}.$anvil->Log->switches, 
 					job_data        => "form::config_step2", 
 					job_name        => "configure::network", 
 					job_title       => "job_0001", 
@@ -462,7 +462,7 @@ sub run_manifests
 		# Tell node 1 to join.
 		my ($node1_job_uuid) = $anvil->Database->insert_or_update_jobs({
 			job_host_uuid   => $node1_host_uuid, 
-			job_command     => $anvil->data->{path}{exe}{'anvil-join-anvil'}, 
+			job_command     => $anvil->data->{path}{exe}{'anvil-join-anvil'}.$anvil->Log->switches, 
 			job_data        => "as_machine=node1,manifest_uuid=".$manifest_uuid.",anvil_uuid=".$anvil_uuid, 
 			job_name        => "join_anvil::node1", 
 			job_title       => "job_0072", 
@@ -480,7 +480,7 @@ sub run_manifests
 		
 		my ($node2_job_uuid) = $anvil->Database->insert_or_update_jobs({
 			job_host_uuid   => $node2_host_uuid, 
-			job_command     => $anvil->data->{path}{exe}{'anvil-join-anvil'}, 
+			job_command     => $anvil->data->{path}{exe}{'anvil-join-anvil'}.$anvil->Log->switches, 
 			job_data        => "as_machine=node2,manifest_uuid=".$manifest_uuid.",anvil_uuid=".$anvil_uuid, 
 			job_name        => "join_anvil::node2", 
 			job_title       => "job_0072", 
@@ -500,7 +500,7 @@ sub run_manifests
 		{
 			my ($dr1_job_uuid) = $anvil->Database->insert_or_update_jobs({
 				job_host_uuid   => $dr1_host_uuid, 
-				job_command     => $anvil->data->{path}{exe}{'anvil-join-anvil'}, 
+				job_command     => $anvil->data->{path}{exe}{'anvil-join-anvil'}.$anvil->Log->switches, 
 				job_data        => "as_machine=dr1,manifest_uuid=".$manifest_uuid.",anvil_uuid=".$anvil_uuid, 
 				job_name        => "join_anvil::dr1", 
 				job_title       => "job_0072", 
@@ -717,7 +717,7 @@ sub initialize_machines
 						debug           => 2,
 						file            => $THIS_FILE, 
 						line            => __LINE__, 
-						job_command     => $anvil->data->{path}{exe}{'striker-initialize-host'}, 
+						job_command     => $anvil->data->{path}{exe}{'striker-initialize-host'}.$anvil->Log->switches, 
 						job_data        => $job_data, 
 						job_name        => "initialize::".$machine_type."::".$use_ip, 
 						job_title       => $machine_type eq "dr" ? "job_0021" : "job_0020", 
@@ -1340,8 +1340,8 @@ fi;
 						
 						# Register a job and then wait for it to show up in our database.
 						my $job_data    =  "password=".$anvil->data->{base}{password}{desired}."\n";
-						   $job_data    .= "peer_job_command=".$anvil->data->{path}{exe}{'striker-manage-peers'}." --add --host-uuid ".$anvil->Get->host_uuid." --host ".$our_ip." --port 5432 --ping 1";
+						   $job_data    .= "peer_job_command=".$anvil->data->{path}{exe}{'striker-manage-peers'}." --add --host-uuid ".$anvil->Get->host_uuid." --host ".$our_ip." --port 5432 --ping 1".$anvil->Log->switches;
-						my $job_command =  $anvil->data->{path}{exe}{'striker-manage-peers'}." --add --host-uuid ".$host_uuid." --host ".$ip." --port 5432 --ping 1";
+						my $job_command =  $anvil->data->{path}{exe}{'striker-manage-peers'}." --add --host-uuid ".$host_uuid." --host ".$ip." --port 5432 --ping 1".$anvil->Log->switches;
 						   $waiting     =  0;
 						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 							job_command => $job_command,
@@ -1665,7 +1665,7 @@ sub striker_stage1
 	my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
 		debug           => 2,
 		job_host_uuid   => $anvil->data->{cgi}{host_uuid}{value}, 
-		job_command     => $anvil->data->{path}{exe}{'anvil-configure-host'}, 
+		job_command     => $anvil->data->{path}{exe}{'anvil-configure-host'}.$anvil->Log->switches, 
 		job_data        => "form::config_step2", 
 		job_name        => "configure::network", 
 		job_title       => "job_0001", 
@@ -1680,7 +1680,7 @@ sub striker_stage1
 		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
 			debug           => 2,
 			job_host_uuid   => $anvil->data->{cgi}{host_uuid}{value}, 
-			job_command     => $anvil->data->{path}{exe}{'striker-auto-initialize-all'}." --config ".$anvil->data->{switches}{config}, 
+			job_command     => $anvil->data->{path}{exe}{'striker-auto-initialize-all'}." --config ".$anvil->data->{switches}{config}.$anvil->Log->switches, 
 			job_data        => "", 
 			job_name        => "configure::auto_initialize", 
 			job_title       => "job_0225", 

tools/striker-manage-peers

@@ -525,15 +525,71 @@ sub process_entry
 		update_progress($anvil, 80, "message_0073");
 		update_progress($anvil, 82, "job_0131");
 		
+		# Reconnect. This can take a few tries as we ma need to initialize the DB as well.
+		my $waiting    = 1;
+		my $restarted  = 0;
+		my $abort_time = time + 5;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { abort_time => $abort_time }});
+		while($waiting)
+		{
+			$waiting = 0;
+			
 			# Disconnect (if no connections exist, will still clear out known databases).
-		$anvil->Database->disconnect({debug => 2});
+			$anvil->Database->disconnect();
 			
 			# Re-read the config.
-		sleep 1;
+			$anvil->refresh();
-		$anvil->Storage->read_config({debug => 3});
+			
+			# Connect, and configure, if needed.
+			$anvil->Database->connect({
+				debug               => 3,
+				check_if_configured => $host_uuid eq $anvil->Get->host_uuid ? 1 : 0,
+			});
+			
+			my $target_handle = $anvil->data->{cache}{database_handle}{$host_uuid} ? $anvil->data->{cache}{database_handle}{$host_uuid} : "";
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_handle => $target_handle }});
 			
-		# Reconnect
+			if ((not $target_handle) or ($target_handle !~ /^DBI::db=HASH/))
-		$anvil->Database->connect({debug => 3});
+			{
+				# not connected yet.
+				my $time = time;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					'time'     => $time, 
+					abort_time => $abort_time,
+				}});
+				if ($time < $abort_time)
+				{
+					# sleep and try again.
+					$anvil->Database->disconnect();
+					
+					# Try restarting the database if the $host_uuid is us.
+					if (($host_uuid eq $anvil->Get->host_uuid) && (not $restarted))
+					{
+						my $stop_return_code  = $anvil->System->stop_daemon({debug => 2, daemon => "postgresql.service"});
+						my $start_return_code = $anvil->System->start_daemon({debug => 2, daemon => "postgresql.service"});
+						   $restarted   = 1;
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+							's1:restarted'         => $restarted, 
+							's2:stop_return_code'  => $stop_return_code,
+							's3:start_return_code' => $start_return_code, 
+						}});
+					}
+					
+					$waiting = 1;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
+					sleep 1;
+				}
+				else
+				{
+					# Timed out.
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0120", variables => { uuid => $host_uuid }});
+				}
+			}
+			else
+			{
+				# Connected!
+			}
+		}
 	}
 	
 	# If I've been asked to have the peer add us, disconnect from the database, re-read the new config 

tools/striker-prep-database

@@ -31,8 +31,10 @@ my $anvil = Anvil::Tools->new();
 $anvil->Get->switches;
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
 
+$anvil->System->_check_anvil_conf({debug => 2});
+
 my $local_uuid = $anvil->Database->get_local_uuid();
-$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { local_uuid => $local_uuid }});
+$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { local_uuid => $local_uuid }});
 
 # If we didn't get the $local_uuid, then there is no entry for this system in anvil.conf yet, so we'll add it.
 if (not $local_uuid)
@@ -45,10 +47,10 @@ if (not $local_uuid)
 if ($local_uuid)
 {
 	# Start checks
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "sys::service::postgresql" => $anvil->data->{sys}{daemon}{postgresql} }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::service::postgresql" => $anvil->data->{sys}{daemon}{postgresql} }});
 	
 	my $running = $anvil->System->check_daemon({debug => 3, daemon => $anvil->data->{sys}{daemon}{postgresql}});
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { running => $running }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { running => $running }});
 	if ($running eq "2")
 	{
 		# Not installed.
@@ -67,8 +69,11 @@ if ($local_uuid)
 		else
 		{
 			# Initialize.
-			my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{'postgresql-setup'}." initdb", source => $THIS_FILE, line => __LINE__});
+			my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{'postgresql-setup'}." initdb", source => $THIS_FILE, line => __LINE__});
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				output      => $output, 
+				return_code => $return_code,
+			}});
 			
 			# Did it succeed?
 			if (not -e $anvil->data->{path}{configs}{'pg_hba.conf'})
@@ -127,7 +132,7 @@ if ($local_uuid)
 			if ($update_file)
 			{
 				$anvil->Storage->write_file({
-					debug     => 2,
+					debug     => 3,
 					file      => $anvil->data->{path}{configs}{'postgresql.conf'}, 
 					body      => $new_postgresql_conf,
 					user      => "postgres", 
@@ -174,7 +179,7 @@ if ($local_uuid)
 			if ($update_file)
 			{
 				$anvil->Storage->write_file({
-					debug     => 2,
+					debug     => 3,
 					file      => $anvil->data->{path}{configs}{'pg_hba.conf'}, 
 					body      => $new_pg_hba_conf,
 					user      => "postgres", 
@@ -187,7 +192,7 @@ if ($local_uuid)
 		}
 		
 		# Start the daemon. '0' = started, anything else is a problem.
-		my $return_code = $anvil->System->start_daemon({debug => 3, daemon => $anvil->data->{sys}{daemon}{postgresql}});
+		my $return_code = $anvil->System->start_daemon({debug => 2, daemon => $anvil->data->{sys}{daemon}{postgresql}});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
 		if ($return_code eq "0")
 		{
@@ -195,7 +200,7 @@ if ($local_uuid)
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0059"});
 			
 			# Make sure it is enabled on boot.
-			my $return_code = $anvil->System->enable_daemon({debug => 3, daemon => $anvil->data->{sys}{daemon}{postgresql}});
+			my $return_code = $anvil->System->enable_daemon({debug => 2, daemon => $anvil->data->{sys}{daemon}{postgresql}});
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
 		}
 		else
@@ -208,15 +213,16 @@ if ($local_uuid)
 	
 	# Create the .pgpass file, if needed.
 	my $created_pgpass = 0;
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { 
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { 
 		'path::secure::postgres_pgpass'     => $anvil->data->{path}{secure}{postgres_pgpass},
 		"database::${local_uuid}::password" => $anvil->data->{database}{$local_uuid}{password}, 
 	}});
 	if ((not -e $anvil->data->{path}{secure}{postgres_pgpass}) && ($anvil->data->{database}{$local_uuid}{password}))
 	{
 		my $body = "*:*:*:postgres:".$anvil->data->{database}{$local_uuid}{password}."\n";
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { body => $body }});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { body => $body }});
 		$anvil->Storage->write_file({
+			debug     => 3,
 			file      => $anvil->data->{path}{secure}{postgres_pgpass},  
 			body      => $body,
 			user      => "postgres", 
@@ -228,41 +234,52 @@ if ($local_uuid)
 		if (-e $anvil->data->{path}{secure}{postgres_pgpass})
 		{
 			$created_pgpass = 1;
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { created_pgpass => $created_pgpass }});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { created_pgpass => $created_pgpass }});
 		}
 	}
 	
 	# Does the database user exist?
 	my $create_user   = 1;
 	my $database_user = $anvil->data->{database}{$local_uuid}{user} ? $anvil->data->{database}{$local_uuid}{user} : $anvil->data->{sys}{database}{user};
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_user => $database_user }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { database_user => $database_user }});
 	if (not $database_user)
 	{
 		# No database user defined
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0099", variables => { uuid => $local_uuid }});
 		$anvil->nice_exit({exit_code => 3});
 	}
-	my ($user_list, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT usename, usesysid FROM pg_catalog.pg_user;'\"", source => $THIS_FILE, line => __LINE__});
+	my ($user_list, $return_code) = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT usename, usesysid FROM pg_catalog.pg_user;'\"", source => $THIS_FILE, line => __LINE__});
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { user_list => $user_list, return_code => $return_code }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		user_list   => $user_list, 
+		return_code => $return_code,
+	}});
 	foreach my $line (split/\n/, $user_list)
 	{
 		if ($line =~ /^ $database_user\s+\|\s+(\d+)/)
 		{
 			# User exists already
 			my $id = $1;
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0060", variables => { user => $database_user, id => $id }});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0060", variables => { user => $database_user, id => $id }});
 			$create_user = 0;
 			last;
 		}
 	}
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { create_user => $create_user }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_user => $create_user }});
 	if ($create_user)
 	{
 		# Create the user
-		my ($create_output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{createuser}." --no-superuser --createdb --no-createrole $database_user\"", source => $THIS_FILE, line => __LINE__});
+		my ($create_output, $return_code) = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{createuser}." --no-superuser --createdb --no-createrole $database_user\"", source => $THIS_FILE, line => __LINE__});
-		(my $user_list, $return_code)     = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT usename, usesysid FROM pg_catalog.pg_user;'\"", source => $THIS_FILE, line => __LINE__});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			create_output => $create_output, 
+			user_list     => $user_list,
+		}});
+		
+		(my $user_list, $return_code) = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT usename, usesysid FROM pg_catalog.pg_user;'\"", source => $THIS_FILE, line => __LINE__});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			create_output => $create_output, 
+			user_list     => $user_list,
+		}});
 		my $user_exists   = 0;
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_output => $create_output, user_list => $user_list }});
 		foreach my $line (split/\n/, $user_list)
 		{
 			if ($line =~ /^ $database_user\s+\|\s+(\d+)/)
@@ -287,9 +304,13 @@ if ($local_uuid)
 			foreach my $user ("postgres", $database_user)
 			{
 				my ($update_output, $return_code) = $anvil->System->call({secure => 1, shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c \\\"ALTER ROLE $user WITH PASSWORD '".$anvil->data->{database}{$local_uuid}{password}."';\\\"\"", source => $THIS_FILE, line => __LINE__});
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { update_output => $update_output, return_code => $return_code }});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { 
+					update_output => $update_output, 
+					return_code   => $return_code,
+				}});
 				foreach my $line (split/\n/, $user_list)
 				{
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
 					if ($line =~ /ALTER ROLE/)
 					{
 						# Password set
@@ -303,13 +324,17 @@ if ($local_uuid)
 	# Create the database, if needed.
 	my $create_database = 1;
 	my $database_name   = $anvil->data->{database}{$local_uuid}{name} ? $anvil->data->{database}{$local_uuid}{name} : $anvil->data->{sys}{database}{name};
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_name => $database_name }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { database_name => $database_name }});
 	
 	undef $return_code;
 	(my $database_list, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT datname FROM pg_catalog.pg_database;'\"", source => $THIS_FILE, line => __LINE__});
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_list => $database_list, return_code => $return_code }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		database_list => $database_list, 
+		return_code   => $return_code,
+	}});
 	foreach my $line (split/\n/, $database_list)
 	{
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
 		if ($line =~ /^ $database_name$/)
 		{
 			# Database already exists.
@@ -322,14 +347,21 @@ if ($local_uuid)
 	if ($create_database)
 	{
 		my ($create_output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{createdb}."  --owner ".$database_user." ".$database_name."\"", source => $THIS_FILE, line => __LINE__});
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_output => $create_output, return_code => $return_code }});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			create_output => $create_output, 
+			return_code   => $return_code,
+		}});
 		
 		undef $return_code;
 		my $database_exists               = 0;
 		(my $database_list, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT datname FROM pg_catalog.pg_database;'\"", source => $THIS_FILE, line => __LINE__});
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { database_list => $database_list, return_code => $return_code }});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			database_list => $database_list, 
+			return_code   => $return_code,
+		}});
 		foreach my $line (split/\n/, $database_list)
 		{
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
 			if ($line =~ /^ $database_name$/)
 			{
 				# Database created
@@ -365,6 +397,7 @@ if ($local_uuid)
 	if (not -e $anvil->data->{path}{data}{host_uuid})
 	{
 		$anvil->Storage->write_file({
+			debug     => 3,
 			file      => $anvil->data->{path}{data}{host_uuid}, 
 			body      => $anvil->Get->host_uuid,
 			user      => "apache", 
@@ -389,7 +422,7 @@ RateLimitInterval=0
 RateLimitBurst=0
 ";
 		$anvil->Storage->write_file({
-			debug     => 2,
+			debug     => 3,
 			file      => $anvil->data->{path}{configs}{'journald_anvil'}, 
 			body      => $body,
 			user      => "root", 
@@ -400,7 +433,10 @@ RateLimitBurst=0
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0012", variables => { file => $anvil->data->{path}{configs}{'journald_anvil'} }});
 		
 		my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{systemctl}." restart systemd-journald.service", source => $THIS_FILE, line => __LINE__});
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			output      => $output, 
+			return_code => $return_code,
+		}});
 	}
 }
 else
@@ -424,6 +460,7 @@ sub add_to_local_config
 	# Write the password to a file.
 	my $password_file = "/tmp/striker-manage-peers.".$anvil->Get->uuid;
 	$anvil->Storage->write_file({
+		debug     => 3,
 		secure    => 1, 
 		file      => $password_file, 
 		body      => "Initial1", 
@@ -434,17 +471,20 @@ sub add_to_local_config
 	# Make the shell call, and parse the output looking for our own entry
 	my $host_uuid              = $anvil->Get->host_uuid();
 	my ($output, $return_code) = $anvil->System->call({
-		shell_call => $anvil->data->{path}{exe}{'striker-manage-peers'}." --add --host-uuid ".$anvil->Get->host_uuid." --host localhost --port 5432 --password-file ".$password_file." --ping 0", 
+		debug      => 2,
+		shell_call => $anvil->data->{path}{exe}{'striker-manage-peers'}." --add --host-uuid ".$anvil->Get->host_uuid." --host localhost --port 5432 --password-file ".$password_file." --ping 0".$anvil->Log->switches, 
 		source     => $THIS_FILE, 
 		line       => __LINE__,
 	});
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		output      => $output, 
+		return_code => $return_code,
+	}});
 	
 	# Remove the password.
 	unlink $password_file;
 	
 	# Re-read the config and make sure we have our own entry.
-	sleep 1;
 	$anvil->Storage->read_config({file => $anvil->data->{path}{configs}{'anvil.conf'}});
 	
 	# If we still don't have a local_uuid, something went wrong.