anvil/tools/striker-manage-peers

#!/usr/bin/perl
# 
# This program adds, edits and removes striker peers (for replicating Anvil! database data).
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Program not run as root.
# 2 = A switch is missing or invalid.
# 3 = 
# 
# Calling this with --add, will insert an entry if it's not found. Calling it with no switch will update the 
# entry if it exists. Calling it with --remove will delete it.
# 
### Show existing entries
# /usr/sbin/striker-manage-peers --list
# 
### Add a new entry, or edit an existing one
# /usr/sbin/striker-manage-peers --add --host-uuid e20c3f10-c35d-4543-b5e6-8a373f27977a --host localhost --port 5432 --password-file /tmp/striker-manage-peers.2e410b43-42a0-4eaf-985c-670f92c482b8 --ping 0
#
### Edit an existing entry, but don't add it if it wasn't found.
# /usr/sbin/striker-manage-peers --host-uuid e20c3f10-c35d-4543-b5e6-8a373f27977a --host localhost --port 5432 --password-file /tmp/striker-manage-peers.2e410b43-42a0-4eaf-985c-670f92c482b8 --ping 0
# 
### Remove an entry
# /usr/sbin/striker-manage-peers --remove --host-uuid e20c3f10-c35d-4543-b5e6-8a373f27977a 
# 

use strict;
use warnings;
use Anvil::Tools;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

# Read switches
$anvil->Get->switches({list => ["add", "list", "remove", "host", "host-uuid", "job-uuid", "port", "password-file", "ping"], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
if ($anvil->data->{switches}{'port'} eq "")
{
	$anvil->data->{switches}{'port'} = 5432;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => 1, level => 3, list => { "switches::port" => $anvil->data->{switches}{'port'} }});
}

# Make sure we're running as 'root'
# $< == real UID, $> == effective UID
if (($< != 0) && ($> != 0))
{
	# Not root
	print $anvil->Words->string({key => "error_0005"})."\n";
	$anvil->nice_exit({exit_code => 1});
}

# We'll try to connect in case we're adding additional peers.
$anvil->data->{sys}{database}{resync_needed} = 1;
$anvil->Database->connect({
	debug            => 2,
	check_for_resync => 1,
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});

# Am I adding, editing or deleting?
if ((not $anvil->data->{switches}{list}) && ($anvil->data->{switches}{'host-uuid'}))
{
	process_entry($anvil);
}

### Report the peers.
# First sort by host name/ip
foreach my $uuid (keys %{$anvil->data->{database}})
{
	next if not $uuid;
	my $host = $anvil->data->{database}{$uuid}{host};
	$anvil->data->{sorted}{db}{$host} = $uuid;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => 0, level => 2, list => { 
		host                  => $host,
		"sorted::db::${host}" => $anvil->data->{sorted}{db}{$host},
	}});
}

foreach my $host (sort {$a cmp $b} keys %{$anvil->data->{sorted}{db}})
{
	my $uuid     = $anvil->data->{sorted}{db}{$host};
	my $port     = $anvil->data->{database}{$uuid}{port}     ? $anvil->data->{database}{$uuid}{port}     : 5432;
	my $name     = $anvil->data->{database}{$uuid}{name}     ? $anvil->data->{database}{$uuid}{name}     : "anvil";
	my $user     = $anvil->data->{database}{$uuid}{user}     ? $anvil->data->{database}{$uuid}{user}     : "admin";
	my $password = $anvil->data->{database}{$uuid}{password} ? $anvil->data->{database}{$uuid}{password} : "";
	print $anvil->Words->string({key => "message_0032", variables => { 
		peer => $user."\@".$host.":".$port,
		name => $name, 
		uuid => $uuid,
	}})."\n";
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0190", variables => { 
		peer     => $user."\@".$host.":".$port,
		name     => $name, 
		password => $anvil->Log->is_secure($password),
		uuid     => $uuid,
	}});
}

# We're done!
update_progress($anvil, 100, "message_0025");
$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This updates the progress if we were called with a job UUID.
sub update_progress
{
	my ($anvil, $progress, $message) = @_;
	$message = "" if not defined $message;
	
	# Log the progress percentage.
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		progress                     => $progress,
		message                      => $message, 
		"jobs::job_uuid"             => $anvil->data->{jobs}{job_uuid}, 
		"sys::database::connections" => $anvil->data->{sys}{database}{connections},
	}});
	
	if (($anvil->data->{jobs}{job_uuid}) && ($anvil->data->{sys}{database}{connections}))
	{
		$anvil->Job->update_progress({
			debug    => 3, 
			progress => $progress, 
			message  => $message, 
			job_uuid => $anvil->data->{jobs}{job_uuid},
		});
	}
	
	return(0);
}

sub process_entry
{
	my ($anvil) = @_;
	
	my $job_uuid      = $anvil->data->{switches}{'job-uuid'};
	my $host_uuid     = $anvil->data->{switches}{'host-uuid'};
	my $host          = $anvil->data->{switches}{'host'};
	my $port          = $anvil->data->{switches}{'port'};
	my $password_file = $anvil->data->{switches}{'password-file'};
	my $ping          = $anvil->data->{switches}{'ping'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => 0, level => 2, list => { 
		job_uuid      => $job_uuid, 
		host_uuid     => $host_uuid,
		host          => $host,
		port          => $port,
		password_file => $password_file,
		ping          => $ping, 
	}});
	
	# If I have a job_uuid, store it in the data hash for 'update_progress'.
	if ($job_uuid)
	{
		$anvil->data->{jobs}{job_uuid} = $job_uuid;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => 0, level => 2, list => { "jobs::job_uuid" => $anvil->data->{jobs}{job_uuid} }});
		
		my $message = $anvil->data->{switches}{remove} ? "message_0067" : "message_0066";
		update_progress($anvil, 0, "clear");
		update_progress($anvil, 1, $message);
	}
	else
	{
		$anvil->data->{jobs}{job_uuid} = "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => 0, level => 2, list => { "jobs::job_uuid" => $anvil->data->{jobs}{job_uuid} }});
	}

	# Read in the anvil.conf, we're going to need it in any case.
	$anvil->data->{body}{'anvil.conf'} = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'anvil.conf'}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => 1, level => 3, list => { "body::anvil.conf" => $anvil->data->{body}{'anvil.conf'} }});

	# If we don't find the entry, or if the entry exists but has changed, this will be set to '1' so we'll 
	# rewrite the file.
	$anvil->data->{config}{rewrite} = 0;
	
	# Is anything missing?
	if ((not $host_uuid) or (not $anvil->Validate->uuid({uuid => $host_uuid})))
	{
		# Invalid UUID.
		print $anvil->Words->string({key => "error_0031", variables => { host_uuid => $host_uuid }})."\n";
		update_progress($anvil, 100, "error_0031,!!host_uuid!".$host_uuid."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "error_0031", variables => { host_uuid => $host_uuid }});
		$anvil->nice_exit({exit_code => 2});
	}
	if ((not $host) && (not $anvil->data->{switches}{remove}))
	{
		# Invalid Host name.
		print $anvil->Words->string({key => "error_0032", variables => { switch => "host" }})."\n";
		update_progress($anvil, 100, "error_0032,!!switch!host!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "error_0032", variables => { switch => "host" }});
		$anvil->nice_exit({exit_code => 2});
	}
	if (($port =~ /\D/) or ($port < 1) or ($port > 65535))
	{
		# Invalid port.
		print $anvil->Words->string({key => "error_0038", variables => { port => $port }})."\n";
		update_progress($anvil, 100, "error_0038,!!port!".$port."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "error_0038", variables => { port => $port }});
		$anvil->nice_exit({exit_code => 2});
	}
	
	# The password might be in 'job_data' or a password file.
	my $password         = "";
	my $add_peer_command = "";
	if ($password_file)
	{
		# Pull the password out of the file.
		update_progress($anvil, 100, "message_0070,!!port!".$port."!!");
		$password = $anvil->Storage->read_file({file => $password_file});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => 1, level => 2, list => { password => $password }});
	}
	if ((not $password) && ($job_uuid))
	{
		# Do we have a database connection?
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::database::connections" => $anvil->data->{sys}{database}{connections}}});
		if ($anvil->data->{sys}{database}{connections})
		{
			# See if the password is in job_data.
			my $query = "SELECT job_data FROM jobs WHERE job_uuid = ".$anvil->Database->quote($job_uuid).";";
			my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
			my $count   = @{$results};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				results => $results, 
				count   => $count, 
			}});
			if ($count)
			{
				my $job_data = $results->[0]->[0];
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { job_data => $job_data }});
				
				foreach my $line (split/\n/, $job_data)
				{
					my $secure = $line =~ /password/ ? 1 : 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => $secure, list => { line => $line }});
					if ($line =~ /password=(.*)$/)
					{
						$password = $1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { password => $password }});
					}
					if ($line =~ /job_command=(.*)$/)
					{
						$add_peer_command = $1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, list => { add_peer_command => $add_peer_command }});
					}
				}
			}
		}
	}
	
	# If I am not removing, I need a password.
	if ((not $anvil->data->{switches}{remove}) && (not $password))
	{
		print $anvil->Words->string({key => "error_0039"})."\n";
		update_progress($anvil, 100, "error_0039");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "error_0039"});
		$anvil->nice_exit({exit_code => 2});
	}
	update_progress($anvil, 25, "message_0068");
	
	# If the config already exists, we'll look at each of the values to see if any changed (or are not defaults). If so, we'll rewrite
	my $host_variable      = "database::${host_uuid}::host";
	my $host_different     = 1;
	my $port_variable      = "database::${host_uuid}::port";
	my $port_different     = 1;
	my $password_variable  = "database::${host_uuid}::password";
	my $password_different = 1;
	my $ping_variable      = "database::${host_uuid}::ping";
	my $ping_different     = 1;
	my $peer_seen          = 0;
	
	# If we don't see this peer, this will be inserted.
	my $insert =  $host_variable."		=	".$host."\n";
	   $insert .= $port_variable."		=	".$port."\n";
	   $insert .= $password_variable."	=	".$password."\n";
	   $insert .= $ping_variable."		=	".$ping."\n";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => 1, level => 2, list => { insert => $insert }});
	
	# Loop through the existing file.
	my $delete_reported = 0;
	my $update_reported = 0;
	my $new_body        = "";
	my $just_deleted    = 0;
	my $test_line       = "database::${host_uuid}::";
	foreach my $line (split/\n/, $anvil->data->{body}{'anvil.conf'})
	{
		# If I removed an entry, I also want to delete the white space after it.
		if (($just_deleted) && ((not $line) or ($line =~ /^\s+$/)))
		{
			$just_deleted = 0;
			next;
		}
		$just_deleted = 0;
		
		# Secure password lines.
		my $secure = (($line =~ /password/) && ($line !~ /^#/)) ? 1 : 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => $secure, level => 3, list => { line => $line }});
		
		# If we've hit the end of the DB list, see if we need to insert a new entry.
		if ($line eq "### end db list ###")
		{
			# If I've not seen this DB, enter it.
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => 0, level => 2, list => { 
				peer_seen       => $peer_seen,
				"switches::add" => $anvil->data->{switches}{add},
			}});
			if ((not $peer_seen) && ($anvil->data->{switches}{add}))
			{
				$new_body                       .= $insert."\n";
				$anvil->data->{config}{rewrite} =  1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => 1, level => 2, list => { 
					new_body          => $new_body,
					"config::rewrite" => $anvil->data->{config}{rewrite},
				}});
				update_progress($anvil, 50, "message_0069");
			}
		}
		
		# Skip comments.
		if ($line =~ /^#/)
		{
			$new_body .= $line."\n";
			next;
		}
		if ($line =~ /^(.*?)(\s*)=(\s*)(.*)$/)
		{
			my $variable    = $1;
			my $left_space  = $2;
			my $right_space = $3; 
			my $value       = $4;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				"s1:variable"    => $variable,
				"s2:value"       => $value, 
				"s3:left_space"  => $left_space, 
				"s4:right_space" => $right_space, 
			}});
			
			if ($variable eq $host_variable)
			{
				$peer_seen = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:value"     => $value,
					"s2:host"      => $host, 
					"s3:peer_seen" => $peer_seen, 
				}});
				if ($anvil->data->{switches}{remove})
				{
					$just_deleted                   = 1;
					$anvil->data->{config}{rewrite} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						just_deleted      => $just_deleted, 
						"config::rewrite" => $anvil->data->{config}{rewrite},
					}});
					update_progress($anvil, 50, "message_0070") if not $delete_reported;
					$delete_reported = 1;
					next;
				}
				elsif ($value eq $host)
				{
					# No change.
					$host_different = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_different => $host_different }});
				}
				else
				{
					$line                           = $variable.$left_space."=".$right_space.$host;
					$anvil->data->{config}{rewrite} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						line              => $line,
						"config::rewrite" => $anvil->data->{config}{rewrite}, 
					}});
				}
			}
			elsif ($variable eq $port_variable)
			{
				$peer_seen = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:value"     => $value,
					"s2:port"      => $port, 
					"s3:peer_seen" => $peer_seen, 
				}});
				if ($anvil->data->{switches}{remove})
				{
					$just_deleted                   = 1;
					$anvil->data->{config}{rewrite} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						just_deleted      => $just_deleted, 
						"config::rewrite" => $anvil->data->{config}{rewrite},
					}});
					update_progress($anvil, 50, "message_0070") if not $delete_reported;
					$delete_reported = 1;
					next;
				}
				elsif ($value eq $port)
				{
					# No change.
					$port_different = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { port_different => $port_different }});
				}
				else
				{
					$line                           = $variable.$left_space."=".$right_space.$port;
					$anvil->data->{config}{rewrite} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						line              => $line,
						"config::rewrite" => $anvil->data->{config}{rewrite}, 
					}});
					update_progress($anvil, 50, "message_0071") if not $update_reported;
					$update_reported = 1;
				}
			}
			elsif ($variable eq $password_variable)
			{
				$peer_seen = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { 
					"s1:value"     => $value,
					"s2:password"  => $password, 
					"s3:peer_seen" => $peer_seen, 
				}});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { 
					value    => $value,
					password => $password, 
				}});
				if ($anvil->data->{switches}{remove})
				{
					$just_deleted                   = 1;
					$anvil->data->{config}{rewrite} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						just_deleted      => $just_deleted, 
						"config::rewrite" => $anvil->data->{config}{rewrite},
					}});
					update_progress($anvil, 50, "message_0070") if not $delete_reported;
					$delete_reported = 1;
					next;
				}
				elsif ($value eq $password)
				{
					# No change.
					$password_different = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { password_different => $password_different }});
				}
				else
				{
					$line                           = $variable.$left_space."=".$right_space.$password;
					$anvil->data->{config}{rewrite} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						line              => $anvil->Log->is_secure($line),
						"config::rewrite" => $anvil->data->{config}{rewrite}, 
					}});
					update_progress($anvil, 50, "message_0071") if not $update_reported;
					$update_reported = 1;
				}
			}
			elsif ($variable eq $ping_variable)
			{
				$peer_seen = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:value"     => $value,
					"s2:ping"      => $ping, 
					"s3:peer_seen" => $peer_seen, 
				}});
				if ($anvil->data->{switches}{remove})
				{
					$just_deleted                   = 1;
					$anvil->data->{config}{rewrite} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						just_deleted      => $just_deleted, 
						"config::rewrite" => $anvil->data->{config}{rewrite},
					}});
					update_progress($anvil, 50, "message_0070") if not $delete_reported;
					$delete_reported = 1;
					next;
				}
				elsif ($value eq $ping)
				{
					# No change.
					$ping_different = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ping_different => $ping_different }});
				}
				else
				{
					$line                           = $variable.$left_space."=".$right_space.$ping;
					$anvil->data->{config}{rewrite} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						line              => $line,
						"config::rewrite" => $anvil->data->{config}{rewrite}, 
					}});
					update_progress($anvil, 50, "message_0071") if not $update_reported;
					$update_reported = 1;
				}
			}
		}
		$new_body .= $line."\n";
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "config::rewrite" => $anvil->data->{config}{rewrite} }});
	if ($anvil->data->{config}{rewrite})
	{
		# Backup the original
		my $backup_file = $anvil->Storage->backup({secure => 1, file => $anvil->data->{path}{configs}{'anvil.conf'}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});
		update_progress($anvil, 75, "message_0072,!!backup!".$backup_file."!!");
		
		# Now update!
		$anvil->Storage->write_file({
			debug     => 3,
			secure    => 1, 
			file      => $anvil->data->{path}{configs}{'anvil.conf'}, 
			body      => $new_body, 
			user      => "admin", 
			group     => "admin", 
			mode      => "0644",
			overwrite => 1,
		});
		update_progress($anvil, 80, "message_0073");
		update_progress($anvil, 82, "job_0131");
		
		# Reconnect. This can take a few tries as we ma need to initialize the DB as well.
		my $waiting    = 1;
		my $restarted  = 0;
		my $abort_time = time + 5;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { abort_time => $abort_time }});
		while($waiting)
		{
			$waiting = 0;
			
			# Disconnect (if no connections exist, will still clear out known databases).
			$anvil->Database->disconnect();
			
			# Re-read the config.
			$anvil->refresh();
			
			# Flag a resync, connect, and configure, if needed.
			$anvil->data->{sys}{database}{resync_needed} = 1;
			$anvil->Database->connect({
				debug               => 3,
				check_for_resync    => 1,
				check_if_configured => $host_uuid eq $anvil->Get->host_uuid ? 1 : 0,
			});
			
			my $target_handle = $anvil->data->{cache}{database_handle}{$host_uuid} ? $anvil->data->{cache}{database_handle}{$host_uuid} : "";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_handle => $target_handle }});
			
			if ((not $target_handle) or ($target_handle !~ /^DBI::db=HASH/))
			{
				# not connected yet.
				my $time = time;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					'time'     => $time, 
					abort_time => $abort_time,
				}});
				if ($time < $abort_time)
				{
					# sleep and try again.
					$anvil->Database->disconnect();
					
					# Try restarting the database if the $host_uuid is us.
					if (($host_uuid eq $anvil->Get->host_uuid) && (not $restarted))
					{
						my $stop_return_code  = $anvil->System->stop_daemon({debug => 2, daemon => "postgresql.service"});
						my $start_return_code = $anvil->System->start_daemon({debug => 2, daemon => "postgresql.service"});
						   $restarted   = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							's1:restarted'         => $restarted, 
							's2:stop_return_code'  => $stop_return_code,
							's3:start_return_code' => $start_return_code, 
						}});
					}
					
					$waiting = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
					sleep 1;
				}
				else
				{
					# Timed out.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0120", variables => { uuid => $host_uuid }});
				}
			}
			else
			{
				# Connected!
			}
		}
	}
	
	# If I've been asked to have the peer add us, disconnect from the database, re-read the new config 
	# and then reconnect. This should cause a sync of 'hosts' and allow us to insert a job for the peer 
	# to add us.
	if ($add_peer_command)
	{
		update_progress($anvil, 85, "message_0074");
		
		# Now loop until we see the peer's host_uuid show up and we have a connection to the peer's
		# database.
		my $peer_connected = 0;
		until($peer_connected)
		{
			my $say_host = $anvil->Get->host_name_from_uuid({host_uuid => $host_uuid});
			   $say_host = $host_uuid if not $say_host;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				say_host                               => $say_host, 
				"cache::database_handle::${host_uuid}" => $anvil->data->{cache}{database_handle}{$host_uuid},
			}});
			if ($anvil->data->{cache}{database_handle}{$host_uuid} =~ /^DBI::db=HASH/)
			{
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0223", variables => { host => $say_host }});
				$peer_connected = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { peer_connected => $peer_connected }});
			}
			else
			{
				# Sleep and then try again to connect.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0222", variables => { host => $say_host }});
				
				sleep 1;
				
				$anvil->data->{sys}{database}{resync_needed} = 1;
				$anvil->Database->connect({
					debug            => 2,
					check_for_resync => 1, 
					db_uuid          => $host_uuid,
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
			}
		}
		
		my $peer_host_name = "";
		my $host_seen      = 0;
		until($host_seen)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0212"});
			my $query = "SELECT host_name FROM hosts WHERE host_uuid = ".$anvil->Database->quote($host_uuid).";";
			my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
			my $count   = @{$results};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				results => $results, 
				count   => $count, 
			}});
			if ($count)
			{
				$peer_host_name = $results->[0]->[0];
				$host_seen      = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					peer_host_name => $peer_host_name, 
					host_seen      => $host_seen, 
				}});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0214", variables => { peer_name => $peer_host_name }});
				update_progress($anvil, 90, "message_0075,!!host!".$peer_host_name."!!");
			}
			else
			{
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0213", variables => { peer_uuid => $host_uuid }});
				sleep(1);
			}
		}
		
		# Now add the job.
		my $our_host_uuid = $anvil->Get->host_uuid;
		my $sql_password  = $anvil->data->{database}{$our_host_uuid}{password};
		my $job_command   = $add_peer_command;
		my $job_data      = "password=".$sql_password;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, list => { job_command => $job_command }});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { job_data    => $job_data    }});
		
		# Store the job
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2,
			file            => $THIS_FILE, 
			line            => __LINE__, 
			job_host_uuid   => $host_uuid, 
			job_command     => $job_command, 
			job_data        => $job_data, 
			job_name        => "striker-peer::add", 
			job_title       => "job_0011", 
			job_description => "job_0012", 
			job_progress    => 0,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		update_progress($anvil, 95, "message_0076");
	}
	
	return(0);
}