Merge pull request #530 from ClusterLabs/anvil-tools-dev

Anvil tools dev

Anvil/Tools/Database.pm

@@ -4203,6 +4203,14 @@ WHERE
 			"sys::hosts::by_uuid::${host_uuid}" => $anvil->data->{sys}{hosts}{by_uuid}{$host_uuid}, 
 			"sys::hosts::by_name::${host_name}" => $anvil->data->{sys}{hosts}{by_name}{$host_name}, 
 		}});
+		
+		# Record hosts by type.
+		$anvil->data->{sys}{hosts}{by_type}{$host_type}{host_name}{$host_name}{host_uuid}       = $host_uuid;
+		$anvil->data->{sys}{hosts}{by_type}{$host_type}{host_name}{$host_name}{short_host_name} = $short_host_name;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			"sys::hosts::by_type::${host_type}::host_name::${host_name}::host_uuid"       => $anvil->data->{sys}{hosts}{by_type}{$host_type}{host_name}{$host_name}{host_uuid}, 
+			"sys::hosts::by_type::${host_type}::host_name::${host_name}::short_host_name" => $anvil->data->{sys}{hosts}{by_type}{$host_type}{host_name}{$host_name}{short_host_name}, 
+		}});
 	}
 	
 	my $return_count = @{$return};
@@ -5898,10 +5906,12 @@ WHERE
 			"servers::server_uuid::${server_uuid}::server_definition_xml"           => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_definition_xml}, 
 		}});
 		
-		# Store the servers in a hash under each Anvil!, sortable.
+		# Store the servers in a hash by name and under each Anvil!, sortable.
 		$anvil->data->{servers}{anvil_uuid}{$server_anvil_uuid}{server_name}{$server_name}{server_uuid} = $server_uuid;
+		$anvil->data->{servers}{server_name}{$server_name}{server_uuid}                                 = $server_uuid;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			"servers::anvil_uuid::${server_anvil_uuid}::server_name::${server_name}::server_uuid" => $anvil->data->{servers}{anvil_uuid}{$server_anvil_uuid}{server_name}{$server_name}{server_uuid}, 
+			"servers::server_name::${server_name}::server_uuid"                                   => $anvil->data->{servers}{server_name}{$server_name}{server_uuid}, 
 		}});
 	}
 	

Anvil/Tools/Remote.pm

@@ -489,6 +489,12 @@ sub call
 		foreach (my $i = 0; $i <= $last_loop; $i++)
 		{
 			last if $connected;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				's1:i'           => $i, 
+				's2:target'      => $target, 
+				's3:remote_user' => $remote_user, 
+				's4:port'        => $port, 
+			}});
 			($connect_output) = capture_merged {
 				$ssh_fh = Net::OpenSSH->new($target, 
 					user       => $remote_user,
@@ -500,12 +506,9 @@ sub call
 			$connect_output =~ s/\r//gs;
 			$connect_output =~ s/\n$//; 
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-				's1:i'              => $i, 
+				's1:ssh_fh'         => $ssh_fh,
-				's2:target'         => $target, 
+				's2:ssh_fh->error'  => $ssh_fh->error,
-				's3:port'           => $port, 
+				's3:connect_output' => $connect_output, 
-				's4:ssh_fh'         => $ssh_fh,
-				's5:ssh_fh->error'  => $ssh_fh->error,
-				's6:connect_output' => $connect_output, 
 			}});
 			
 			# Any fatal issues reaching the target?
@@ -668,7 +671,6 @@ sub call
 				ssh_fh_key => $ssh_fh_key, 
 			}});
 		}
-	
 	}
 	
 	# If I have a valid handle, try to call our command now. Note that if we're using a cached connection

man/anvil-manage-dr.8

@@ -21,6 +21,9 @@ When logging, record sensitive data, like passwords.
 Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
 .SS "Commands:"
 .TP
+\fB\-\-anvil\fR
+When linking or unlinking a DR host to an Anvil! node, this switch indicates the Anvil! being linked or unlinked to.
+.TP
 \fB\-\-connect\fR
 Connect a server already on DR to it's DR copy, update the data there if needed and begin streaming replication.
 .TP
@@ -28,7 +31,7 @@ Connect a server already on DR to it's DR copy, update the data there if needed
 Disconnect a server from the DR image. This will end streaming replication.
 .TP
 \fB\-\-dr\-host\fR
-This is the host name or host UUID for the DR to use. It is optional if only one DR host is connected to this Anvil! node, but required if two or more are defined.
+This is the host name or host UUID for the DR to use. It is optional if only one DR host is connected to this Anvil! node, but required if two or more are defined. It is also use when linking or unlinking DR hosts to Anvil! nodes.
 .TP
 \fB\-\-license\-file\fR
 This is the path to the license file, needed when setting up "long-throw" DR for the first time.
@@ -36,6 +39,9 @@ This is the path to the license file, needed when setting up "long-throw" DR for
 \fB\-\-link\fR
 This takes an --anvil and a --dr-host to enable using the DR host as a target for the Anvil! node.
 .TP
+\fB\-\-list\fR
+This will show the list of available Anvil! nodes, DR hosts and servers.
+.TP
 \fB\-\-protect\fR
 The sets up the server to be imaged on DR, if it isn't already protected.
 .TP

man/anvil-manage-keys.1

@@ -0,0 +1,33 @@
+.\" Manpage for the Anvil! server system manager
+.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
+.TH anvil-manage-keys "8" "November 13 2023" "Anvil! Intelligent Availability™ Platform"
+.SH NAME
+anvil-manage-keys \- Tool used to manage ssh keys
+.SH SYNOPSIS
+.B anvil-manage-keys 
+\fI\,<command> \/\fR[\fI\,options\/\fR]
+.SH DESCRIPTION
+anvil-manage-keys \- This tool is used remove old SSH keys from a machine that has been rebuilt.
+.TP
+.SH OPTIONS
+.TP
+\-?, \-h, \fB\-\-help\fR
+Show this man page.
+.TP
+\fB\-\-log-secure\fR
+When logging, record sensitive data, like passwords.
+.TP
+\-v, \-vv, \-vvv
+Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
+.SS "Commands:"
+.TP
+\fB\-\-confirm\fR
+When called without a job-uuid, a search is made to find any bad keys. If they are found, the user is prompted to delete them. This switch assumes consent is given and the keys are removed.
+.TP
+\fB\-\-job\-uuid\fR
+This is the jobs -> job_uuid to execute. Generally this is only used by other programs.
+.IP
+.SH AUTHOR
+Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
+.SH "REPORTING BUGS"
+Report bugs to users@clusterlabs.org

share/words.xml

@@ -1129,7 +1129,7 @@ resource #!variable!server!# {
 		<key name="header_0078">#!free!#</key>
 		<key name="header_0079">Used</key>
 		<key name="header_0080">Free</key>
-		<key name="header_0081">Anvil! Node</key>
+		<key name="header_0081">Anvil! Nodes</key>
 		<key name="header_0082">Interface</key>
 		<key name="header_0083">Gateway (*DG)</key>
 		<key name="header_0084">Transmitted</key>
@@ -1149,7 +1149,7 @@ resource #!variable!server!# {
 		<key name="header_0098">Recipient UUID</key>
 		<key name="header_0099">Alert Overrides</key>
 		<key name="header_0100">Recipient</key>
-		<key name="header_0101">Anvil! Node</key>
+		<key name="header_0101">-=] Anvil! Node</key>
 		<key name="header_0102">Alert Override UUID</key>
 		<key name="header_0103">Host Type</key>
 		<key name="header_0104">Host UUID</key>
@@ -1163,6 +1163,8 @@ resource #!variable!server!# {
 		<key name="header_0112">Queued Jobs</key>
 		<key name="header_0113">Jobs In Progress</key>
 		<key name="header_0114">Completed Jobs</key>
+		<key name="header_0115">-=] DR Hosts</key>
+		<key name="header_0116">-=] Servers</key>
 
 		<!-- Strings used by jobs -->
 		<key name="job_0001">Configure Network</key>
@@ -3223,6 +3225,17 @@ proceeding.
 		<key name="message_0361">- [ Note ] - The original partition table for: [#!variable!device_path!#] has been saved to: [#!variable!partition_backup!#]
              If anything goes wrong, we will attempt to recover automatically. If needed, you can try
              recovering with: [#!variable!restore_command!#]</key>
+		<key name="message_0362">Bad SSH key target: [#!variable!target!#], file:line with the bad entry: [#!variable!file!#:#!variable!line!#].</key>
+		<key name="message_0363">[ Warning ] - Only proceed if you know the target has been rebuily!
+- Do you want to remove these bad keys? [y/N]</key>
+		<key name="message_0364">- Node Name: [#!variable!anvil_name!#], Description: [#!variable!description!#]</key>
+		<key name="message_0365"> - Linked: [#!variable!host_name!#], link UUID: [#!variable!link_uuid!#]</key>
+		<key name="message_0366"> - No linked DR hosts yet.</key>
+		<key name="message_0367">- No Anvil! nodes exist yet.</key>
+		<key name="message_0368">- Name: [#!variable!host_name!#]</key>
+		<key name="message_0369">- No DR hosts exist yet</key>
+		<key name="message_0370">- Server name: [#!variable!server_name!#] on Anvil! Node: [#!variable!anvil_name!#]</key>
+		<key name="message_0371">- No servers yet installed</key>
 
 		<!-- Translate names (protocols, etc) -->
 		<key name="name_0001">Normal Password</key>	<!-- none in mail-server -->
@@ -3573,7 +3586,7 @@ If you are comfortable that the target has changed for a known reason, you can s
 		<key name="striker_0303">This indicates when, in unix time, the database was last archived.</key>
 		<key name="striker_0304">This indicates when, in unix time, the local install target data was updated.</key>
 		<key name="striker_0305">This indicates when, in unix time, the OUI data was last update. The OUI data is a list of MAC address prefixes and which companies they've been assigned to.</key>
-		<key name="striker_0306">This indicates when, in unix time, the network was last scanned. This is done to determine what IPs are used by servers on the Anvil! cluster, and to try to identify foundation pack devices on the network. These scans are simple ping sweeps used to get the MAC addresses of devices with IPs.</key>
+		<key name="striker_0306">This indicates when, in unix time, the network was last scanned. This is done to determine what IPs are used by servers on the Anvil! node, and to try to identify foundation pack devices on the network. These scans are simple ping sweeps used to get the MAC addresses of devices with IPs.</key>
 
 		<!-- These are generally units and appended to numbers -->
 		<key name="suffix_0001">#!variable!number!#/sec</key>

tools/anvil-manage-alerts

@@ -827,7 +827,7 @@ sub show_existing
 	$anvil->data->{longest}{recipient_uuid}                = length($anvil->Words->string({key => "header_0098"}));
 	$anvil->data->{longest}{alert_override_recipient_name} = length($anvil->Words->string({key => "header_0100"}));
 	$anvil->data->{longest}{alert_override_host_name}      = length($anvil->Words->string({key => "header_0026"}));
-	$anvil->data->{longest}{alert_override_anvil_name}     = length($anvil->Words->string({key => "header_0101"}));
+	$anvil->data->{longest}{alert_override_anvil_name}     = length($anvil->Words->string({key => "header_0081"}));
 	$anvil->data->{longest}{alert_override_alert_level}    = length($anvil->Words->string({key => "header_0096"}));
 	$anvil->data->{longest}{alert_override_uuid}           = length($anvil->Words->string({key => "header_0102"}));
 	$anvil->data->{longest}{host_name}                     = length($anvil->Words->string({key => "header_0026"}));
@@ -1333,7 +1333,7 @@ sub show_existing
 	
 		my $header_line =  "| ".sprintf("%-${longest_alert_override_recipient_name}s", $anvil->Words->string({key => "header_0100"}))." ";
 		   $header_line .= "| ".sprintf("%-${longest_alert_override_host_name}s", $anvil->Words->string({key => "header_0026"}))." ";
-		   $header_line .= "| ".sprintf("%-${longest_alert_override_anvil_name}s", $anvil->Words->string({key => "header_0101"}))." ";
+		   $header_line .= "| ".sprintf("%-${longest_alert_override_anvil_name}s", $anvil->Words->string({key => "header_0081"}))." ";
 		   $header_line .= "| ".sprintf("%-${longest_alert_override_alert_level}s", $anvil->Words->string({key => "header_0096"}))." ";
 		   $header_line .= "| ".sprintf("%-${longest_alert_override_uuid}s", $anvil->Words->string({key => "header_0102"}))." |";
 		   

tools/anvil-manage-dr

@@ -42,6 +42,7 @@ $anvil->Get->switches({list => [
 	"job-uuid",
 	"license-file",
 	"link",
+	"list", 
 	"protect",
 	"protocol",
 	"remove",
@@ -74,6 +75,11 @@ if ($anvil->data->{switches}{'job-uuid'})
 {
 	load_job($anvil);
 }
+elsif ($anvil->data->{switches}{list})
+{
+	# If '--list' was used, show all Nodes, DR hosts and servers.
+	show_list($anvil);
+}
 
 my $terminal = "";
 if (not $anvil->data->{switches}{'job-uuid'})
@@ -98,6 +104,115 @@ $anvil->nice_exit({exit_code => 0});
 # Functions                                                                                                 #
 #############################################################################################################
 
+sub show_list
+{
+	my ($anvil) = @_;
+	
+	# Show Anvil! nodes
+	$anvil->Database->get_hosts();
+	$anvil->Database->get_storage_group_data({debug => 2});
+	$anvil->Database->get_dr_links({debug => 2});
+	$anvil->Database->get_servers();
+	
+	# Show Anvil! nodes.
+	print $anvil->Words->string({key => "header_0081"})."\n";
+	my $anvil_count = keys %{$anvil->data->{anvils}{anvil_name}};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_count => $anvil_count }});
+	if ($anvil_count)
+	{
+		foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
+		{
+			my $anvil_uuid        = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
+			my $anvil_description = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description};
+			my $dr_link_count     = keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				anvil_name        => $anvil_name,
+				anvil_uuid        => $anvil_uuid, 
+				anvil_description => $anvil_description, 
+				dr_link_count     => $dr_link_count, 
+			}});
+			print $anvil->Words->string({key => "message_0364", variables => { 
+				anvil_name  => $anvil_name,
+				description => $anvil_description,
+			}})."\n";
+			if ($dr_link_count)
+			{
+				foreach my $dr_link_host_name (sort {$a cmp $b} keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}})
+				{
+					my $dr_link_uuid = $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}{$dr_link_host_name}{dr_link_uuid};
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+						dr_link_host_name => $dr_link_host_name,
+						dr_link_uuid      => $dr_link_uuid, 
+					}});
+					print $anvil->Words->string({key => "message_0365", variables => { 
+						host_name => $dr_link_host_name,
+						link_uuid => $dr_link_uuid,
+					}})."\n";
+				}
+			}
+			else
+			{
+				# No DR hosts yet
+				print $anvil->Words->string({key => "message_0366"})."\n";
+			}
+		}
+	}
+	else
+	{
+		# No nodes yet
+		print $anvil->Words->string({key => "message_0367"})."\n";
+	}
+	
+	# DR Hosts
+	print "\n".$anvil->Words->string({key => "header_0115"})."\n";
+	my $dr_host_count = keys %{$anvil->data->{sys}{hosts}{by_type}{dr}{host_name}};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_host_count => $dr_host_count }});
+	if ($dr_host_count)
+	{
+		# TODO: Show free space of VGs, available RAM and CPU core count and type/speed
+		foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_type}{dr}{host_name}})
+		{
+			print $anvil->Words->string({key => "message_0368", variables => { host_name => $host_name }})."\n";
+		}
+	}
+	else
+	{
+		# No DR hosts yet
+		print $anvil->Words->string({key => "message_0366"})."\n";
+	}
+	
+	# DR Hosts
+	print "\n".$anvil->Words->string({key => "header_0116"})."\n";
+	my $server_count = keys %{$anvil->data->{servers}{server_uuid}};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_count => $server_count }});
+	if ($server_count)
+	{
+		# If there's servers, there has to be Anvil! nodes.
+		foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
+		{
+			# TODO: Show each server's disk(s) size, RAM and CPU core count
+			my $anvil_uuid  = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
+			foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
+			{
+				my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
+				my $server_name = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
+				print $anvil->Words->string({key => "message_0370", variables => { 
+					server_name => $server_name,
+					anvil_name  => $anvil_name,
+				}})."\n";
+			}
+		}
+	}
+	else
+	{
+		# No servers yet exist
+		print $anvil->Words->string({key => "message_0371"})."\n";
+	}
+	$anvil->nice_exit({exit_code => 0});
+	
+	return(0);
+}
+
 sub handle_links
 {
 	my ($anvil) = @_;
@@ -668,6 +783,7 @@ sub sanity_check
 		"sys::dr_short_host_name" => $anvil->data->{sys}{dr_short_host_name},
 	}});
 
+	### TODO: Switch to: my $target_ip = $anvil->Network->find_target_ip({});
 	# Can we access DR, if we're not the DR host?
 	$anvil->Database->get_ip_addresses({debug => 2});
 	my $password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
@@ -684,7 +800,7 @@ sub sanity_check
 		if ((not $dr_ip) or ($dr_ip eq "!!error!!"))
 		{
 			# Failed to find an IP we can access the DR host. Has it been configured? Is it running? Exiting.
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "error_0334", variables => { host_name => $dr_host_name }});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, secure => 0, key => "error_0334", variables => { host_name => $dr_host_name }});
 			$anvil->Job->update_progress({
 				progress   => 0,
 				message    => "error_0334,!!host_name!".$dr_host_name."!!",

tools/anvil-manage-keys

@@ -34,12 +34,12 @@ my $anvil = Anvil::Tools->new();
 
 # Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is 
 # passed directly, it will be used. Otherwise, the password will be read from the database.
-$anvil->data->{switches}{'job-uuid'} = "";
+$anvil->Get->switches({list => [
-$anvil->Get->switches;
+	"confirm", 
-$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
+	"job-uuid", 
-$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+	], man => $THIS_FILE});
-	'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'},
+$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
-}});
+$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 
 $anvil->Database->connect();
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
@@ -329,10 +329,128 @@ sub load_job_data
 {
 	my ($anvil) = @_;
 	
+	$anvil->data->{state_uuids} = [];
 	if (not $anvil->data->{switches}{'job-uuid'})
 	{
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0080"});
+		# See if we can get the data from the command line switches.
-		$anvil->nice_exit({exit_code => 1});
+		my $bad_keys = 0;
+		my $query    = "
+SELECT 
+    state_uuid, 
+    state_name, 
+    state_note 
+FROM 
+    states 
+WHERE 
+    state_name 
+LIKE 
+    'host_key_changed::%' 
+AND 
+    state_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
+;";
+		# Collect from all DBs.
+		foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
+		{
+			my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
+			my $count   = @{$results};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				results => $results, 
+				count   => $count, 
+			}});
+			foreach my $row (@{$results})
+			{
+				
+				my $state_uuid = $row->[0];
+				my $state_name = $row->[1];
+				my $state_note = $row->[2];
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					's1:state_uuid' => $state_uuid, 
+					's2:state_name' => $state_name, 
+					's2:state_note' => $state_note, 
+				}});
+				
+				my $bad_file = "";
+				my $bad_line = "";
+				foreach my $pair (split/,/, $state_note)
+				{
+					my ($variable, $value) = ($pair =~ /^(.*?)=(.*)$/);
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+						pair     => $pair, 
+						variable => $variable, 
+						value    => $value,
+					}});
+					if ($variable eq "file")
+					{
+						$bad_file = $value;
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bad_file => $bad_file }});
+					}
+					if ($variable eq "line")
+					{
+						$bad_line = $value;
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bad_line => $bad_line }});
+					}
+				}
+				   $bad_keys++;
+				my ($target) = ($state_name =~ /host_key_changed::(.*)$/);
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					bad_keys => $bad_keys, 
+					target   => $target, 
+					bad_file => $bad_file, 
+					bad_line => $bad_line, 
+				}});
+				
+				$anvil->data->{bad_keys}{$target}{$bad_file}{$bad_line}{$state_uuid} = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					"bad_keys::${target}::${bad_file}::${bad_line}" => $anvil->data->{bad_keys}{$target}{$bad_file}{$bad_line}, 
+				}});
+			}
+		}
+		
+		foreach my $target (sort {$a cmp $b} keys %{$anvil->data->{bad_keys}})
+		{
+			foreach my $bad_file (sort {$a cmp $b} keys %{$anvil->data->{bad_keys}{$target}})
+			{
+				foreach my $bad_line (sort {$a cmp $b} keys %{$anvil->data->{bad_keys}{$target}{$bad_file}})
+				{
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0362", variables => {
+						target => $target,
+						file   => $bad_file, 
+						line   => $bad_line, 
+					}});
+					foreach my $state_uuid (sort {$a cmp $b} keys %{$anvil->data->{bad_keys}{$target}{$bad_file}{$bad_line}})
+					{
+						push @{$anvil->data->{state_uuids}}, $state_uuid;
+					}
+				}
+			}
+		}
+		
+		# Ask the user to confirm?
+		if ($anvil->data->{switches}{confirm})
+		{
+			# No.
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0359"});
+			return(0);
+		}
+		
+		# Ask the user to confirm.
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0363"});
+		
+		my $answer = <STDIN>;
+		chomp $answer;
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "log_0828", variables => { answer => $answer }});
+		if ((lc($answer) eq "y") or (lc($answer) eq "yes"))
+		{
+			# Proceed.
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0175"});
+			return(0);
+		}
+		else
+		{
+			# Abort.
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0022"});
+			$anvil->nice_exit({exit_code => 0});
+		}
 	}
 	
 	my $query = "SELECT job_data FROM jobs WHERE job_uuid = ".$anvil->Database->quote($anvil->data->{switches}{'job-uuid'}).";";
@@ -372,7 +490,6 @@ sub load_job_data
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0048"});
 	
 	# Break the job up.
-	$anvil->data->{state_uuids} = [];
 	foreach my $state_uuid (split/,/, $job_data)
 	{
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { state_uuid => $state_uuid }});

tools/anvil-manage-server-storage

@@ -41,7 +41,7 @@ $| = 1;
 
 my $anvil = Anvil::Tools->new();
 
-# Read switches (target ([user@]host[:port]) and the file with the target's password.
+# Read switches
 $anvil->Get->switches({list => [
 	"add", 
 	"anvil",