Merge pull request #408 from ClusterLabs/beta-fixes

* Added PID logging as an option, and enabled it in ocf:alteeve:server

Anvil/Tools.pm

@@ -981,6 +981,7 @@ sub _set_defaults
 			facility	=>	"local0",
 			language	=>	"en_CA",
 			level		=>	1,
+			pids		=>	0,
 			secure		=>	0,
 			server		=>	"",
 			tag		=>	"anvil",
@@ -1253,7 +1254,7 @@ sub _set_paths
 				pamscale			=>	"/usr/bin/pamscale",
 				pamtopng			=>	"/usr/bin/pamtopng",
 				passwd				=>	"/usr/bin/passwd",
-				pcs				=>	"/usr/sbin/pcs",
+				pcs				=>	"/usr/sbin/anvil-pcs-wrapper",
 				perccli64			=>	"/opt/MegaRAID/perccli/perccli64",
 				pidof				=>	"/usr/sbin/pidof",
 				ping				=>	"/usr/bin/ping",

Anvil/Tools/DRBD.pm

@@ -93,7 +93,7 @@ sub parent
 
 =head2 allow_two_primaries
 
-This enables dual-primary for the given resource. This is meant to be called prior to a live migration, and should be disabled again as soon as possible via C<< DRBD->reload_defaults >>.
+This enables or disables dual-primary for the given resource. This is meant to be called prior to a live migration, and should be disabled again as soon as possible. The return code of the C<< drbdsetup >> call is returned. If there is a problem, C<< 255 >> is returned.
 
 Parameters; 
 
@@ -209,6 +209,36 @@ sub allow_two_primaries
 		}
 	}
 	
+	# If set to 'yes', make sure the peer is connected. Otherwise we'll just cause problems later when 
+	# they do try to connect.
+	if ($set_to eq "yes")
+	{
+		my $host = $anvil->Get->short_host_name;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host => $host }});
+		if (not exists $anvil->data->{drbd}{config}{$host})
+		{
+			$anvil->DRBD->get_status({debug => $debug});
+		}
+		my $peer_name        = $anvil->data->{drbd}{config}{$host}{peer};
+		my $connection_state = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{'connection-state'};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			peer_name        => $peer_name,
+			resource         => $resource,
+			connection_state => $connection_state, 
+		}});
+		
+		if (lc($connection_state) ne "connected")
+		{
+			# Don't do this!
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "error_0421", variables => { 
+				resource         => $resource,
+				peer_name        => $peer_name, 
+				connection_state => $target_node_id, 
+			}});
+			return($return_code);
+		}
+	}
+	
 	my $key = $set_to eq "yes" ? "log_0350" : "log_0642";
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 1, key => $key, variables => { 
 		resource       => $resource,
@@ -2451,7 +2481,7 @@ sub get_status
 
 =head2 manage_resource
 
-This takes a task, C<< up >>, C<< down >>, C<< primary >>, or C<< secondary >> and a resource name and acts on the request.
+This takes a task, C<< up >>, C<< down >>, C<< primary >>, C<< secondary >>, or C<< adjust >> and a resource name and acts on the request.
 
 This returns the return code from the C<< drbdadm >> call. If C<< 255 >> is returned, then we did not get the actual return code from C<< drbdadm >>.
 
@@ -2475,7 +2505,7 @@ This is the name of the resource being acted upon.
 
 =head3 task (required)
 
-This is the action to take. Valid tasks are: C<< up >>, C<< down >>, C<< primary >>, and C<< secondary >>.
+This is the action to take. Valid tasks are: C<< up >>, C<< down >>, C<< primary >>, C<< secondary >>, and C<< adjust >>.
 
 If C<< target >> is set, this will be the user we connect to the remote machine as.
 
@@ -2498,6 +2528,7 @@ sub manage_resource
 	my $resource    = defined $parameter->{resource}    ? $parameter->{resource}    : "";
 	my $task        = defined $parameter->{task}        ? $parameter->{task}        : "";
 	my $target      = defined $parameter->{target}      ? $parameter->{target}      : "";
+	my $return_code = 255; 
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		password    => $anvil->Log->is_secure($password),
 		port        => $port, 
@@ -2518,6 +2549,44 @@ sub manage_resource
 		return(1);
 	}
 	
+	# If the task is 'adjust', do just that.
+	if ($task eq "adjust")
+	{
+		# Reset to the values in the config and return.
+		my $shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$resource;
+		my $output     = "";
+		if ($anvil->Network->is_local({host => $target}))
+		{
+			# Local.
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
+			($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				output      => $output,
+				return_code => $return_code,
+			}});
+		}
+		else
+		{
+			# Remote call.
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
+			($output, my $error, $return_code) = $anvil->Remote->call({
+				debug       => $debug, 
+				shell_call  => $shell_call, 
+				target      => $target,
+				port        => $port, 
+				password    => $password,
+				remote_user => $remote_user, 
+			});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				error       => $error,
+				output      => $output,
+				return_code => $return_code,
+			}});
+		}
+		
+		return($return_code);
+	}
+	
 	### TODO: When taking down a resource, check to see if any machine is SyncTarget and take it/them 
 	###       down first. See anvil-rename-server -> verify_server_is_off() for the logic.
 	### TODO: Sanity check the resource name and task requested.
@@ -2525,19 +2594,80 @@ sub manage_resource
 	###       This ensures that they're set to 'no' before connecting.
 	if ($task eq "up")
 	{
+		# If our connection state is 'StandAlone', try to connect.
+		my $host = $anvil->Get->short_host_name;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host => $host }});
+		if (not exists $anvil->data->{drbd}{config}{$host})
+		{
+			$anvil->DRBD->get_status({debug => $debug});
+		}
+		my $peer_name        = $anvil->data->{drbd}{config}{$host}{peer}                                                            // "";
+		my $connection_state = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{'connection-state'} // "";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			peer_name        => $peer_name,
+			resource         => $resource,
+			connection_state => $connection_state, 
+		}});
+		if (($connection_state) && (lc($connection_state) eq "standalone"))
+		{
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "log_0746", variables => { resource => $resource }});
+			
+			my $shell_call = $anvil->data->{path}{exe}{drbdadm}." connect ".$resource;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
+			my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				output      => $output,
+				return_code => $return_code,
+			}});
+			sleep 1;
+			
+			# Loop to see if it connects.
+			my $waiting    = 1;
+			my $wait_until = time + 10;
+			while ($waiting)
+			{
+				$anvil->DRBD->get_status({debug => $debug});
+				my $connection_state = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{'connection-state'};
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { connection_state => $connection_state }});
+				if (lc($connection_state eq "connecting"))
+				{
+					if (time > $wait_until)
+					{
+						# Stop waiting.
+						$waiting = 0;
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
+					}
+					else
+					{
+						# Keep waiting.
+						sleep 1;
+					}
+				}
+				else
+				{
+					# Done!
+					$waiting = 0;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
+				}
+			}
+		}
+		
 		# This generally brings up the resource
-		my ($return) = $anvil->DRBD->allow_two_primaries({
+		if ($peer_name)
-			debug    => 2, 
+		{
-			resource => $resource, 
+			# This isn't fatal when bringing up the resource
-			set_to   => "no", 
+			my ($return_code) = $anvil->DRBD->allow_two_primaries({
-		});
+				debug    => 2, 
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'return' => $return }});
+				resource => $resource, 
+				set_to   => "no", 
+			});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
+		}
 		
 		# Now call an adjust to make sure all other config details are loaded. It also up's the 
 		# resource.
-		my $shell_call  = $anvil->data->{path}{exe}{drbdadm}." adjust ".$resource;
+		my $shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$resource;
-		my $output      = "";
+		my $output     = "";
-		my $return_code = 255; 
 		if ($anvil->Network->is_local({host => $target}))
 		{
 			# Local.
@@ -2573,9 +2703,8 @@ sub manage_resource
 	
 	# If we 'adjust'ed above, this will likely complain that the backing disk already exists, and that's 
 	# fine.
-	my $shell_call  = $anvil->data->{path}{exe}{drbdadm}." ".$task." ".$resource;
+	my $shell_call = $anvil->data->{path}{exe}{drbdadm}." ".$task." ".$resource;
-	my $output      = "";
+	my $output     = "";
-	my $return_code = 255; 
 	if ($anvil->Network->is_local({host => $target}))
 	{
 		# Local.

Anvil/Tools/Database.pm

@@ -4984,10 +4984,18 @@ FROM
 		
 		if (not exists $anvil->data->{hosts}{host_uuid}{$scan_lvm_pv_host_uuid})
 		{
-			$anvil->Database->get_hosts({debug => $debug});
+			$anvil->Database->get_hosts({
+				debug           => $debug,
+				include_deleted => 1,
+			});
 		}
 		my $short_host_name = $anvil->data->{hosts}{host_uuid}{$scan_lvm_pv_host_uuid}{short_host_name};
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { short_host_name => $short_host_name }});
+		my $host_key        = $anvil->data->{hosts}{host_uuid}{$scan_lvm_pv_host_uuid}{short_host_name};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			short_host_name => $short_host_name,
+			host_key        => $host_key, 
+		}});
+		next if $host_key eq "DELETED";
 		
 		$anvil->data->{lvm}{host_name}{$short_host_name}{pv}{$scan_lvm_pv_name}{scan_lvm_pv_uuid}          = $scan_lvm_pv_uuid;
 		$anvil->data->{lvm}{host_name}{$short_host_name}{pv}{$scan_lvm_pv_name}{scan_lvm_pv_internal_uuid} = $scan_lvm_pv_internal_uuid;

Anvil/Tools/Log.pm

@@ -373,6 +373,10 @@ sub entry
 		$job_uuid =~ s/^(\w+?)-.*$/$1/;
 		$string   .= "[".$job_uuid."]:";
 	}
+	if ($anvil->data->{defaults}{'log'}{pids})
+	{
+		$string   .= "[".$$."]:";
+	}
 	if (exists $anvil->data->{'log'}{scan_agent})
 	{
 		$string   .= "[".$anvil->data->{'log'}{scan_agent}."]:";

Anvil/Tools/Server.pm

@@ -1148,6 +1148,16 @@ sub migrate_virsh
 		});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { return_code => $return_code }});
 		
+		if ($return_code) 
+		{
+			# Abort the migration.
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "error_0422", variables => { 
+				server_name => $server,
+				return_code => $return_code, 
+			}});
+			return(0);
+		}
+		
 		push @{$resources_to_disable_dual_primary}, $resource;
 	}
 	

cgi-bin/striker

@@ -34,8 +34,6 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 }
 
 my $anvil = Anvil::Tools->new();
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
 
 ### NOTE: We'll print the headers only when we need to. If we print them here, it will block cookies being set.
 # Setup some variables.

cgi-bin/upload.pl

@@ -22,7 +22,6 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 }
 
 my $anvil = Anvil::Tools->new();
-$anvil->Log->level({set => 2});
 
 $anvil->Get->switches;
 my $cgi = CGI->new; 

man/Makefile.am

@@ -12,27 +12,68 @@ dist_man8_MANS		= \
 			  anvil-change-password.8 \
 			  anvil-check-memory.8 \
 			  anvil-configure-host.8 \
+			  anvil-cycle-vm-nics.8 \
 			  anvil-daemon.8 \
 			  anvil-delete-server.8 \
+			  anvil-download-file.8 \
+			  anvil-file-details.8 \
+			  anvil-join-anvil.8 \
+			  anvil-maintenance-mode.8 \
 			  anvil-manage-alerts.8 \
 			  anvil-manage-dr.8 \
 			  anvil-manage-files.8 \
+			  anvil-manage-firewall.8 \
+			  anvil-manage-host.8 \
 			  anvil-manage-power.8 \
 			  anvil-manage-server.8 \
 			  anvil-manage-server-storage.8 \
 			  anvil-manage-storage-groups.8 \
+			  anvil-migrate-server.8 \
+			  anvil-network-profiler.8 \
+			  anvil-parse-fence-agents.8 \
+			  anvil-pcs-wrapper.8 \
+			  anvil-provision-server.8 \
+			  anvil-rename-server.8 \
 			  anvil-report-usage.8 \
 			  anvil-safe-start.8 \
 			  anvil-safe-stop.8 \
+			  anvil-scan-network.8 \
+			  anvil-show-local-ips.8 \
 			  anvil-shutdown-server.8 \
 			  anvil-special-operations.8 \
+			  anvil-sync-shared.8 \
+			  anvil-test-alerts.8 \
+			  anvil-update-definition.8 \
+			  anvil-update-issue.8 \
 			  anvil-update-system.8 \
+			  anvil-version-changes.8 \
+			  anvil-virsh-wrapper.8 \
+			  anvil-watch-bonds.8 \
 			  anvil-watch-drbd.8 \
+			  anvil-watch-power.8 \
+			  fence_delay.8 \
+			  fence_pacemaker.8 \
 			  scancore.8 \
+			  striker-auto-initialize-all.8 \
+			  striker-boot-machine.8 \
 			  striker-check-machines.8 \
 			  striker-collect-debug.8 \
+			  striker-db-report.8 \
+			  striker-db-status.8 \
+			  striker-file-manager.8 \
+			  striker-get-peer-data.8 \
 			  striker-initialize-host.8 \
-			  striker-update-cluster.8 
+			  striker-manage-install-target.8 \
+			  striker-manage-peers.8 \
+			  striker-parse-os-list.8 \
+			  striker-parse-oui.8 \
+			  striker-prep-database.8 \
+			  striker-purge-target.8 \
+			  striker-scan-network.8 \
+			  striker-show-db-counts.8 \
+			  striker-update-cluster.8 \
+			  tool-fio-tester.8 \
+			  unfence_pacemaker.8
 
 
 

man/alteeve-repo-setup.8

@@ -33,7 +33,7 @@ To access our enterprise repository with Alteeve support please visit:
 \-?, \-h, \fB\-\-help\fR
 Show this man page.
 .TP
-\fB\-\-log-secure\fR
+\fB\-\-log\-secure\fR
 When logging, record sensitive data, like passwords.
 .TP
 \-d, \fB\-\-debug\fR

man/anvil-change-password.8

@@ -28,12 +28,12 @@ When set, this changes the password on the target Anvil! sub-cluster. This is th
 .TP
 When not set, the Striker dashboard this command is run on will have it's passwords updated.
 .TP
-\fB\-\-new-password\fR <secret>
+\fB\-\-new\-password\fR <secret>
 This is the new password to set. See '\fB\-\-password-file\fR' below for an alternate way to pass in the password. 
 .TP
 If not set, you will be prompted to enter the new password.
 .TP
-\fB\-\-password-file\fR </path/to/file>
+\fB\-\-password\-file\fR </path/to/file>
 This is an alternative way to pass the new password to this program. If set, the file is read in and the file contents are used. Be sure to use one line only in the file.
 .IP
 .SH AUTHOR

man/anvil-configure-host.8

@@ -21,7 +21,7 @@ When logging, record sensitive data, like passwords.
 Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
 .SS "Commands:"
 .TP
-\fB\-\-job-uuid\fR <name>
+\fB\-\-job\-uuid\fR <name>
 The program is normally run as a job, with data on how to configure the host defined in the job. This switch allows the running of a specific job. If this is not set, the program will search for a job that has not yet been picked up by another process. If found, that job UUID is used automatically.
 .IP
 .SH AUTHOR

man/anvil-cycle-vm-nics.8

@@ -0,0 +1,20 @@
+.\" Manpage for the Anvil! tool to cycle NICs on a kvm/qemu VM
+.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
+.TH anvil-cycle-vm-nics "8" "July 26 2023" "Anvil! Intelligent Availability™ Platform"
+.SH NAME
+anvil-cycle-vm-nics \- Tool used to unplug and plug virtual network cables into kvm/qemu VM network interfaces
+.SH SYNOPSIS
+.B anvil-cycle-vm-nics
+\fI\,<server_name> \/\fR
+.SH DESCRIPTION
+anvil-cycle-vm-nics \- This tool makes it easier to map virtual network interfaces during the configuration of VMs in a virtual Anvil! cluster.
+.TP
+The tool takes the name of the server (as displayed in virsh) and the sole argument. There are no other commands.
+.SH NOTE
+.TP
+This cycles NICs connected to bridges with the prefix, in order, 'bcn', 'sn', 'ifn', and 'mn'. 
+.IP
+.SH AUTHOR
+Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
+.SH "REPORTING BUGS"
+Report bugs to users@clusterlabs.org

man/anvil-daemon.8

@@ -1,6 +1,6 @@
 .\" Manpage for the Anvil! daemon. 
 .\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
-.TH anvil-daemon "8" "July 29 2022" "Anvil! Intelligent Availability™ Platform"
+.TH anvil-daemon "8" "July 26 2023" "Anvil! Intelligent Availability™ Platform"
 .SH NAME
 anvil-daemon \- Main systemd daemon for the M3 Anvil! IA cluster. Provides all job management, monitoring and Striker back-end functions. 
 .SH SYNOPSIS
@@ -13,26 +13,23 @@ anvil-daemon \- Main systemd daemon that can be run manually for testing and deb
 \-?, \-h, \fB\-\-help\fR
 Show this man page.
 .TP
-\fB\-\-log-secure\fR
+\fB\-\-log\-secure\fR
 When logging, record sensitive data, like passwords.
 .TP
 \-v, \-vv, \-vvv
 Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
 .SS "Commands:"
 .TP
-\fB\-\-refresh-json\fR (derecated)
+\fB\-\-main\-loop\-only\fR
-Short hand for '\fB\-\-run-once\fR', '\fB\-\-main-loop-only\fR' and '\fB\-\-no-start\fR'. Used to be use to refresh the JSON file used by striker's web interface to know hardware states. 
-.TP
-\fB\-\-main-loop-only\fR
 This skips the one-time, start-up tasks and just goes into the main-loop.
 .TP
-\fB\-\-no-start\fR
+\fB\-\-no\-start\fR
 This will prevent any pending jobs from being picked up and started in this run. Note that other job checks will still happen.
 .TP
-\fB\-\-run-once\fR
+\fB\-\-run\-once\fR
 This will tell the program to exit after running the main loop once.
 .TP
-\fB\-\-startup-only\fR
+\fB\-\-startup\-only\fR
 This will tell the program to exit after running the start up tasks, so the main loop won't run.
 .IP
 .SH AUTHOR

man/anvil-delete-server.8

@@ -16,14 +16,14 @@ This action is permanent!
 \-?, \-h, \fB\-\-help\fR
 Show this man page.
 .TP
-\fB\-\-log-secure\fR
+\fB\-\-log\-secure\fR
 When logging, record sensitive data, like passwords.
 .TP
 \-v, \-vv, \-vvv
 Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
 .SS "Commands:"
 .TP
-\fB\-\-job-uuid\fR <UUID>
+\fB\-\-job\-uuid\fR <UUID>
 The program is normally run as a job, with data on how to configure the host defined in the job. This switch allows the running of a specific job. If this is not set, the program will search for a job that has not yet been picked up by another process. If found, that job UUID is used automatically.
 .IP
 .SS "Commands:"

man/anvil-download-file.8

@@ -0,0 +1,50 @@
+.\" Manpage for the Anvil! tool to 
+.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
+.TH anvil-download-file "8" "July 26 2023" "Anvil! Intelligent Availability™ Platform"
+.SH NAME
+anvil-download-file \- This program downloads a file from a URL
+.SH SYNOPSIS
+.B anvil-download-file 
+\fI\,<options> \/\fR
+.SH DESCRIPTION
+This takes a URL (ftp, http or https) and downloads the file. If it is called without --url, it shows the progress of any other instances currently downloading files.
+.TP
+.SH NOTE
+This tool is not complete and should not be used yet
+.TP
+.SH OPTIONS
+.TP
+\-?, \-h, \fB\-\-help\fR
+Show this man page.
+.TP
+\fB\-\-log\-secure\fR
+When logging, record sensitive data, like passwords.
+.TP
+\-v, \-vv, \-vvv
+Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
+.SS "Commands:"
+.TP
+\fB\-\-abort\fR (to be implemented)
+.TP
+Abort an active download.
+.TP
+\fB\-\-overwrite\fR 
+.TP
+If the file exists already, this switch will overwrite the existing file with the new download.
+.TP
+\fB\-\-save\-to\fR </path/to/download/directory>
+.TP
+By default, downloaded files are saved in /mnt/shared/files (copied there after downloading to /mnt/shared/incoming). If you would like to save the downloaded file elsewhere, you can use this switch
+.TP
+\fB\-\-script\fR 
+.TP
+If this is used, the file being downloaded will be set to be executable. 
+.TP
+\fB\-\-url\fR 
+.TP
+This is the ftp://, http:// or https:// path to the file to be downloaded
+.IP
+.SH AUTHOR
+Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
+.SH "REPORTING BUGS"
+Report bugs to users@clusterlabs.org

man/anvil-file-details.8

@@ -0,0 +1,58 @@
+.\" Manpage for the Anvil! file detail tool
+.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
+.TH anvil-file-details "8" "August 11 2023" "Anvil! Intelligent Availability™ Platform"
+.SH NAME
+anvil-file-details \- Display details of the file passed in
+.SH SYNOPSIS
+.B anvil-file-details 
+\fI\,</path/to/file> \/\fR
+.SH DESCRIPTION
+All this does is stat a file and return the information in a parsable way. For this reason, translatable strings are not used.
+.TP
+.SH OPTIONS
+.TP
+\-?, \-h, \fB\-\-help\fR
+Show this man page.
+.TP
+\fB\-\-log\-secure\fR
+When logging, record sensitive data, like passwords.
+.TP
+\-v, \-vv, \-vvv
+Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
+.SS "Commands:"
+.TP
+\fB\-\-file\fR /path/to/file
+.TP
+This is the file being examined.
+.TP
+\fB\-\-with\-md5sum\fR
+.TP
+Given how long calculating MD5 sums can take on large files, by default this is not done. If you want the md5sum calculated, use this switch.
+.TP
+.SS "EXAMPLE"
+.TP
+The output will show the following information:
+.TP
+.Bl -width
+.It
+[root@an-striker01 ~]# anvil-file-details --with-md5sum --file /mnt/shared/files/rhel-9.2-x86_64-dvd.iso
+.It
+ File: [/mnt/shared/files/rhel-9.2-x86_64-dvd.iso]
+.It
+ size: [9595977728]
+.It
+ mode: [04644]
+.It
+ uid: [1000]
+.It 
+ gid: [1000]
+.It
+ mtime: [1690423778]
+.It
+ md5sum: [90cf58ff7a8f6ef8cb20b8ff091e84b7]
+.El
+.IP
+.SH AUTHOR
+Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
+.SH "REPORTING BUGS"
+Report bugs to users@clusterlabs.org

man/anvil-join-anvil.8

@@ -0,0 +1,35 @@
+.\" Manpage for the Anvil! node assembly tool
+.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
+.TH anvil-join-anvil "8" "August 10 2023" "Anvil! Intelligent Availability™ Platform"
+.SH NAME
+anvil-join-anvil \- This tool assembles two subnodes into a node. 
+.SH SYNOPSIS
+.B anvil-join-anvil 
+\fI\,<command> \/\fR[\fI\,options\/\fR]
+.SH DESCRIPTION
+This program takes two subnodes and merges them into an Anvil! node. This can be two new subnodes, or an existing subnode with a replacement subnode after a subnode failure.
+.TP
+.B Note:
+.TP
+As if this time, this tool only runs from a job registered in the database. As such, the job must be recorded using the Striker web interface
+.TP
+.SH OPTIONS
+.TP
+\-?, \-h, \fB\-\-help\fR
+Show this man page.
+.TP
+\fB\-\-log-secure\fR
+When logging, record sensitive data, like passwords.
+.TP
+\-v, \-vv, \-vvv
+Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
+.SS "Commands:"
+.TP
+\fB\-\-job\-uuid\fR
+.TP
+This is the job UUID that will be run. 
+.IP
+.SH AUTHOR
+Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
+.SH "REPORTING BUGS"
+Report bugs to users@clusterlabs.org

man/anvil-maintenance-mode.8

@@ -0,0 +1,35 @@
+.\" Manpage for the Anvil! maintenance mode tool.
+.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
+.TH anvil-maintenance-mode "8" "Aug 10 2023" "Anvil! Intelligent Availability™ Platform"
+.SH NAME
+anvil-maintenance-mode \- This program manages flagging a machine in an Anvil! cluster as being in maintenance mode
+.SH SYNOPSIS
+.B anvil-maintenance-mode 
+\fI\,<command> \/\fR[\fI\,options\/\fR]
+.SH DESCRIPTION
+This can set or unset a machine in the Anvil! cluster as being in maintenance mode or not. When called without any switches, it reports if the machine is in maintenance mode or not.
+.TP
+.B Note:
+.TP
+Currently, "maintenance mode" is not fully implemented in the Anvil! cluster.
+.TP
+.SH OPTIONS
+.TP
+\-?, \-h, \fB\-\-help\fR
+Show this man page.
+.TP
+\fB\-\-log-secure\fR
+When logging, record sensitive data, like passwords.
+.TP
+\-v, \-vv, \-vvv
+Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
+.SS "Commands:"
+.TP
+\fB\-\-set\fR {0,1}
+.TP
+Passing '0' disables maintenance mode, '1' enables it. 
+.IP
+.SH AUTHOR
+Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
+.SH "REPORTING BUGS"
+Report bugs to users@clusterlabs.org

man/anvil-manage-alerts.8

@@ -17,7 +17,7 @@ When called without any switches, the list of currect mail servers, alert recipi
 \-?, \-h, \fB\-\-help\fR
 Show this man page.
 .TP
-\fB\-\-log-secure\fR
+\fB\-\-log\-secure\fR
 When logging, record sensitive data, like passwords.
 .TP
 \-v, \-vv, \-vvv
@@ -35,19 +35,19 @@ NOTE: All fields are required when editing an existing mail server or recipient!
 \fB\-\-delete\fR 
 This deletes an existing mail server or alert recipient.
 .TP
-\fB\-\-alert-overrides\fR 
+\fB\-\-alert\-overrides\fR 
 This is where an alert recipient can have alert-override overrides. Typically this is used so that a given user can ignore alerts from a specific Anvil! node pair.
 .TP
-\fB\-\-alert-override-uuid\fR <uuid>
+\fB\-\-alert\-override\-uuid\fR <uuid>
 This is required for \fB\-\-edit\fR and \fB\-\-delete\fR. It is the existing alert-override override being worked on.
 .TP
-\fB\-\-alert-override-recipient-uuid\fR <uuid>
+\fB\-\-alert\-override\-recipient\-uuid\fR <uuid>
 This is the recipients -> recipient_uuid who we are creating the override for.
 .TP
-\fB\-\-alert-override-host-uuid\fR 
+\fB\-\-alert\-override\-host\-uuid\fR 
 This is the hosts -> host_uuid of the machine that you are creating the alert 
 .TP
-\fB\-\-alert-override-alert-level\fR <1, 2, 3 or 4>
+\fB\-\-alert\-override\-alert\-level\fR <1, 2, 3 or 4>
 This is the desired override alert level.
 
 Valid values are: 
@@ -73,41 +73,41 @@ Valid values are:
 
 4 or "info"
 .TP
-\fB\-\-mail-servers\fR 
+\fB\-\-mail\-servers\fR 
 This is used to manage mail servers. Specifically, this control the mail server that we send alert emails to. The options used with this are;
 .TP
-\fB\-\-mail-server-uuid\fR <uuid>
+\fB\-\-mail\-server\-uuid\fR <uuid>
 This is required for \fB\-\-edit\fR and \fB\-\-delete\fR. It is the existing mail server being worked on.
 .TP
-\fB\-\-mail-server-address\fR <URL or IP>
+\fB\-\-mail\-server\-address\fR <URL or IP>
 This is the URL or IP address of the mail server we're logging into to send email. 
 
 Example: mail.example.com
 .TP
-\fB\-\-mail-server-port\fR 
+\fB\-\-mail\-server\-port\fR 
 This is the TCP port used when connecting to the target mail server.
 
 Example: 587
 .TP
-\fB\-\-mail-server-username\fR 
+\fB\-\-mail\-server\-username\fR 
 This is the mail server user name (usually an email address) used when authenticating against the mail server.
 
 Example: admin@example.com
 .TP
-\fB\-\-mail-server-password\fR 
+\fB\-\-mail\-server\-password\fR 
 This is the password used along with \fB\-\-mail-server-username\fR when authenticating against the mail server. Not all mail servers require a password, so this is optional.
 .TP
-\fB\-\-mail-server-security\fR <none, starttls or tls-ssl>
+\fB\-\-mail\-server\-security\fR <none, starttls or tls-ssl>
 This is the security type used when authenticating against the mail server. 
 
 Valid values are: 'none', 'starttls' or 'tls-ssl'.
 .TP
-\fB\-\-mail-server-authentication\fR <none, plain-text, or encrypted>
+\fB\-\-mail\-server\-authentication\fR <none, plain-text, or encrypted>
 This is how passwords are passed to the mail server. 
 
 Valid values are: 'none', 'plain-text', or 'encrypted'
 .TP
-\fB\-\-mail-server-helo-domain\fR 
+\fB\-\-mail\-server\-helo\-domain\fR 
 This is the 'HELO' domain name used when communicating with the mail server. This is the domain we're telling the mail server that the email is coming from. You can use your domain, or the domain of the host.
 
 Example: example.com
@@ -117,23 +117,23 @@ See: https://www.ibm.com/docs/en/zos/2.2.0?topic=sc-helo-command-identify-domain
 \fB\-\-recipients\fR 
 This is used to manage alert recipients. Specifically, this control the mail server that we send alert emails to. The options used with this are;
 .TP
-\fB\-\-recipient-uuid\fR 
+\fB\-\-recipient\-uuid\fR 
 This is required for \fB\-\-edit\fR and \fB\-\-delete\fR. It is the existing alert recipient is being worked on.
 .TP
-\fB\-\-recipient-name\fR 
+\fB\-\-recipient\-name\fR 
 This is the name of the person receiving the alerts. This is used in the email header.
 
 Example: Austin Powers
 .TP
-\fB\-\-recipient-email\fR 
+\fB\-\-recipient\-email\fR 
 This is the email address for the alert recipient.
 
 Example: notaspy@example.com
 .TP
-\fB\-\-recipient-language\fR <en_CA>
+\fB\-\-recipient\-language\fR <en_CA>
 In the future, languages will be added and this can be used to indicate what language the user will receive their alerts in. At the time of writing this man page, only 'en_CA' is supported.
 .TP
-\fB\-\-recipient-level\fR <1, 2, 3 or 4>
+\fB\-\-recipient\-level\fR <1, 2, 3 or 4>
 This is the default alert level this recipient is interested in. It can be adjusted on a per-host basis via the 'alert-overrides' over-rides.
 
 Valid values are: 
@@ -148,7 +148,7 @@ Valid values are:
 .TP
 \fB\-\-test\fR
 Tells the program to send a test alert at the desired \fB\-\-level\fR. The requested level is required.
-.TP
+.IP
 .SH AUTHOR
 Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
 .SH "REPORTING BUGS"

man/anvil-manage-files.8

@@ -26,7 +26,7 @@ This will delete the \fB\-\-file\fR </path/to/file> from the entire Anvil! clust
 .TP
 This action is permanent!
 .TP
-\fB\-\-job-uuid\fR <name>
+\fB\-\-job\-uuid\fR <name>
 The program is normally run as a job, with data on how to configure the host defined in the job. This switch allows the running of a specific job. If this is not set, the program will search for a job that has not yet been picked up by another process. If found, that job UUID is used automatically.
 .IP
 .SH AUTHOR

man/striker-update-cluster.8

@@ -57,9 +57,9 @@ Must be used with \fB\-\-reboot\-self\fR to reboot the local system. Otherwise,
 .TP
 \fB\-\-timeout\fR <seconds, Nm, Nh>
 .TP
-When given, if a system update doesn't complete in this amount of time, error out and abort the update. By default, updates will wait for 24 hours.
+When given, if a system update doesn't complete in this amount of time, error out and abort the update. This timeout is used when waiting for actions like waiting for a machine to update, reboot, join a subcluster, and so forth. 
 .TP
-If this is set to an integer, it is treated as a number of seconds. If this ends in 'm' or 'h', then the preceding number is treated as a number of minutes or hours, respectively.
+If this is set to an integer, it is treated as a number of seconds. If this ends in 'm' or 'h', then the preceding number is treated as a number of minutes or hours, respectively. By default, updates will wait for 24 hours.
 .IP
 .SH AUTHOR
 Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.

ocf/alteeve/server

@@ -100,7 +100,8 @@ $| = 1;
 # NOTE: Setting 'log_level' and 'log_secure' here will get overridden in the main lopp. Use the Log methods 
 #       in the loop as well to override defaults in code.
 my $anvil = Anvil::Tools->new();
-#$anvil->Log->level({set => 2});
+# Log PIDs
+$anvil->data->{defaults}{'log'}{pids} = 1;
 
 ### Read or Set the environment variables
 # This is the name of the server we're managing.                # Example values:
@@ -123,7 +124,7 @@ $anvil->data->{environment}{OCF_RESOURCE_PROVIDER}              = defined $ENV{O
 $anvil->data->{environment}{OCF_RESOURCE_TYPE}                  = defined $ENV{OCF_RESOURCE_TYPE}                  ? $ENV{OCF_RESOURCE_TYPE}                  : "server";
 $anvil->data->{environment}{OCF_ROOT}                           = defined $ENV{OCF_ROOT}                           ? $ENV{OCF_ROOT}                           : "/usr/lib/ocf";
 $anvil->data->{environment}{OCF_RESKEY_log_level}               = defined $ENV{OCF_RESKEY_log_level}               ? $ENV{OCF_RESKEY_log_level}               : "";
-$anvil->data->{environment}{OCF_RESKEY_log_secure}              = defined $ENV{OCF_RESKEY_log_secure}              ? $ENV{OCF_RESKEY_log_secure}              : "";
+$anvil->data->{environment}{OCF_RESKEY_log_secure}              = defined $ENV{OCF_RESKEY_log_secure}              ? $ENV{OCF_RESKEY_log_secure}              : 0;
 # These are set during a migration
 $anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_source} = defined $ENV{OCF_RESKEY_CRM_meta_migrate_source} ? $ENV{OCF_RESKEY_CRM_meta_migrate_source} : "";
 $anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target} = defined $ENV{OCF_RESKEY_CRM_meta_migrate_target} ? $ENV{OCF_RESKEY_CRM_meta_migrate_target} : "";
@@ -829,6 +830,23 @@ sub start_drbd_resource
 			}
 		}
 	}
+	else
+	{
+		# Call an adjust to reset the config, in case something like 'allow-two-primaries' was 
+		# enabled on one node but not the other. 
+		foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{$local_host}{$server}{resource}})
+		{
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0745", variables => { 
+				server   => $server,
+				resource => $resource,
+			}});
+			
+			$anvil->DRBD->manage_resource({
+				resource => $resource, 
+				task     => "adjust",
+			});
+		}
+	}
 	
 	### NOTE: We always check the peer now, in case it's resource is down and ours happens to be up.
 	# See if we're inconsistent and, if so, if we can connect our peers.
@@ -1479,6 +1497,7 @@ sub migrate_server
 		
 		if (not $server_host)
 		{
+			# The server wasn't found on this host
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0344", variables => { server => $server }});
 			$anvil->nice_exit({exit_code => 1});
 		}
@@ -1511,6 +1530,7 @@ sub migrate_server
 			{
 				# Try to bring the resource up on the peer now.
 				$anvil->DRBD->manage_resource({
+					debug    => 2,
 					resource => $resource, 
 					task     => "up",
 					target   => $target, 

scancore-agents/scan-apc-pdu/scan-apc-pdu

@@ -169,6 +169,18 @@ if ($problem)
 	$anvil->nice_exit({exit_code => 1});
 }
 
+# Purge can run from anywhere
+if ($anvil->data->{switches}{purge})
+{
+	# This can be called when doing bulk-database purges.
+	my $schema_file = $anvil->data->{path}{directories}{scan_agents}."/".$THIS_FILE."/".$THIS_FILE.".sql";
+	$anvil->Database->purge_data({
+		debug  => 2,
+		tables => $anvil->Database->get_tables_from_schema({schema_file => $schema_file}),
+	});
+	$anvil->nice_exit({exit_code => 0});
+}
+
 # The APC PDUs only allow one connection at a time. As such, we only run on the striker that is also the
 # active DB.
 my $host_uuid = $anvil->Get->host_uuid();
@@ -184,17 +196,6 @@ if ((not $anvil->data->{switches}{force}) && ($anvil->data->{sys}{database}{acti
 	$anvil->nice_exit({exit_code => 0});
 }
 
-if ($anvil->data->{switches}{purge})
-{
-	# This can be called when doing bulk-database purges.
-	my $schema_file = $anvil->data->{path}{directories}{scan_agents}."/".$THIS_FILE."/".$THIS_FILE.".sql";
-	$anvil->Database->purge_data({
-		debug  => 2,
-		tables => $anvil->Database->get_tables_from_schema({schema_file => $schema_file}),
-	});
-	$anvil->nice_exit({exit_code => 0});
-}
-
 # Find the PDUs. The number of PDUs found is returned. If 0, we exit
 if (not find_pdus($anvil))
 {

scancore-agents/scan-ipmitool/scan-ipmitool

@@ -99,8 +99,6 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 }
 
 my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
 
 # Make sure we're running as 'root'
 # $< == real UID, $> == effective UID

share/words.xml

@@ -607,6 +607,8 @@ The error was:
 ========
 #!variable!output!#
 ========</key>
+		<key name="error_0421">Failed to enable dual-primary for the resource: [#!variable!resource!#]! The peer: [#!variable!peer_name!#]'s connection state is: [#!variable!connection_state!#] (must be 'connected').</key>
+		<key name="error_0422">Failed to enable dual-primary for the server: [#!variable!server!#]! The call to enable dual-primary, necessary for the live migration, returned a non-zero return code: [#!variable!return_code!#].</key>
 
 		<!-- Files templates -->
 		<!-- NOTE: Translating these files requires an understanding of which lines are translatable -->
@@ -2409,6 +2411,8 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
 		<key name="log_0742">The job with the command: [#!variable!command!#] and job UUID: [#!variable!job_uuid!#] is restarting.</key>
 		<key name="log_0743">Will run without connecting to the databases. Some features will be unavailable.</key>
 		<key name="log_0744">A cached request to reboot this host was found (likely from a --no-db update). Registering a job to reboot now!</key>
+		<key name="log_0745">Adjusting the resource: [#!variable!resource!#] to ensure it's compatible with the peer's config prior to connection.</key>
+		<key name="log_0746">The local resource: [#!variable!resource!#] is StandAlone, attempting to connect.</key>
 
 		<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
 		<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>

tools/Makefile.am

@@ -28,6 +28,7 @@ dist_sbin_SCRIPTS	= \
 			  anvil-migrate-server \
 			  anvil-network-profiler \
 			  anvil-parse-fence-agents \
+			  anvil-pcs-wrapper \
 			  anvil-provision-server \
 			  anvil-rename-server \
 			  anvil-report-usage \

tools/anvil-cycle-vm-nics

@@ -84,6 +84,10 @@ sub cycle_nics
 				{
 					$say_network = "Internet-Facing Network ".$network_number." - Bridge ".$bridge_number;
 				}
+				elsif ($bridge_type eq "mn")
+				{
+					$say_network = "Migration Network ".$network_number." - Bridge ".$bridge_number;
+				}
 				
 				print __LINE__."; [ Debug ]   - bridge_number: [".$bridge_number."]\n" if $debug >= 2;
 				foreach my $this_vnet (sort {$a cmp $b} keys %{$nics->{$bridge_type}{$network_number}{$bridge_number}})

tools/anvil-download-file

@@ -43,6 +43,15 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 
 my $anvil = Anvil::Tools->new();
 
+$anvil->Get->switches({list => [
+	"abort",
+	"job-uuid",
+	"overwrite",
+	"save-to",
+	"script",
+	"url",], man => $THIS_FILE});
+$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
+
 $anvil->Database->connect;
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
 if (not $anvil->data->{sys}{database}{connections})
@@ -52,22 +61,6 @@ if (not $anvil->data->{sys}{database}{connections})
 	$anvil->nice_exit({exit_code => 1});
 }
 
-$anvil->data->{switches}{abort}      = "";
-$anvil->data->{switches}{'job-uuid'} = "";
-$anvil->data->{switches}{overwrite}  = ""; 
-$anvil->data->{switches}{'save-to'}  = ""; 	# /mnt/shared/files by default
-$anvil->data->{switches}{script}     = "";
-$anvil->data->{switches}{url}        = ""; 
-$anvil->Get->switches;
-$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-	'switches::abort'     => $anvil->data->{switches}{abort}, 
-	'switches::job-uuid'  => $anvil->data->{switches}{'job-uuid'}, 
-	'switches::overwrite' => $anvil->data->{switches}{overwrite}, 
-	'switches::save-to'   => $anvil->data->{switches}{'save-to'}, 
-	'switches::script'    => $anvil->data->{switches}{script}, 
-	'switches::url'       => $anvil->data->{switches}{url}, 
-}});
-
 # If I don't have --abort or --url, see if there is a job waiting
 if ((not $anvil->data->{switches}{abort}) && (not $anvil->data->{switches}{url}))
 {
@@ -195,7 +188,7 @@ sub download_file
 	my $failed        =  0;
 	my $url           =  $anvil->data->{switches}{url};
 	my $file_name     =  ($url =~ /^.*\/(.*)$/)[0];
-	my $temp_file     =  $anvil->data->{path}{directories}{shared}{temp}."/".$file_name;
+	my $temp_file     =  $anvil->data->{path}{directories}{shared}{incoming}."/".$file_name;
 	my $save_to       =  $anvil->data->{switches}{'save-to'} ? $anvil->data->{switches}{'save-to'} : $anvil->data->{path}{directories}{shared}{files};
 	my $out_file      =  $save_to."/".$file_name;
 	   $save_to       =~ s/\/\///g;
@@ -384,7 +377,7 @@ sub download_file
 					average_rate     => $average_rate, 
 				}});
 				
-				my $line        =  "bytes_downloaded=$bytes_downloaded percent=$percent current_rate=$byte_rate average_rate=$average_rate seconds_running=$running_time seconds_left=$seconds_left url=$url out_file=$out_file"; 
+				my $line        =  "bytes_downloaded=".$bytes_downloaded." percent=".$percent." current_rate=".$byte_rate." average_rate=".$average_rate." seconds_running=".$running_time." seconds_left=".$seconds_left." url=".$url." out_file=".$out_file; 
 				   $next_report += $report_interval;
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 					line        => $line,

tools/anvil-file-details

@@ -29,13 +29,10 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 
 my $anvil = Anvil::Tools->new();
 
-$anvil->data->{switches}{file}          = "";
+$anvil->Get->switches({list => [
-$anvil->data->{switches}{'with-md5sum'} = "";
+	"file", 
-$anvil->Get->switches;
+	"with-md5sum"], man => $THIS_FILE});
-$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
-	"switches::file"        => $anvil->data->{switches}{file},
-	"switches::with-md5sum" => $anvil->data->{switches}{'with-md5sum'},
-}});
 
 my $file = $anvil->data->{switches}{file};
 if (not $file)
@@ -84,7 +81,5 @@ mtime: [".$mtime."]
 md5sum: [".$md5sum."]\n";
 }
 
-
-
 # We're done
 $anvil->nice_exit({exit_code => 0});

tools/anvil-manage-server

@@ -29,11 +29,6 @@ $| = 1;
 
 my $anvil = Anvil::Tools->new();
 
-### TODO: Remove this before final release
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
-##########################################
-
 # Read switches (target ([user@]host[:port]) and the file with the target's password.
 $anvil->Get->switches({list => ["anvil", "boot", "cores", "drive", "eject", "expand-to", "insert", "ram", "server", "y"], man => $THIS_FILE});
 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});

tools/anvil-manage-server-storage

@@ -39,11 +39,6 @@ $| = 1;
 
 my $anvil = Anvil::Tools->new();
 
-### TODO: Remove this before final release
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
-##########################################
-
 # Read switches (target ([user@]host[:port]) and the file with the target's password.
 $anvil->Get->switches({list => [
 	"add", 

tools/anvil-pcs-wrapper

@@ -0,0 +1,10 @@
+#!/bin/sh -e
+
+(
+	if [ "$1" == "status" ]; then
+		pcs $@
+		exit $?
+	fi
+	flock -e 200
+	pcs $@
+) 200>/tmp/pcs.lock

tools/anvil-safe-start

@@ -44,11 +44,6 @@ $anvil->Get->switches({list => [], man => $THIS_FILE});
 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 
-### TODO: Remove this before final release
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
-##########################################
-
 # Make sure we're running as 'root'
 # $< == real UID, $> == effective UID
 if (($< != 0) && ($> != 0))

tools/anvil-update-system

@@ -37,11 +37,6 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 
 my $anvil = Anvil::Tools->new();
 
-### TODO: Remove this before final release
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
-##########################################
-
 # Read switches (target ([user@]host[:port]) and the file with the target's password.
 $anvil->Get->switches({list => [
 	"clear-cache",

tools/striker-auto-initialize-all

@@ -35,10 +35,6 @@ if (not $anvil->data->{sys}{database}{connections})
 $anvil->data->{switches}{config}     = "";
 $anvil->data->{switches}{'job-uuid'} = "";
 $anvil->Get->switches;
-### TODO: Remove this before final release
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
-##########################################
 
 # Read in the config file
 if ((not $anvil->data->{switches}{config}) or (not -f $anvil->data->{switches}{config}))

tools/striker-prep-database

@@ -35,8 +35,6 @@ $| = 1;
 
 my $anvil = Anvil::Tools->new();
 
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
 $anvil->Get->switches;
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 

tools/striker-purge-target

@@ -213,6 +213,64 @@ else
 	}
 }
 
+# If we're purging an Anvil!, we need to purge servers on that anvil node first.
+if ($anvil->data->{purge}{anvil_uuid})
+{
+	# Get the servers, and walk through those on this Anvil, and then delete their definitions before 
+	# deleting the server.
+	$anvil->Database->get_servers({include_deleted => 1});
+	my $anvil_uuid = $anvil->data->{purge}{anvil_uuid};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
+	
+	foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
+	{
+		my $server_uuid       = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
+		my $server_anvil_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			's1:server_name'       => $server_name,
+			's2:server_uuid'       => $server_uuid, 
+			's3:server_anvil_uuid' => $server_anvil_uuid, 
+		}});
+		
+		my $queries = [];
+		push @{$queries}, "DELETE FROM history.server_definitions WHERE server_definition_server_uuid = ".$anvil->Database->quote($server_uuid).";";
+		push @{$queries}, "DELETE FROM server_definitions WHERE server_definition_server_uuid = ".$anvil->Database->quote($server_uuid).";";
+		push @{$queries}, "DELETE FROM history.servers WHERE server_uuid = ".$anvil->Database->quote($server_uuid).";";
+		push @{$queries}, "DELETE FROM servers WHERE server_uuid = ".$anvil->Database->quote($server_uuid).";";
+		foreach my $query (@{$queries})
+		{
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+		}
+		my $problem = $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
+	}
+	# Refresh
+	$anvil->Database->get_servers();
+	
+	# Unlink any DR hosts linked to this Anvil!.
+	$anvil->Database->get_dr_links({include_deleted => 1});
+	foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}})
+	{
+		my $dr_link_uuid = $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}{$host_name}{dr_link_uuid};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			's1:host_name'    => $host_name,
+			's2:dr_link_uuid' => $dr_link_uuid, 
+		}});
+		
+		my $queries = [];
+		push @{$queries}, "DELETE FROM history.dr_links WHERE dr_link_uuid = ".$anvil->Database->quote($dr_link_uuid).";";
+		push @{$queries}, "DELETE FROM dr_links WHERE dr_link_uuid = ".$anvil->Database->quote($dr_link_uuid).";";
+		foreach my $query (@{$queries})
+		{
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+		}
+		my $problem = $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
+	}
+	# Refresh
+	$anvil->Database->get_dr_links();
+}
+
 # List all database tables in reverse order with X_host_uuid tables
 $anvil->Database->find_host_uuid_columns({debug => 3, search_column => "host_uuid", main_table => "hosts"});
 
@@ -269,21 +327,13 @@ WHERE
 		if (($table eq "anvils") && ($anvil->data->{purge}{anvil_uuid}))
 		{
 			# Storage groups
-			$query = "
+			my $query = "
 SELECT 
-    storage_group_member_uuid 
+    storage_group_uuid 
 FROM 
-    storage_group_members 
+    storage_groups 
 WHERE 
-    storage_group_member_storage_group_uuid = 
+    storage_group_anvil_uuid = ".$anvil->Database->quote($anvil->data->{purge}{anvil_uuid})."
-    (
-        SELECT 
-            storage_group_uuid 
-        FROM 
-            storage_groups 
-        WHERE 
-            storage_group_anvil_uuid = ".$anvil->Database->quote($anvil->data->{purge}{anvil_uuid})."
-    )
 ;";
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
 			my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
@@ -294,16 +344,38 @@ WHERE
 			}});
 			foreach my $row (@{$results})
 			{
-				my $storage_group_member_uuid = $row->[0];
+				my $storage_group_uuid = $row->[0];
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { storage_group_member_uuid => $storage_group_member_uuid }});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { storage_group_uuid => $storage_group_uuid }});
-				
-				my $query = "DELETE FROM history.storage_group_members WHERE storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid).";";
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
-				push @{$queries}, $query;
 				
-				$query = "DELETE FROM storage_group_members WHERE storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid).";";
+				# Delete members
+				my $query = "
+SELECT 
+    storage_group_member_uuid 
+FROM 
+    storage_group_members 
+WHERE 
+    storage_group_member_storage_group_uuid = ".$anvil->Database->quote($storage_group_uuid)."
+;";
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
-				push @{$queries}, $query;
+				my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
+				my $count   = @{$results};
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					results => $results, 
+					count   => $count, 
+				}});
+				foreach my $row (@{$results})
+				{
+					my $storage_group_member_uuid = $row->[0];
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { storage_group_member_uuid => $storage_group_member_uuid }});
+					
+					my $query = "DELETE FROM history.storage_group_members WHERE storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid).";";
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+					push @{$queries}, $query;
+					
+					$query = "DELETE FROM storage_group_members WHERE storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid).";";
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+					push @{$queries}, $query;
+				}
 			}
 		
 			$query = "DELETE FROM history.storage_groups WHERE storage_group_anvil_uuid = ".$anvil->Database->quote($anvil->data->{purge}{anvil_uuid}).";";

tools/striker-update-cluster

@@ -47,11 +47,6 @@ $anvil->Get->switches({list => [
 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 
-### TODO: Remove this before final release
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
-##########################################
-
 # Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks
 # is to setup the database server.
 $anvil->Database->connect();