Merge pull request #333 from ClusterLabs/anvil-tools-dev

Anvil tools dev

Anvil/Tools/Cluster.pm

@@ -185,7 +185,13 @@ sub add_server
 		password        => $anvil->Log->is_secure($password),
 	}});
 	
-	# Verify that the server is here or on the peer. We need to add the command to t
+	# Verify that the server is here or on the peer. Given they could be called at the same time that the
+	# server is being provisioned, we'll wait up to 15 seconds for it to appear.
+	my $waiting    = 1;
+	my $wait_until = time + 15;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { wait_until => $wait_until }});
+	while ($waiting)
+	{
 		$anvil->Server->find({
 			debug  => $debug,
 			server => $server_name, 
@@ -198,10 +204,28 @@ sub add_server
 			server   => $server_name, 
 		});
 		
+		if (exists $anvil->data->{server}{location}{$server_name}{status})
+		{
+			$waiting = 0;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				waiting                                       => $waiting,
+				"server::location::${server_name}::status"    => $anvil->data->{server}{location}{$server_name}{status},
+				"server::location::${server_name}::host_name" => $anvil->data->{server}{location}{$server_name}{host_name},
+			}});
+		}
+		
+		if (($waiting) && (time > $wait_until))
+		{
+			# Stop waiting.
+			$waiting = 0;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { waiting => $waiting }});
+		}
+	}
+	
 	# The host here is the full host name.
 	my $host_name    = $anvil->Get->host_name();
-	my $server_state = $anvil->data->{server}{location}{$server_name}{status};
+	my $server_state = defined $anvil->data->{server}{location}{$server_name}{status}    ? $anvil->data->{server}{location}{$server_name}{status}    : "";
-	my $server_host  = $anvil->data->{server}{location}{$server_name}{host_name};
+	my $server_host  = defined $anvil->data->{server}{location}{$server_name}{host_name} ? $anvil->data->{server}{location}{$server_name}{host_name} : "";
 	my $target_role  = $server_state eq "running" ? "started" : "stopped";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		host_name    => $host_name, 
@@ -4278,10 +4302,14 @@ This tries to recover a C<< FAILED >> resource (server).
 
 Parameters;
 
-=head3 server_ (required)
+=head3 server (required)
 
 This is the server (resource) name to try to recover.
 
+=head3 running (required)
+
+This indicates if the server should be recovered into the running state when set to C<< 1 >>, or stopped state when set to C<< 0 >>.
+
 =cut
 sub recover_server
 {
@@ -4291,8 +4319,10 @@ sub recover_server
 	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Cluster->recover_server()" }});
 	
+	my $running = defined $parameter->{running} ? $parameter->{running} : "";
 	my $server  = defined $parameter->{server}  ? $parameter->{server}  : "";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		running => $running,
 		server  => $server,
 	}});
 	
@@ -4301,8 +4331,15 @@ sub recover_server
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Cluster->recover_server()", parameter => "server" }});
 		return("!!error!!");
 	}
+	if ($running eq "")
+	{
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Cluster->recover_server()", parameter => "running" }});
+		return("!!error!!");
+	}
 	
-	my $shell_call = $anvil->data->{path}{exe}{crm_resource}." --resource ".$server." --refresh";
+	# Set the desired state post recovery.
+	my $wanted_state = $running ? "enable" : "disable";
+	my $shell_call   = $anvil->data->{path}{exe}{pcs}." resource ".$wanted_state." ".$server;
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
 	
 	my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call});
@@ -4311,6 +4348,16 @@ sub recover_server
 		return_code => $return_code, 
 	}});
 	
+	# Now tell it to refresh
+	$shell_call = $anvil->data->{path}{exe}{crm_resource}." --resource ".$server." --refresh";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
+	
+	($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		output      => $output,
+		return_code => $return_code, 
+	}});
+	
 	return(0);
 }
 

Anvil/Tools/Database.pm

@@ -6450,6 +6450,7 @@ SELECT
     power_on_battery, 
     power_seconds_left, 
     power_charge_percentage, 
+    modified_date, 
     round(extract(epoch from modified_date)) 
 FROM 
     power 
@@ -6468,13 +6469,15 @@ FROM
 		my $power_on_battery        = $row->[2];
 		my $power_seconds_left      = $row->[3]; 
 		my $power_charge_percentage = $row->[4];
-		my $modified_date_unix      = $row->[5];
+		my $modified_date           = $row->[5];
+		my $modified_date_unix      = $row->[6];
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			power_uuid              => $power_uuid, 
 			power_ups_uuid          => $power_ups_uuid, 
 			power_on_battery        => $power_on_battery, 
 			power_seconds_left      => $power_seconds_left, 
 			power_charge_percentage => $power_charge_percentage, 
+			modified_date           => $modified_date, 
 			modified_date_unix      => $modified_date_unix, 
 		}});
 		
@@ -6483,12 +6486,14 @@ FROM
 		$anvil->data->{power}{power_uuid}{$power_uuid}{power_on_battery}        = $power_on_battery;
 		$anvil->data->{power}{power_uuid}{$power_uuid}{power_seconds_left}      = $power_seconds_left;
 		$anvil->data->{power}{power_uuid}{$power_uuid}{power_charge_percentage} = $power_charge_percentage;
+		$anvil->data->{power}{power_uuid}{$power_uuid}{modified_date}           = $modified_date;
 		$anvil->data->{power}{power_uuid}{$power_uuid}{modified_date_unix}      = $modified_date_unix;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			"power::power_uuid::${power_uuid}::power_ups_uuid"          => $anvil->data->{power}{power_uuid}{$power_uuid}{power_ups_uuid}, 
 			"power::power_uuid::${power_uuid}::power_on_battery"        => $anvil->data->{power}{power_uuid}{$power_uuid}{power_on_battery}, 
 			"power::power_uuid::${power_uuid}::power_seconds_left"      => $anvil->data->{power}{power_uuid}{$power_uuid}{power_seconds_left}, 
 			"power::power_uuid::${power_uuid}::power_charge_percentage" => $anvil->data->{power}{power_uuid}{$power_uuid}{power_charge_percentage}, 
+			"power::power_uuid::${power_uuid}::modified_date"           => $anvil->data->{power}{power_uuid}{$power_uuid}{modified_date}, 
 			"power::power_uuid::${power_uuid}::modified_date_unix"      => $anvil->data->{power}{power_uuid}{$power_uuid}{modified_date_unix}, 
 		}});
 		
@@ -6496,12 +6501,14 @@ FROM
 		$anvil->data->{power}{power_ups_uuid}{$power_ups_uuid}{power_on_battery}        = $power_on_battery;
 		$anvil->data->{power}{power_ups_uuid}{$power_ups_uuid}{power_seconds_left}      = $power_seconds_left;
 		$anvil->data->{power}{power_ups_uuid}{$power_ups_uuid}{power_charge_percentage} = $power_charge_percentage;
+		$anvil->data->{power}{power_ups_uuid}{$power_ups_uuid}{modified_date}           = $modified_date;
 		$anvil->data->{power}{power_ups_uuid}{$power_ups_uuid}{modified_date_unix}      = $modified_date_unix;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			"power::power_ups_uuid::${power_ups_uuid}::power_uuid"              => $anvil->data->{power}{power_ups_uuid}{$power_ups_uuid}{power_uuid}, 
 			"power::power_ups_uuid::${power_ups_uuid}::power_on_battery"        => $anvil->data->{power}{power_ups_uuid}{$power_ups_uuid}{power_on_battery}, 
 			"power::power_ups_uuid::${power_ups_uuid}::power_seconds_left"      => $anvil->data->{power}{power_ups_uuid}{$power_ups_uuid}{power_seconds_left}, 
 			"power::power_ups_uuid::${power_ups_uuid}::power_charge_percentage" => $anvil->data->{power}{power_ups_uuid}{$power_ups_uuid}{power_charge_percentage}, 
+			"power::power_ups_uuid::${power_ups_uuid}::modified_date"           => $anvil->data->{power}{power_ups_uuid}{$power_ups_uuid}{modified_date}, 
 			"power::power_ups_uuid::${power_ups_uuid}::modified_date_unix"      => $anvil->data->{power}{power_ups_uuid}{$power_ups_uuid}{modified_date_unix}, 
 		}});
 	}
@@ -6568,7 +6575,8 @@ SELECT
     ups_name, 
     ups_agent, 
     ups_ip_address, 
-    modified_date 
+    modified_date, 
+    round(extract(epoch from modified_date)) 
 FROM 
     upses ";
 	if (not $include_deleted)
@@ -6593,12 +6601,14 @@ WHERE
 		my $ups_agent          = $row->[2];
 		my $ups_ip_address     = $row->[3]; 
 		my $modified_date      = $row->[4];
+		my $modified_date_unix = $row->[5];
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			ups_uuid           => $ups_uuid, 
 			ups_name           => $ups_name, 
 			ups_agent          => $ups_agent, 
 			ups_ip_address     => $ups_ip_address, 
 			modified_date      => $modified_date, 
+			modified_date_unix => $modified_date_unix, 
 		}});
 		
 		# Record the data in the hash, too.
@@ -6606,24 +6616,28 @@ WHERE
 		$anvil->data->{upses}{ups_uuid}{$ups_uuid}{ups_agent}          = $ups_agent;
 		$anvil->data->{upses}{ups_uuid}{$ups_uuid}{ups_ip_address}     = $ups_ip_address;
 		$anvil->data->{upses}{ups_uuid}{$ups_uuid}{modified_date}      = $modified_date;
+		$anvil->data->{upses}{ups_uuid}{$ups_uuid}{modified_date_unix} = $modified_date_unix;
 		$anvil->data->{upses}{ups_uuid}{$ups_uuid}{power_uuid}         = "";
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			"upses::ups_uuid::${ups_uuid}::ups_name"           => $anvil->data->{upses}{ups_uuid}{$ups_uuid}{ups_name}, 
 			"upses::ups_uuid::${ups_uuid}::ups_agent"          => $anvil->data->{upses}{ups_uuid}{$ups_uuid}{ups_agent}, 
 			"upses::ups_uuid::${ups_uuid}::ups_ip_address"     => $anvil->data->{upses}{ups_uuid}{$ups_uuid}{ups_ip_address}, 
 			"upses::ups_uuid::${ups_uuid}::modified_date"      => $anvil->data->{upses}{ups_uuid}{$ups_uuid}{modified_date}, 
+			"upses::ups_uuid::${ups_uuid}::modified_date_unix" => $anvil->data->{upses}{ups_uuid}{$ups_uuid}{modified_date_unix}, 
 		}});
 		
 		$anvil->data->{upses}{ups_name}{$ups_name}{ups_uuid}           = $ups_uuid;
 		$anvil->data->{upses}{ups_name}{$ups_name}{ups_agent}          = $ups_agent;
 		$anvil->data->{upses}{ups_name}{$ups_name}{ups_ip_address}     = $ups_ip_address;
 		$anvil->data->{upses}{ups_name}{$ups_name}{modified_date}      = $modified_date;
+		$anvil->data->{upses}{ups_name}{$ups_name}{modified_date_unix} = $modified_date;
 		$anvil->data->{upses}{ups_name}{$ups_name}{power_uuid}         = "";
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			"upses::ups_name::${ups_name}::ups_uuid"           => $anvil->data->{upses}{ups_name}{$ups_name}{ups_uuid}, 
 			"upses::ups_name::${ups_name}::ups_agent"          => $anvil->data->{upses}{ups_name}{$ups_name}{ups_agent}, 
 			"upses::ups_name::${ups_name}::ups_ip_address"     => $anvil->data->{upses}{ups_name}{$ups_name}{ups_ip_address}, 
 			"upses::ups_name::${ups_name}::modified_date"      => $anvil->data->{upses}{ups_name}{$ups_name}{modified_date}, 
+			"upses::ups_name::${ups_name}::modified_date_unix" => $anvil->data->{upses}{ups_name}{$ups_name}{modified_date_unix}, 
 		}});
 		
 		# Collect power information from 'power'.

Anvil/Tools/Server.pm

@@ -416,6 +416,7 @@ sub find
 	# Clear any old data
 	if ((exists $anvil->data->{server}{location}) && ($refresh))
 	{
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0700", variables => { hash => "server::location" }});
 		delete $anvil->data->{server}{location};
 	}
 	

man/anvil-delete-server.8

@@ -1,13 +1,13 @@
 .\" Manpage for the Anvil! server removal tool
 .\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
-.TH anvil-delete-server "8" "August 02 2022" "Anvil! Intelligent Availability™ Platform"
+.TH anvil-delete-server "8" "June 06 2023" "Anvil! Intelligent Availability™ Platform"
 .SH NAME
 anvil-delete-server \- This program deletes a server from an Anvil! sub-cluster.
 .SH SYNOPSIS
 .B anvil-delete-server 
 \fI\,<command> \/\fR[\fI\,options\/\fR]
 .SH DESCRIPTION
-This deletes a server, running or stopped, from an Anvil! subcluster. The resource allocated to this server are removed and returned to the resource pool as well.
+This deletes a server, running or stopped, from an Anvil! sub-cluster. The resource allocated to this server are removed and returned to the resource pool as well.
 .TP
 This action is permanent!
 .TP
@@ -23,9 +23,17 @@ When logging, record sensitive data, like passwords.
 Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
 .SS "Commands:"
 .TP
-\fB\-\-job-uuid\fR <name>
+\fB\-\-job-uuid\fR <UUID>
 The program is normally run as a job, with data on how to configure the host defined in the job. This switch allows the running of a specific job. If this is not set, the program will search for a job that has not yet been picked up by another process. If found, that job UUID is used automatically.
 .IP
+.SS "Commands:"
+.TP
+\fB\-\-force\fR
+When used with \fB\-\-server\fR, no confirmation prompt will be shown. Use this with caution!
+.TP
+\fB\-\-server\fR <name or UUID>
+This is the server to be deleted. It can either be the name or UUID. If not provided, a list of servers on the Anvil! node this command is run from will be shown where you can select the one to delete.
+.TP
 .SH AUTHOR
 Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
 .SH "REPORTING BUGS"

ocf/alteeve/server

@@ -100,8 +100,7 @@ $| = 1;
 # NOTE: Setting 'log_level' and 'log_secure' here will get overridden in the main lopp. Use the Log methods 
 #       in the loop as well to override defaults in code.
 my $anvil = Anvil::Tools->new();
-
+#$anvil->Log->level({set => 2});
-$anvil->Log->level({set => 2});
 
 ### Read or Set the environment variables
 # This is the name of the server we're managing.                # Example values:

scancore-agents/scan-cluster/scan-cluster

@@ -201,6 +201,7 @@ sub check_resources
 				$anvil->Cluster->recover_server({
 					debug   => 2,
 					server  => $server,
+					running => $server_found,
 				});
 				
 				# It'll leave 'failed state' for a bit, so we need to wait.

share/words.xml

@@ -1019,6 +1019,10 @@ resource #!variable!server!# {
 		<key name="header_0104">Host UUID</key>
 		<key name="header_0105">Machines</key>
 		<key name="header_0106">MN link #!variable!number!#</key>
+		<key name="header_0107">Charge %</key>
+		<key name="header_0108">On Battery</key>
+		<key name="header_0109">Estimated Runtime</key>
+		<key name="header_0110">Last Updated</key>
 
 		<!-- Strings used by jobs -->
 		<key name="job_0001">Configure Network</key>
@@ -2384,6 +2388,7 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
 		<key name="log_0735">The DR host: [#!variable!host!#] as been _unlinked_ to the Anvil! node: [#!variable!anvil!#].</key>
 		<key name="log_0736">The DR host: [#!variable!host!#] was not linked to the Anvil! node: [#!variable!anvil!#], nothing to do.</key>
 		<key name="log_0737">The job: [#!variable!command!#] (with job UUID: [#!variable!job_uuid!#]) is being skipped for now, already started a job (started job_uuid: [#!variable!started_job!#]) with this command on this loop.</key>
+		<key name="log_0738">There are no databases available at this time.</key>
 
 		<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
 		<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>

tools/anvil-delete-server

@@ -29,7 +29,10 @@ my $anvil = Anvil::Tools->new();
 
 # Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is 
 # passed directly, it will be used. Otherwise, the password will be read from the database.
-$anvil->Get->switches({list => ["job-uuid"], man => $THIS_FILE});
+$anvil->Get->switches({list => [
+	"force",
+	"server",
+	"job-uuid"], man => $THIS_FILE});
 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
 
@@ -91,6 +94,11 @@ if ($anvil->data->{switches}{'job-uuid'})
 	# Job data will be in $anvil->data->{jobs}{job_data}
 	run_jobs($anvil);
 }
+elsif ($anvil->data->{switches}{server})
+{
+	# User specified what they want deleted.
+	confirm_server_delete($anvil);
+}
 else
 {
 	if (not $anvil->data->{sys}{anvil_uuid})
@@ -111,7 +119,6 @@ $anvil->nice_exit({exit_code => 0});
 # Functions                                                                                                 #
 #############################################################################################################
 
-
 # This actually provisions a VM.
 sub run_jobs
 {
@@ -496,6 +503,38 @@ sub parse_job_data
 	return(0);
 }
 
+# This allows a user to specify the server they want deleted without going through the meny system.
+sub confirm_server_delete
+{
+	my ($anvil) = @_;
+	
+	$anvil->Get->server_from_switch({
+		debug      => 2,
+		server     => $anvil->data->{switches}{server}, 
+		anvil_uuid => $anvil->data->{sys}{anvil_uuid},
+	});
+	
+	if (not $anvil->data->{switches}{server_name})
+	{
+		# Not found.
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0277", variables => { server => $anvil->data->{switches}{server} }});
+		$anvil->nice_exit({exit_code => 1});
+		
+	}
+	
+	# Ask the user to confirm, if needed. Note that this method requires '--force', not '-y' or '--Yes'. 
+	if (not $anvil->data->{switches}{force})
+	{
+		get_confirmation($anvil);
+	}
+	else
+	{
+		# They're forcing, save.
+		save_job($anvil);
+	}
+	
+	return(0);
+}
 
 # This will ask the user to select a server.
 sub ask_for_server
@@ -588,7 +627,23 @@ sub ask_for_server
 		}
 	}
 	
+	$anvil->Get->server_from_switch({
+		debug      => 2,
+		server     => $delete_uuid, 
+		anvil_uuid => $anvil->data->{sys}{anvil_uuid},
+	});
+	
 	# Ask the user to confirm.
+	get_confirmation($anvil);
+
+	return(0);
+}
+
+sub get_confirmation
+{
+	my ($anvil) = @_;
+	
+	my $delete_uuid = $anvil->data->{switches}{server_uuid};
 	my $server_name = $anvil->data->{servers}{server_uuid}{$delete_uuid}{server_name};
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 		delete_uuid => $delete_uuid,
@@ -600,15 +655,35 @@ sub ask_for_server
 	chomp $answer;
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
 	
-	if (lc($answer) eq "yes")
+	if ($answer eq "Yes")
 	{
 		### Save the job!
+		save_job($anvil);
+	}
+	else
+	{
+		# Abort.
+		print $anvil->Words->string({key => "message_0022"})."\n";
+		$anvil->nice_exit({exit_code => 0});
+	}
+	
+	return(0);
+}
+
+sub save_job
+{
+	my ($anvil) = @_;
+	
 	# Is the server running?
 	print $anvil->Words->string({key => "message_0213"})."\n";
 	
 	$anvil->Database->get_anvils();
 	my $hosts       = [];
+	my $anvil_uuid  = $anvil->data->{sys}{anvil_uuid};;
 	my $password    = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
+	my $server_name = $anvil->data->{switches}{server_name};
+	my $server_uuid = $anvil->data->{switches}{server_uuid};
+	my $delete_uuid = $server_uuid;
 	push @{$hosts}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
 	push @{$hosts}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
 	if ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid})
@@ -691,13 +766,6 @@ sub ask_for_server
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
 	
 	$anvil->nice_exit({exit_code => 0});
-	}
-	else
-	{
-		# Abort.
-		print $anvil->Words->string({key => "message_0022"})."\n";
-		$anvil->nice_exit({exit_code => 0});
-	}
 	
 	return(0);
 }

tools/anvil-provision-server

@@ -3463,6 +3463,8 @@ server_uuid=".$anvil->data->{new_server}{uuid};
 		$target_host_uuid = $node1_host_uuid;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_uuid => $target_host_uuid }});
 	}
+	my $host_name = $anvil->Get->host_name_from_uuid({host_uuid => $target_host_uuid});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }});
 	my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
 		debug           => 2, 
 		job_command     => $anvil->data->{path}{exe}{'anvil-provision-server'}.$anvil->Log->switches,