Merge pull request #153 from ClusterLabs/scancore-debugging

Scancore debugging

Anvil/Tools/Cluster.pm

@@ -2799,6 +2799,7 @@ sub migrate_server
 	if ($node)
 	{
 		$anvil->Cluster->_set_server_constraint({
+			debug          => 2,
 			server         => $server,
 			preferred_node => $node,
 		});

Anvil/Tools/DRBD.pm

@@ -108,6 +108,10 @@ If C<< target >> is set, this will be the user we connect to the remote machine
 
 This is the name of the resource to enable two primaries on.
 
+=head3 set_to (optional, default 'yes')
+
+This can be set to C<< yes >> to allow two-primary, or C<< no >> to disable it. 
+
 =head3 target (optional)
 
 This is the IP or host name of the machine to read the version of. If this is not set, the local system's version is checked.
@@ -129,12 +133,14 @@ sub allow_two_primaries
 	my $port           = defined $parameter->{port}           ? $parameter->{port}           : "";
 	my $remote_user    = defined $parameter->{remote_user}    ? $parameter->{remote_user}    : "root";
 	my $resource       = defined $parameter->{resource}       ? $parameter->{resource}       : "";
+	my $set_to         = defined $parameter->{set_to}         ? $parameter->{set_to}         : "yes";
 	my $target         = defined $parameter->{target}         ? $parameter->{target}         : "";
 	my $target_node_id = defined $parameter->{target_node_id} ? $parameter->{target_node_id} : "";
 	my $return_code    = 255; 
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		password       => $anvil->Log->is_secure($password),
 		port           => $port, 
+		set_to         => $set_to, 
 		remote_user    => $remote_user,
 		resource       => $resource, 
 		target         => $target, 
@@ -147,6 +153,12 @@ sub allow_two_primaries
 		return($return_code);
 	}
 	
+	if (($set_to ne "yes") && ($set_to ne "no"))
+	{
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0312", variables => { set_to => $set_to }});
+		return($return_code);
+	}
+	
 	# Do we need to scan devices?
 	my $host = $anvil->Get->short_host_name;
 	if (not $anvil->data->{drbd}{config}{$host}{peer})
@@ -188,13 +200,14 @@ sub allow_two_primaries
 		}
 	}
 	
-	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 1, key => "log_0350", variables => { 
+	my $key = $set_to eq "yes" ? "log_0350" : "log_0642";
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 1, key => "", variables => { 
 		resource       => $resource,
 		target_name    => $peer_name, 
 		target_node_id => $target_node_id, 
 	}});
 	
-	my $shell_call = $anvil->data->{path}{exe}{drbdsetup}." net-options ".$resource." ".$target_node_id." --allow-two-primaries=yes";
+	my $shell_call = $anvil->data->{path}{exe}{drbdsetup}." net-options ".$resource." ".$target_node_id." --allow-two-primaries=".$set_to;
 	my $output     = "";
 	if ($anvil->Network->is_local({host => $target}))
 	{
@@ -1876,7 +1889,7 @@ sub reload_defaults
 	
 	if (not $resource)
 	{
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "DRBD->allow_two_primaries()", parameter => "resource" }});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "DRBD->reload_defaults()", parameter => "resource" }});
 		return($return_code);
 	}
 	

Anvil/Tools/ScanCore.pm

@@ -371,7 +371,7 @@ WHERE
 		my $health_agent_name    = $row->[0];
 		my $health_source_name   = $row->[1];
 		my $health_source_weight = $row->[2];
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			's1:health_agent_name'    => $health_agent_name, 
 			's2:health_source_name'   => $health_source_name, 
 			's3:health_source_weight' => $health_source_weight, 
@@ -380,7 +380,7 @@ WHERE
 		if ($health_source_weight)
 		{
 			$health_score += $health_source_weight;
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { health_score => $health_score }});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { health_score => $health_score }});
 		}
 	}
 	
@@ -1294,7 +1294,7 @@ sub post_scan_analysis_node
 		debug     => $debug, 
 		host_uuid => $anvil->data->{cib}{parsed}{peer}{host_uuid},
 	});
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 		local_health => $local_health,
 		peer_health  => $peer_health, 
 	}});

Anvil/Tools/Server.pm

@@ -974,13 +974,18 @@ sub migrate_virsh
 	}
 	
 	# Enable dual-primary for any resources we know about for this server.
+	my $resources_to_disable_dual_primary = [];
 	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{$source}{$server}{resource}})
 	{
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { resource => $resource }});
 		my ($return_code) = $anvil->DRBD->allow_two_primaries({
-			debug    => $debug, 
+			debug    => 2, 
 			resource => $resource, 
+			set_to   => "yes",
 		});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { return_code => $return_code }});
+		
+		push @{$resources_to_disable_dual_primary}, $resource;
 	}
 	
 	### NOTE: This method is called by ocf:alteeve:server, which operates without database access. As 
@@ -1189,13 +1194,20 @@ WHERE
 	}
 	
 	# Switch off dual-primary.
-	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{$target}{$server}{resource}})
+	foreach my $resource (sort {$a cmp $b} @{$resources_to_disable_dual_primary})
 	{
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { resource => $resource }});
-		$anvil->DRBD->reload_defaults({
+		my ($return_code) = $anvil->DRBD->allow_two_primaries({
-			debug    => $debug, 
+			debug    => 2, 
 			resource => $resource, 
+			set_to   => "no", 
 		});
+		
+# 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { resource => $resource }});
+# 		$anvil->DRBD->reload_defaults({
+# 			debug    => 2, 
+# 			resource => $resource, 
+# 		});
 	}
 	
 	return($success);

ocf/alteeve/server

@@ -135,7 +135,7 @@ foreach my $key (sort {$a cmp $b} keys %ENV)
 # If pacemaker is in debug, so are we,
 if ($anvil->data->{environment}{PCMK_debug})
 {
-	$anvil->Log->level({set => 3});
+	$anvil->Log->level({set => 2});
 }
 
 # Originally, this was designed to start and stop a server's DRBD resources on demand. Early testing appears
@@ -1225,7 +1225,6 @@ pmsuspended - The domain has been suspended by guest power management, e.g. ente
 sub migrate_server
 {
 	my ($anvil) = @_;
-
 	
 	### NOTE: For now, we're not going to block if the target is not UpToDate. There are times when a
 	###       user might want to do this (ie: sync will be done soon and the need to evacuate the node
@@ -1723,7 +1722,6 @@ sub validate_storage_drbd
 			drbd_device => $drbd_device,
 			lv_path     => $on_lv, 
 		}});
-		
 	}
 	
 	### NOTE: Checking/Managing firewall ports is expensive option, so DRBD ports are permanently opened 

share/words.xml

@@ -423,6 +423,7 @@ The attempt to start the servers appears to have failed. The return code '0' was
 		<key name="error_0309">I tried to change the fencing preferred node to: [#!variable!prefer!#], but it doesn't appear to have worked. The preferred node is: [#!variable!current!#] ('--' means there is no preferred node)</key>
 		<key name="error_0310">I tried to remove the fence delay from the node: [#!variable!node!#], but it doesn't appear to have worked. The preferred node is: [#!variable!current!#] ('--' means there is no preferred node)</key>
 		<key name="error_0311">Failed to find the UUID column for the table: [#!variable!table!#].</key>
+		<key name="error_0312">The 'set_to' parameter: [#!variable!set_to!#] is invalid. It must be 'yes' or 'no'.</key>
 		
 		<!-- Files templates -->
 		<!-- NOTE: Translating these files requires an understanding of which lines are translatable -->
@@ -1843,6 +1844,7 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
 		<key name="log_0639">The Anvil! daemon is in startup mode, and the job: [#!variable!job_uuid!#], command: [#!variable!job_command!#] is not a startup job, ignoring it for now.</key>
 		<key name="log_0640">Out peer is online, no need to check server location constraints.</key>
 		<key name="log_0641">The server: [#!variable!server!#] has a location constraint that preferres our peer, but our peer is offline. Updating the location constraint to prefer this node.</key>
+		<key name="log_0642">Disabling dual primary for the resource: [#!variable!resource!#] to the node: [#!variable!target_name!# (#!variable!target_node_id!#)].</key>
 		
 		<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
 		<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>

tools/anvil-configure-host

@@ -112,6 +112,19 @@ sub do_reboot
 {
 	my ($anvil) = @_;
 	
+	# Mark that a reboot is needed, in case something kills us before we actually reboot.
+	my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
+		file            => $THIS_FILE, 
+		line            => __LINE__, 
+		job_command     => $anvil->data->{path}{exe}{'anvil-manage-power'}." --reboot -y".$anvil->Log->switches, 
+		job_data        => "", 
+		job_name        => "reboot::system", 
+		job_title       => "job_0009", 
+		job_description => "job_0006", 
+		job_progress    => 0,
+	});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
+	
 	my $time_left = 60;
 	while ($time_left)
 	{

tools/anvil-manage-power

@@ -251,7 +251,7 @@ sub do_poweroff
 	$anvil->Database->update_host_status({
 		debug       => 2,
 		host_uuid   => $anvil->Get->host_uuid,
-		host_status => $task eq "poweroff" ? "rebooting" : "stopping";,
+		host_status => $task eq "poweroff" ? "rebooting" : "stopping",
 	});
 	
 	# Now do the deed.

tools/anvil-update-states

@@ -2,6 +2,9 @@
 # 
 # This updates things like the current network configuration, shared file data and writes it out to a json file.
 # 
+# TODO:
+# - Retire this once scan-network is finished. 
+# 
 use strict;
 use warnings;
 use Anvil::Tools;
@@ -25,7 +28,7 @@ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure =
 # If there's no DB (or cached data isn't recorded to the database yet), this will store those records.
 $anvil->data->{cache}{new_file} = "# interface,timestamp,mac_address,speed,link_state,operational\n";
 
-$anvil->Database->connect({debug => 3});
+$anvil->Database->connect();
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
 if (not $anvil->data->{sys}{database}{connections})
 {