Merge pull request #183 from ClusterLabs/anvil-tools-dev

Fixed a race condition where configuring the IPMI BMC would appear to…

Anvil/Tools/Database.pm

@@ -7407,7 +7407,7 @@ WHERE
 		$anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
 	}
 	
-	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0126", variables => { method => "Database->insert_or_update_hosts()" }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }});
 	return($host_uuid);
 }
 
@@ -15388,6 +15388,7 @@ sub update_host_status
 	# We're only updating the status, so we'll read in the current data to pass back in.
 	$anvil->Database->get_hosts({debug => $debug});
 	$anvil->Database->insert_or_update_hosts({
+		debug       => $debug, 
 		host_ipmi   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}, 
 		host_key    => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, 
 		host_name   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, 

Anvil/Tools/System.pm

@@ -2063,9 +2063,14 @@ LIMIT 1
 		}
 	}
 	
+	# This can take a while to come up after cold resetting a BMC. So we'll try for a minute.
 	my $user_name   = "";
 	my $user_number = "";
-	($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel});
+	my $waiting     = 1;
+	my $wait_until  = time + 120;
+	while ($waiting)
+	{
+		my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			output      => $output, 
 			return_code => $return_code,
@@ -2096,9 +2101,11 @@ LIMIT 1
 				if (($channel_priv eq "oem") or ($channel_priv eq "administrator"))
 				{
 					# Found the user.
+					$waiting     = 0;
 					$user_name   = $this_user_name;
 					$user_number = $this_user_number;
 					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						waiting     => $waiting, 
 						user_name   => $user_name,
 						user_number => $user_number, 
 					}});
@@ -2106,17 +2113,30 @@ LIMIT 1
 				}
 			}
 		}
-	if (not $user_name)
+		
+		# Try again later or give up?
+		if (time > $wait_until)
 		{
-		# Failed to find a user.
+			$waiting = 0;
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0507", variables => {
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "error_0331", variables => {
 				shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel,
 				output     => $output,
 			}});
+		}
+		else
+		{
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0129", variables => {
+				shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel,
+				output     => $output,
+			}});
+			sleep 10;
+		}
+	}
+	if (not $user_name)
+	{
+		# Failed to find a user.
 		return(0);
 	}
-	$output      = "";
-	$return_code = "";
 	
 	# Now ask the Striker running the database we're using to try to call the IPMI BMC.
 	my $striker_host_uuid = $anvil->data->{sys}{database}{read_uuid};
@@ -2152,6 +2172,7 @@ LIMIT 1
 		# Update the database, in case needed.
 		my $host_uuid = $anvil->Get->host_uuid();
 		$anvil->Database->insert_or_update_hosts({
+			debug       => $debug, 
 			host_ipmi   => $host_ipmi, 
 			host_key    => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, 
 			host_name   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, 
@@ -2185,6 +2206,7 @@ LIMIT 1
 			# Update the database, in case needed.
 			my $host_uuid = $anvil->Get->host_uuid();
 			$anvil->Database->insert_or_update_hosts({
+				debug       => $debug, 
 				host_ipmi   => $host_ipmi, 
 				host_key    => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, 
 				host_name   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, 
@@ -2296,6 +2318,7 @@ LIMIT 1
 			# Update the database, in case needed.
 			my $host_uuid = $anvil->Get->host_uuid();
 			$anvil->Database->insert_or_update_hosts({
+				debug       => $debug, 
 				host_ipmi   => $host_ipmi, 
 				host_key    => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, 
 				host_name   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, 
@@ -2326,6 +2349,7 @@ LIMIT 1
 				# Update the database, in case needed.
 				my $host_uuid = $anvil->Get->host_uuid();
 				$anvil->Database->insert_or_update_hosts({
+					debug       => $debug, 
 					host_ipmi   => $host_ipmi, 
 					host_key    => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, 
 					host_name   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, 

share/words.xml

@@ -447,6 +447,13 @@ Failed to parse the XML in the new definition file. The error was:
 		<key name="error_0328"><![CDATA[Failed to parse the assigned memory from the definition file. Expected the memory defined as '<memory unit='XiB'>Y</memory>'.]]></key>
 		<key name="error_0329">The server UUID: [#!variable!server_uuid!#] in the definition file wasn't found in the database, unable to update.</key>
 		<key name="error_0330">The new definition has changed the server's name from: [#!variable!current_name!#] to: [#!variable!new_name!#]. Changing the server's name must be done with the 'anvil-rename-server' tool.</key>
+		<key name="error_0331">[ Error ] - The IPMI BMC administrator (oem) user was not found. The output (if any) of the call: [#!variable!shell_call!#] was:
+====
+#!variable!output!#
+====
+
+Giving up.
+		</key>
 		
 		<!-- Files templates -->
 		<!-- NOTE: Translating these files requires an understanding of which lines are translatable -->
@@ -2900,6 +2907,13 @@ The error was:
 #!variable!error!#
 ========
 		</key>
+		<key name="warning_0129">[ Warning ] - The IPMI BMC administrator (oem) user was not found. The output (if any) of the call: [#!variable!shell_call!#] was:
+====
+#!variable!output!#
+====
+
+We will sleep a bit and try again.
+		</key>
 
 		<!-- The entries below here are not sequential, but use a key to find the entry. -->
 		<!-- Run 'striker-parse-os-list to find new entries. --> 

tools/anvil-join-anvil

@@ -2195,7 +2195,7 @@ sub check_local_network
 	# Setup IPMI, if needed.
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0114"});
 	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0114");
-	$anvil->System->configure_ipmi({debug => 3, manifest_uuid => $manifest_uuid});
+	$anvil->System->configure_ipmi({debug => 2, manifest_uuid => $manifest_uuid});
 	
 	return(0);
 }

tools/scancore

@@ -360,6 +360,7 @@ sub startup_tasks
 	$anvil->Database->get_hosts({debug => 3});
 	my $host_uuid = $anvil->Get->host_uuid();
 	$anvil->Database->insert_or_update_hosts({
+		debug       => 2,
 		host_ipmi   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}, 
 		host_key    => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, 
 		host_name   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name},