Fixed a race condition where configuring the IPMI BMC would appear to fail because the BMC wouldn't report the user list after a cold reset.

Signed-off-by: Digimer <digimer@alteeve.ca>

Anvil/Tools/Database.pm

@@ -7407,7 +7407,7 @@ WHERE
 		$anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
 	}
 	
-	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0126", variables => { method => "Database->insert_or_update_hosts()" }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }});
 	return($host_uuid);
 }
 
@@ -15388,6 +15388,7 @@ sub update_host_status
 	# We're only updating the status, so we'll read in the current data to pass back in.
 	$anvil->Database->get_hosts({debug => $debug});
 	$anvil->Database->insert_or_update_hosts({
+		debug       => $debug, 
 		host_ipmi   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}, 
 		host_key    => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, 
 		host_name   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, 

Anvil/Tools/System.pm

@@ -2063,60 +2063,80 @@ LIMIT 1
 		}
 	}
 	
+	# This can take a while to come up after cold resetting a BMC. So we'll try for a minute.
 	my $user_name   = "";
 	my $user_number = "";
-	($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel});
+	my $waiting     = 1;
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+	my $wait_until  = time + 120;
-		output      => $output, 
+	while ($waiting)
-		return_code => $return_code,
-	}});
-	foreach my $line (split/\n/, $output)
 	{
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
+		my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel});
-		
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-		next if $line =~ /Empty User/i;
+			output      => $output, 
-		next if $line =~ /NO ACCESS/i;
+			return_code => $return_code,
-		next if $line =~ /Unknown/i;
+		}});
-		if ($line =~ /^(\d+)\s+(.*?)\s+(\w+)\s+(\w+)\s+(\w+)\s+(.*)$/)
+		foreach my $line (split/\n/, $output)
 		{
-			my $this_user_number = $1;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
-			my $this_user_name   = $2;
+			
-			my $callin           = $3;
+			next if $line =~ /Empty User/i;
-			my $link_auth        = $4;
+			next if $line =~ /NO ACCESS/i;
-			my $ipmi_message     = $5;
+			next if $line =~ /Unknown/i;
-			my $channel_priv     = lc($6);
+			if ($line =~ /^(\d+)\s+(.*?)\s+(\w+)\s+(\w+)\s+(\w+)\s+(.*)$/)
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-				this_user_number => $this_user_number,
-				this_user_name   => $this_user_name, 
-				callin           => $callin, 
-				link_auth        => $link_auth, 
-				ipmi_message     => $ipmi_message,
-				channel_priv     => $channel_priv, 
-			}});
-			if (($channel_priv eq "oem") or ($channel_priv eq "administrator"))
 			{
-				# Found the user.
+				my $this_user_number = $1;
-				$user_name   = $this_user_name;
+				my $this_user_name   = $2;
-				$user_number = $this_user_number;
+				my $callin           = $3;
+				my $link_auth        = $4;
+				my $ipmi_message     = $5;
+				my $channel_priv     = lc($6);
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-					user_name   => $user_name,
+					this_user_number => $this_user_number,
-					user_number => $user_number, 
+					this_user_name   => $this_user_name, 
+					callin           => $callin, 
+					link_auth        => $link_auth, 
+					ipmi_message     => $ipmi_message,
+					channel_priv     => $channel_priv, 
 				}});
-				last;
+				if (($channel_priv eq "oem") or ($channel_priv eq "administrator"))
+				{
+					# Found the user.
+					$waiting     = 0;
+					$user_name   = $this_user_name;
+					$user_number = $this_user_number;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						waiting     => $waiting, 
+						user_name   => $user_name,
+						user_number => $user_number, 
+					}});
+					last;
+				}
 			}
 		}
+		
+		# Try again later or give up?
+		if (time > $wait_until)
+		{
+			$waiting = 0;
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "error_0331", variables => {
+				shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel,
+				output     => $output,
+			}});
+		}
+		else
+		{
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0129", variables => {
+				shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel,
+				output     => $output,
+			}});
+			sleep 10;
+		}
 	}
 	if (not $user_name)
 	{
 		# Failed to find a user.
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0507", variables => {
-			shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel,
-			output     => $output,
-		}});
 		return(0);
 	}
-	$output      = "";
-	$return_code = "";
 	
 	# Now ask the Striker running the database we're using to try to call the IPMI BMC.
 	my $striker_host_uuid = $anvil->data->{sys}{database}{read_uuid};
@@ -2152,6 +2172,7 @@ LIMIT 1
 		# Update the database, in case needed.
 		my $host_uuid = $anvil->Get->host_uuid();
 		$anvil->Database->insert_or_update_hosts({
+			debug       => $debug, 
 			host_ipmi   => $host_ipmi, 
 			host_key    => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, 
 			host_name   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, 
@@ -2185,6 +2206,7 @@ LIMIT 1
 			# Update the database, in case needed.
 			my $host_uuid = $anvil->Get->host_uuid();
 			$anvil->Database->insert_or_update_hosts({
+				debug       => $debug, 
 				host_ipmi   => $host_ipmi, 
 				host_key    => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, 
 				host_name   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, 
@@ -2296,6 +2318,7 @@ LIMIT 1
 			# Update the database, in case needed.
 			my $host_uuid = $anvil->Get->host_uuid();
 			$anvil->Database->insert_or_update_hosts({
+				debug       => $debug, 
 				host_ipmi   => $host_ipmi, 
 				host_key    => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, 
 				host_name   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, 
@@ -2326,6 +2349,7 @@ LIMIT 1
 				# Update the database, in case needed.
 				my $host_uuid = $anvil->Get->host_uuid();
 				$anvil->Database->insert_or_update_hosts({
+					debug       => $debug, 
 					host_ipmi   => $host_ipmi, 
 					host_key    => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, 
 					host_name   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, 

share/words.xml

@@ -447,6 +447,13 @@ Failed to parse the XML in the new definition file. The error was:
 		<key name="error_0328"><![CDATA[Failed to parse the assigned memory from the definition file. Expected the memory defined as '<memory unit='XiB'>Y</memory>'.]]></key>
 		<key name="error_0329">The server UUID: [#!variable!server_uuid!#] in the definition file wasn't found in the database, unable to update.</key>
 		<key name="error_0330">The new definition has changed the server's name from: [#!variable!current_name!#] to: [#!variable!new_name!#]. Changing the server's name must be done with the 'anvil-rename-server' tool.</key>
+		<key name="error_0331">[ Error ] - The IPMI BMC administrator (oem) user was not found. The output (if any) of the call: [#!variable!shell_call!#] was:
+====
+#!variable!output!#
+====
+
+Giving up.
+		</key>
 		
 		<!-- Files templates -->
 		<!-- NOTE: Translating these files requires an understanding of which lines are translatable -->
@@ -2900,6 +2907,13 @@ The error was:
 #!variable!error!#
 ========
 		</key>
+		<key name="warning_0129">[ Warning ] - The IPMI BMC administrator (oem) user was not found. The output (if any) of the call: [#!variable!shell_call!#] was:
+====
+#!variable!output!#
+====
+
+We will sleep a bit and try again.
+		</key>
 
 		<!-- The entries below here are not sequential, but use a key to find the entry. -->
 		<!-- Run 'striker-parse-os-list to find new entries. --> 

tools/anvil-join-anvil

@@ -2195,7 +2195,7 @@ sub check_local_network
 	# Setup IPMI, if needed.
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0114"});
 	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0114");
-	$anvil->System->configure_ipmi({debug => 3, manifest_uuid => $manifest_uuid});
+	$anvil->System->configure_ipmi({debug => 2, manifest_uuid => $manifest_uuid});
 	
 	return(0);
 }

tools/scancore

@@ -360,6 +360,7 @@ sub startup_tasks
 	$anvil->Database->get_hosts({debug => 3});
 	my $host_uuid = $anvil->Get->host_uuid();
 	$anvil->Database->insert_or_update_hosts({
+		debug       => 2,
 		host_ipmi   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}, 
 		host_key    => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, 
 		host_name   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name},