From 4c7bb45ab95271d348d5ce2f838bedd735382733 Mon Sep 17 00:00:00 2001 From: Digimer Date: Wed, 25 Aug 2021 21:02:00 -0400 Subject: [PATCH] Fixed a race condition where configuring the IPMI BMC would appear to fail because the BMC wouldn't report the user list after a cold reset. Signed-off-by: Digimer --- Anvil/Tools/Database.pm | 3 +- Anvil/Tools/System.pm | 102 +++++++++++++++++++++++++--------------- share/words.xml | 14 ++++++ tools/anvil-join-anvil | 2 +- tools/scancore | 1 + 5 files changed, 81 insertions(+), 41 deletions(-) diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 1433f805..a6177d09 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -7407,7 +7407,7 @@ WHERE $anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); } - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0126", variables => { method => "Database->insert_or_update_hosts()" }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }}); return($host_uuid); } @@ -15388,6 +15388,7 @@ sub update_host_status # We're only updating the status, so we'll read in the current data to pass back in. $anvil->Database->get_hosts({debug => $debug}); $anvil->Database->insert_or_update_hosts({ + debug => $debug, host_ipmi => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}, host_key => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, host_name => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, diff --git a/Anvil/Tools/System.pm b/Anvil/Tools/System.pm index 5935c0a1..bfd8cbef 100644 --- a/Anvil/Tools/System.pm +++ b/Anvil/Tools/System.pm @@ -2063,60 +2063,80 @@ LIMIT 1 } } + # This can take a while to come up after cold resetting a BMC. So we'll try for a minute. my $user_name = ""; my $user_number = ""; - ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - output => $output, - return_code => $return_code, - }}); - foreach my $line (split/\n/, $output) + my $waiting = 1; + my $wait_until = time + 120; + while ($waiting) { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }}); - - next if $line =~ /Empty User/i; - next if $line =~ /NO ACCESS/i; - next if $line =~ /Unknown/i; - if ($line =~ /^(\d+)\s+(.*?)\s+(\w+)\s+(\w+)\s+(\w+)\s+(.*)$/) + my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + output => $output, + return_code => $return_code, + }}); + foreach my $line (split/\n/, $output) { - my $this_user_number = $1; - my $this_user_name = $2; - my $callin = $3; - my $link_auth = $4; - my $ipmi_message = $5; - my $channel_priv = lc($6); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - this_user_number => $this_user_number, - this_user_name => $this_user_name, - callin => $callin, - link_auth => $link_auth, - ipmi_message => $ipmi_message, - channel_priv => $channel_priv, - }}); - if (($channel_priv eq "oem") or ($channel_priv eq "administrator")) + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }}); + + next if $line =~ /Empty User/i; + next if $line =~ /NO ACCESS/i; + next if $line =~ /Unknown/i; + if ($line =~ /^(\d+)\s+(.*?)\s+(\w+)\s+(\w+)\s+(\w+)\s+(.*)$/) { - # Found the user. - $user_name = $this_user_name; - $user_number = $this_user_number; + my $this_user_number = $1; + my $this_user_name = $2; + my $callin = $3; + my $link_auth = $4; + my $ipmi_message = $5; + my $channel_priv = lc($6); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - user_name => $user_name, - user_number => $user_number, + this_user_number => $this_user_number, + this_user_name => $this_user_name, + callin => $callin, + link_auth => $link_auth, + ipmi_message => $ipmi_message, + channel_priv => $channel_priv, }}); - last; + if (($channel_priv eq "oem") or ($channel_priv eq "administrator")) + { + # Found the user. + $waiting = 0; + $user_name = $this_user_name; + $user_number = $this_user_number; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + waiting => $waiting, + user_name => $user_name, + user_number => $user_number, + }}); + last; + } } } + + # Try again later or give up? + if (time > $wait_until) + { + $waiting = 0; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "error_0331", variables => { + shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel, + output => $output, + }}); + } + else + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0129", variables => { + shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel, + output => $output, + }}); + sleep 10; + } } if (not $user_name) { # Failed to find a user. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0507", variables => { - shell_call => $anvil->data->{path}{exe}{ipmitool}." user list ".$lan_channel, - output => $output, - }}); return(0); } - $output = ""; - $return_code = ""; # Now ask the Striker running the database we're using to try to call the IPMI BMC. my $striker_host_uuid = $anvil->data->{sys}{database}{read_uuid}; @@ -2152,6 +2172,7 @@ LIMIT 1 # Update the database, in case needed. my $host_uuid = $anvil->Get->host_uuid(); $anvil->Database->insert_or_update_hosts({ + debug => $debug, host_ipmi => $host_ipmi, host_key => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, host_name => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, @@ -2185,6 +2206,7 @@ LIMIT 1 # Update the database, in case needed. my $host_uuid = $anvil->Get->host_uuid(); $anvil->Database->insert_or_update_hosts({ + debug => $debug, host_ipmi => $host_ipmi, host_key => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, host_name => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, @@ -2296,6 +2318,7 @@ LIMIT 1 # Update the database, in case needed. my $host_uuid = $anvil->Get->host_uuid(); $anvil->Database->insert_or_update_hosts({ + debug => $debug, host_ipmi => $host_ipmi, host_key => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, host_name => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, @@ -2326,6 +2349,7 @@ LIMIT 1 # Update the database, in case needed. my $host_uuid = $anvil->Get->host_uuid(); $anvil->Database->insert_or_update_hosts({ + debug => $debug, host_ipmi => $host_ipmi, host_key => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, host_name => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, diff --git a/share/words.xml b/share/words.xml index 0a4b68b5..93f4ff27 100644 --- a/share/words.xml +++ b/share/words.xml @@ -447,6 +447,13 @@ Failed to parse the XML in the new definition file. The error was: Y'.]]> The server UUID: [#!variable!server_uuid!#] in the definition file wasn't found in the database, unable to update. The new definition has changed the server's name from: [#!variable!current_name!#] to: [#!variable!new_name!#]. Changing the server's name must be done with the 'anvil-rename-server' tool. + [ Error ] - The IPMI BMC administrator (oem) user was not found. The output (if any) of the call: [#!variable!shell_call!#] was: +==== +#!variable!output!# +==== + +Giving up. + @@ -2900,6 +2907,13 @@ The error was: #!variable!error!# ======== + [ Warning ] - The IPMI BMC administrator (oem) user was not found. The output (if any) of the call: [#!variable!shell_call!#] was: +==== +#!variable!output!# +==== + +We will sleep a bit and try again. + diff --git a/tools/anvil-join-anvil b/tools/anvil-join-anvil index 6e1c2361..8c874e2f 100755 --- a/tools/anvil-join-anvil +++ b/tools/anvil-join-anvil @@ -2195,7 +2195,7 @@ sub check_local_network # Setup IPMI, if needed. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0114"}); update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0114"); - $anvil->System->configure_ipmi({debug => 3, manifest_uuid => $manifest_uuid}); + $anvil->System->configure_ipmi({debug => 2, manifest_uuid => $manifest_uuid}); return(0); } diff --git a/tools/scancore b/tools/scancore index 5893711d..9b131e53 100755 --- a/tools/scancore +++ b/tools/scancore @@ -360,6 +360,7 @@ sub startup_tasks $anvil->Database->get_hosts({debug => 3}); my $host_uuid = $anvil->Get->host_uuid(); $anvil->Database->insert_or_update_hosts({ + debug => 2, host_ipmi => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}, host_key => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, host_name => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name},