* Created tools/striker-boot-machine to, well, boot machines. It uses host_ipmi or, failing that, other fence methods when available to boot a node.
* Created Cluster->get_fence_methods() that parses all fence methods out of a recorded CIB and stores the in a hash for a given host_uuid. * Fixed a bug in ScanCore->post_scan_analysis_striker() where the short_host_name was not being stored correctly. Signed-off-by: Digimer <digimer@alteeve.ca>main
Digimer
4 years ago
parent
159c4a1612
commit
416f51323a
6 changed files with 456 additions and 6 deletions
@ -0,0 +1,272 @@ |
|||||||
|
#!/usr/bin/perl |
||||||
|
# |
||||||
|
# This program will boot a target machine using either it's IPMI interface, if available, or one of the |
||||||
|
# (non-PDU) fence methods, if the target is in an Anvil! and we have a manifest for it. |
||||||
|
# |
||||||
|
# Exit codes; |
||||||
|
# 0 = Normal exit. |
||||||
|
# 1 = No database connection. |
||||||
|
# |
||||||
|
# TODO: |
||||||
|
# |
||||||
|
|
||||||
|
use strict; |
||||||
|
use warnings; |
||||||
|
use Anvil::Tools; |
||||||
|
|
||||||
|
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; |
||||||
|
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; |
||||||
|
if (($running_directory =~ /^\./) && ($ENV{PWD})) |
||||||
|
{ |
||||||
|
$running_directory =~ s/^\./$ENV{PWD}/; |
||||||
|
} |
||||||
|
|
||||||
|
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete. |
||||||
|
$| = 1; |
||||||
|
|
||||||
|
my $anvil = Anvil::Tools->new(); |
||||||
|
|
||||||
|
$anvil->data->{switches}{'job-uuid'} = ""; |
||||||
|
$anvil->data->{switches}{'host-uuid'} = ""; |
||||||
|
$anvil->data->{switches}{'host-name'} = ""; |
||||||
|
$anvil->Get->switches; |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }}); |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||||
|
'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'}, |
||||||
|
'switches::host-uuid' => $anvil->data->{switches}{'host-uuid'}, |
||||||
|
'switches::host-name' => $anvil->data->{switches}{'host-name'}, |
||||||
|
}}); |
||||||
|
|
||||||
|
$anvil->Database->connect(); |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); |
||||||
|
if (not $anvil->data->{sys}{database}{connections}) |
||||||
|
{ |
||||||
|
# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try |
||||||
|
# again after we exit. |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0075"}); |
||||||
|
sleep 10; |
||||||
|
$anvil->nice_exit({exit_code => 1}); |
||||||
|
} |
||||||
|
|
||||||
|
# If we don't have a job UUID, try to find one. |
||||||
|
if (not $anvil->data->{switches}{'job-uuid'}) |
||||||
|
{ |
||||||
|
# Load the job data. |
||||||
|
$anvil->data->{switches}{'job-uuid'} = $anvil->Job->get_job_uuid({program => $THIS_FILE}); |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }}); |
||||||
|
} |
||||||
|
|
||||||
|
if ($anvil->data->{switches}{'job-uuid'}) |
||||||
|
{ |
||||||
|
# Load the job data. |
||||||
|
$anvil->Job->clear(); |
||||||
|
$anvil->Job->get_job_details(); |
||||||
|
$anvil->Job->update_progress({ |
||||||
|
progress => 1, |
||||||
|
job_picked_up_by => $$, |
||||||
|
job_picked_up_at => time, |
||||||
|
message => "job_0283", |
||||||
|
}); |
||||||
|
|
||||||
|
# Pull out the job data. |
||||||
|
foreach my $line (split/\n/, $anvil->data->{jobs}{job_data}) |
||||||
|
{ |
||||||
|
if ($line =~ /host-uuid=(.*?)$/) |
||||||
|
{ |
||||||
|
$anvil->data->{switches}{'host-uuid'} = $1; |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||||
|
'switches::host-uuid' => $anvil->data->{switches}{'host-uuid'}, |
||||||
|
}}); |
||||||
|
} |
||||||
|
if ($line =~ /host-name=(.*?)$/) |
||||||
|
{ |
||||||
|
$anvil->data->{switches}{'host-name'} = $1; |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||||
|
'switches::host-name' => $anvil->data->{switches}{'host-name'}, |
||||||
|
}}); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
# If we have a host name, but not a host-uuid, look up the host UUID. |
||||||
|
if ((not $anvil->data->{switches}{'host-uuid'}) && ($anvil->data->{switches}{'host-name'})) |
||||||
|
{ |
||||||
|
$anvil->data->{switches}{'host-uuid'} = $anvil->Get->host_uuid_from_name({host_name => $anvil->data->{switches}{'host-name'}}); |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||||
|
'switches::host-uuid' => $anvil->data->{switches}{'host-uuid'}, |
||||||
|
}}); |
||||||
|
# host name not found |
||||||
|
if (not $anvil->data->{switches}{'host-uuid'}) |
||||||
|
{ |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0291", variables => { host_name => $anvil->data->{switches}{'host-name'} }}); |
||||||
|
$anvil->Job->update_progress({progress => 100, message => "error_0291,!!host_name!".$anvil->data->{switches}{'host-name'}."!!"}); |
||||||
|
$anvil->nice_exit({exit_code => 1}); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
# If we still don't have a host_uuid, we can't proceed. |
||||||
|
if (not $anvil->data->{switches}{'host-uuid'}) |
||||||
|
{ |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0290"}); |
||||||
|
$anvil->Job->update_progress({progress => 100, message => "error_0290"}); |
||||||
|
$anvil->nice_exit({exit_code => 1}); |
||||||
|
} |
||||||
|
|
||||||
|
find_boot_method($anvil); |
||||||
|
|
||||||
|
$anvil->nice_exit({exit_code => 0}); |
||||||
|
|
||||||
|
|
||||||
|
############################################################################################################# |
||||||
|
# Functions # |
||||||
|
############################################################################################################# |
||||||
|
|
||||||
|
# This will try to boot the node with host_ipmi data, if available. If not, it will try to find a (non-PDU) |
||||||
|
# fence method to try |
||||||
|
sub find_boot_method |
||||||
|
{ |
||||||
|
my ($anvil) = @_; |
||||||
|
|
||||||
|
$anvil->Database->get_hosts_info({debug => 2}); |
||||||
|
|
||||||
|
my $host_uuid = $anvil->data->{switches}{'host-uuid'}; |
||||||
|
my $host_name = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_name}; |
||||||
|
my $host_ipmi = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_ipmi}; |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||||
|
host_uuid => $host_uuid, |
||||||
|
host_name => $host_name, |
||||||
|
host_ipmi => $anvil->Log->is_secure($host_ipmi), |
||||||
|
}}); |
||||||
|
|
||||||
|
$anvil->data->{sys}{progress} = 10; |
||||||
|
|
||||||
|
# If we have IPMI, that's the easiest method. |
||||||
|
if ($host_ipmi) |
||||||
|
{ |
||||||
|
# Got it. |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0327", variables => { host_name => $host_name }}); |
||||||
|
$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}+=10, message => "job_0327,!!host_name!".$host_name."!!"}); |
||||||
|
|
||||||
|
# First, is the node already on? |
||||||
|
my $shell_call = $host_ipmi." -o status"; |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { shell_call => $shell_call }}); |
||||||
|
|
||||||
|
call_fence_agent($anvil, $shell_call); |
||||||
|
} |
||||||
|
|
||||||
|
# If I am here, either there is no IPMI. Can we boot it using another fence method? |
||||||
|
# Is the machine in an Anvil! system? |
||||||
|
my $anvil_uuid = $anvil->Cluster->get_anvil_uuid({host_uuid => $host_uuid}); |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); |
||||||
|
if ($anvil_uuid) |
||||||
|
{ |
||||||
|
my $anvil_name = $anvil->Cluster->get_anvil_name({anvil_uuid => $anvil_uuid}); |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_name => $anvil_name }}); |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0331", variables => { |
||||||
|
host_name => $host_name, |
||||||
|
anvil_name => $anvil_name, |
||||||
|
}}); |
||||||
|
$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}+=10, message => "job_0331,!!host_name!".$host_name."!!,!!anvil_name!".$anvil_name."!!"}); |
||||||
|
|
||||||
|
$anvil->Cluster->get_fence_methods({host_uuid => $host_uuid}); |
||||||
|
foreach my $target_host_name (sort {$a cmp $b} keys %{$anvil->data->{fence_method}}) |
||||||
|
{ |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_name => $target_host_name }}); |
||||||
|
foreach my $order (sort {$a cmp $b} keys %{$anvil->data->{fence_method}{$target_host_name}{order}}) |
||||||
|
{ |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { order => $order }}); |
||||||
|
foreach my $method (sort {$a cmp $b} keys %{$anvil->data->{fence_method}{$target_host_name}{order}{$order}{method}}) |
||||||
|
{ |
||||||
|
next if $method =~ /pdu/; |
||||||
|
my $shell_call = $anvil->data->{fence_method}{$target_host_name}{order}{$order}{method}{$method}{command}; |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||||
|
method => $method, |
||||||
|
shell_call => $anvil->Log->is_secure($shell_call), |
||||||
|
}}); |
||||||
|
call_fence_agent($anvil, $shell_call); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
# If I hit here, we ran out of fence options and can't boot the machine. |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0297"}); |
||||||
|
$anvil->Job->update_progress({progress => 100, message => "error_0297"}); |
||||||
|
$anvil->nice_exit({exit_code => 1}); |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
# Nothing we can do to boot this machine. |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0294"}); |
||||||
|
$anvil->Job->update_progress({progress => 100, message => "error_0294"}); |
||||||
|
$anvil->nice_exit({exit_code => 1}); |
||||||
|
} |
||||||
|
|
||||||
|
return(0); |
||||||
|
} |
||||||
|
|
||||||
|
# This calls a fence agent and exits if it successfully boots the target |
||||||
|
sub call_fence_agent |
||||||
|
{ |
||||||
|
my ($anvil, $shell_call) = @_; |
||||||
|
|
||||||
|
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, secure => 1}); |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||||
|
output => $output, |
||||||
|
return_code => $return_code, |
||||||
|
}}); |
||||||
|
|
||||||
|
if ($return_code eq "0") |
||||||
|
{ |
||||||
|
# The machine is already on |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0328"}); |
||||||
|
$anvil->Job->update_progress({progress => 100, message => "job_0328"}); |
||||||
|
$anvil->nice_exit({exit_code => 0}); |
||||||
|
} |
||||||
|
elsif ($return_code eq "1") |
||||||
|
{ |
||||||
|
# Unable to connect to the fence device. |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0293", variables => { output => $output }}); |
||||||
|
$anvil->Job->update_progress({progress => 100, message => "error_0293,!!output!".$output."!!"}); |
||||||
|
$anvil->nice_exit({exit_code => 1}); |
||||||
|
} |
||||||
|
elsif ($return_code eq "2") |
||||||
|
{ |
||||||
|
# The machine is off, try to start it. |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0329"}); |
||||||
|
$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}+=10, message => "job_0329"}); |
||||||
|
|
||||||
|
# First, is the node already on? |
||||||
|
$shell_call .= " -o on"; |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { shell_call => $shell_call }}); |
||||||
|
|
||||||
|
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, secure => 1}); |
||||||
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||||
|
output => $output, |
||||||
|
return_code => $return_code, |
||||||
|
}}); |
||||||
|
|
||||||
|
if ($return_code eq "0") |
||||||
|
{ |
||||||
|
# Success! |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0330"}); |
||||||
|
$anvil->Job->update_progress({progress => 100, message => "job_0330"}); |
||||||
|
|
||||||
|
# Update the host's status to 'booting' and exit |
||||||
|
$anvil->Database->update_host_status({ |
||||||
|
debug => 2, |
||||||
|
host_uuid => $anvil->data->{switches}{'host-uuid'}, |
||||||
|
host_status => "booting", |
||||||
|
}); |
||||||
|
$anvil->nice_exit({exit_code => 0}); |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
# Failed. |
||||||
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0292", variables => { output => $output }}); |
||||||
|
$anvil->Job->update_progress({progress => 100, message => "error_0292,!!output!".$output."!!"}); |
||||||
|
$anvil->nice_exit({exit_code => 1}); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return(0); |
||||||
|
} |
||||||
Loading…
Reference in new issue