You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
322 lines
13 KiB
322 lines
13 KiB
#!/usr/bin/perl |
|
# |
|
# This program will boot a target machine using either it's IPMI interface, if available, or one of the |
|
# (non-PDU) fence methods, if the target is in an Anvil! and we have a manifest for it. |
|
# |
|
# Exit codes; |
|
# 0 = Normal exit. |
|
# 1 = No database connection. |
|
# |
|
# TODO: |
|
# |
|
|
|
use strict; |
|
use warnings; |
|
use Anvil::Tools; |
|
|
|
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; |
|
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; |
|
if (($running_directory =~ /^\./) && ($ENV{PWD})) |
|
{ |
|
$running_directory =~ s/^\./$ENV{PWD}/; |
|
} |
|
|
|
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete. |
|
$| = 1; |
|
|
|
my $anvil = Anvil::Tools->new(); |
|
|
|
$anvil->Get->switches({list => [ |
|
"host", |
|
"host-uuid", |
|
"job-uuid"], man => $THIS_FILE}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }}); |
|
|
|
$anvil->Database->connect(); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); |
|
if (not $anvil->data->{sys}{database}{connections}) |
|
{ |
|
# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try |
|
# again after we exit. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0075"}); |
|
sleep 10; |
|
$anvil->nice_exit({exit_code => 1}); |
|
} |
|
|
|
if ($anvil->data->{switches}{'job-uuid'}) |
|
{ |
|
# Load the job data. |
|
$anvil->Job->clear(); |
|
$anvil->Job->get_job_details(); |
|
$anvil->Job->update_progress({ |
|
progress => 1, |
|
job_picked_up_by => $$, |
|
job_picked_up_at => time, |
|
message => "job_0283", |
|
}); |
|
|
|
# Pull out the job data. |
|
foreach my $line (split/\n/, $anvil->data->{jobs}{job_data}) |
|
{ |
|
if ($line =~ /host-uuid=(.*?)$/) |
|
{ |
|
$anvil->data->{switches}{'host-uuid'} = $1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
'switches::host-uuid' => $anvil->data->{switches}{'host-uuid'}, |
|
}}); |
|
} |
|
if ($line =~ /host=(.*?)$/) |
|
{ |
|
$anvil->data->{switches}{host} = $1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
'switches::host' => $anvil->data->{switches}{host}, |
|
}}); |
|
} |
|
} |
|
} |
|
|
|
# Get the host info (copy host-uuid to host) |
|
$anvil->data->{switches}{'host-name'} = "" if not defined $anvil->data->{switches}{'host-name'}; |
|
if ((not $anvil->data->{switches}{host}) && ($anvil->data->{switches}{'host-uuid'})) |
|
{ |
|
$anvil->data->{switches}{host} = $anvil->data->{switches}{'host-uuid'}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
'switches::host' => $anvil->data->{switches}{host}, |
|
}}); |
|
} |
|
|
|
# If the host is 'all', don't translate to the host_uuid. Otherwise, look for the host_uuid from the host string |
|
if ((not $anvil->data->{switches}{'host-uuid'}) && ($anvil->data->{switches}{host} ne "all")) |
|
{ |
|
$anvil->data->{switches}{'host-uuid'} = $anvil->Database->get_host_uuid_from_string({string => $anvil->data->{switches}{host}}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
'switches::host-uuid' => $anvil->data->{switches}{'host-uuid'}, |
|
}}); |
|
|
|
# host name not found |
|
if (not $anvil->data->{switches}{'host-uuid'}) |
|
{ |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0291", variables => { host_name => $anvil->data->{switches}{'host-name'} }}); |
|
$anvil->Job->update_progress({progress => 100, message => "error_0291,!!host_name!".$anvil->data->{switches}{'host-name'}."!!"}); |
|
$anvil->nice_exit({exit_code => 1}); |
|
} |
|
|
|
$anvil->data->{switches}{'host-name'} = $anvil->Get->host_name_from_uuid({host_uuid => $anvil->data->{switches}{'host-uuid'}}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
'switches::host-name' => $anvil->data->{switches}{'host-name'}, |
|
}}); |
|
} |
|
|
|
find_boot_method($anvil); |
|
|
|
$anvil->nice_exit({exit_code => 0}); |
|
|
|
|
|
############################################################################################################# |
|
# Functions # |
|
############################################################################################################# |
|
|
|
# This will try to boot the node with host_ipmi data, if available. If not, it will try to find a (non-PDU) |
|
# fence method to try |
|
sub find_boot_method |
|
{ |
|
my ($anvil) = @_; |
|
|
|
$anvil->Database->get_hosts_info({debug => 2}); |
|
|
|
my $hosts = []; |
|
if ($anvil->data->{switches}{'host-uuid'}) |
|
{ |
|
push @{$hosts}, $anvil->data->{switches}{'host-name'}; |
|
} |
|
elsif ($anvil->data->{switches}{host} eq "all") |
|
{ |
|
$anvil->Database->get_hosts({debug => 2}); |
|
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}}) |
|
{ |
|
my $host_uuid = $anvil->data->{sys}{hosts}{by_name}{$host_name}; |
|
my $host_ipmi = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}; |
|
my $host_key = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:host_name' => $host_name, |
|
's2:host_uuid' => $host_uuid, |
|
's3:host_ipmi' => $anvil->Log->is_secure($host_ipmi), |
|
}}); |
|
next if $host_key eq "DELETED"; |
|
next if not $host_ipmi; |
|
|
|
push @{$hosts}, $host_name; |
|
} |
|
} |
|
|
|
my $host_count = @{$hosts}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_count => $host_count }}); |
|
if (not $host_count) |
|
{ |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0357"}); |
|
$anvil->Job->update_progress({progress => 100, message => "message_0357"}); |
|
$anvil->nice_exit({exit_code => 0}); |
|
} |
|
|
|
my $steps = int((80 / $host_count) / 3); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { steps => $steps }}); |
|
|
|
$anvil->data->{sys}{progress} = 5; |
|
foreach my $host_name (sort {$a cmp $b} @{$hosts}) |
|
{ |
|
my $host_uuid = $anvil->Database->get_host_uuid_from_string({string => $host_name}); |
|
my $host_ipmi = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}; |
|
my $host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
host_uuid => $host_uuid, |
|
host_name => $host_name, |
|
host_ipmi => $anvil->Log->is_secure($host_ipmi), |
|
host_type => $host_type, |
|
}}); |
|
|
|
$anvil->data->{sys}{progress} += $steps; |
|
|
|
if ($host_ipmi) |
|
{ |
|
# Got it. |
|
$anvil->data->{sys}{progress} += $steps; |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0327", variables => { host_name => $host_name }}); |
|
$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}, message => "job_0327,!!host_name!".$host_name."!!"}); |
|
|
|
# First, is the node already on? |
|
my $shell_call = $host_ipmi." -o status"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { shell_call => $shell_call }}); |
|
|
|
$anvil->data->{sys}{progress} += $steps; |
|
my $problem = call_fence_agent($anvil, $shell_call, $anvil->data->{sys}{progress}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { problem => $problem }}); |
|
|
|
$anvil->data->{sys}{progress} += $steps; |
|
if ($problem) |
|
{ |
|
# If I am here, there is no IPMI. Can we boot it using another fence method? |
|
# The machine would have to be in an Anvil! for this to work. |
|
my $anvil_uuid = $anvil->Cluster->get_anvil_uuid({host_uuid => $host_uuid}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); |
|
if ($anvil_uuid) |
|
{ |
|
my $anvil_name = $anvil->Cluster->get_anvil_name({anvil_uuid => $anvil_uuid}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_name => $anvil_name }}); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0331", variables => { |
|
host_name => $host_name, |
|
anvil_name => $anvil_name, |
|
}}); |
|
$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}, message => "job_0331,!!host_name!".$host_name."!!,!!anvil_name!".$anvil_name."!!"}); |
|
|
|
$anvil->Cluster->get_fence_methods({host_uuid => $host_uuid}); |
|
foreach my $target_host_name (sort {$a cmp $b} keys %{$anvil->data->{fence_method}}) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_name => $target_host_name }}); |
|
foreach my $order (sort {$a cmp $b} keys %{$anvil->data->{fence_method}{$target_host_name}{order}}) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { order => $order }}); |
|
foreach my $method (sort {$a cmp $b} keys %{$anvil->data->{fence_method}{$target_host_name}{order}{$order}{method}}) |
|
{ |
|
next if $method =~ /pdu/; |
|
my $shell_call = $anvil->data->{fence_method}{$target_host_name}{order}{$order}{method}{$method}{command}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
method => $method, |
|
shell_call => $anvil->Log->is_secure($shell_call), |
|
}}); |
|
my $problem = call_fence_agent($anvil, $shell_call, $anvil->data->{sys}{progress}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); |
|
|
|
if ($problem) |
|
{ |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0297"}); |
|
$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}, message => "error_0297"}); |
|
} |
|
} |
|
} |
|
} |
|
} |
|
else |
|
{ |
|
# Nothing we can do to boot this machine. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0294"}); |
|
$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}, message => "error_0294"}); |
|
} |
|
|
|
} |
|
} |
|
} |
|
|
|
# Done. |
|
$anvil->Job->update_progress({progress => 100, message => "message_0025"}); |
|
$anvil->nice_exit({exit_code => 0}); |
|
|
|
return(0); |
|
} |
|
|
|
# This calls a fence agent and exits if it successfully boots the target |
|
sub call_fence_agent |
|
{ |
|
my ($anvil, $shell_call, $progress) = @_; |
|
|
|
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, secure => 1}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
|
|
if ($return_code eq "0") |
|
{ |
|
# The machine is already on |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0328"}); |
|
$anvil->Job->update_progress({progress => 100, message => "job_0328"}); |
|
return(0); |
|
} |
|
elsif ($return_code eq "1") |
|
{ |
|
# Unable to connect to the fence device. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0293", variables => { output => $output }}); |
|
$anvil->Job->update_progress({progress => 100, message => "error_0293,!!output!".$output."!!"}); |
|
return(1); |
|
} |
|
elsif ($return_code eq "2") |
|
{ |
|
# The machine is off, try to start it. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0329"}); |
|
$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}+=10, message => "job_0329"}); |
|
|
|
# First, is the node already on? |
|
$shell_call .= " -o on"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, secure => 1}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
|
|
if ($return_code eq "0") |
|
{ |
|
# Success! |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0330"}); |
|
$anvil->Job->update_progress({progress => 100, message => "job_0330"}); |
|
|
|
# Update the host's status to 'booting' and exit |
|
$anvil->Database->update_host_status({ |
|
debug => 2, |
|
host_uuid => $anvil->data->{switches}{'host-uuid'}, |
|
host_status => "booting", |
|
}); |
|
return(0); |
|
} |
|
else |
|
{ |
|
# Failed. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0292", variables => { output => $output }}); |
|
$anvil->Job->update_progress({progress => 100, message => "error_0292,!!output!".$output."!!"}); |
|
return(1); |
|
} |
|
} |
|
|
|
return(0); |
|
}
|
|
|