From 416f51323a1e285c8e6c0f8773c8f49361c8b06d Mon Sep 17 00:00:00 2001 From: Digimer Date: Sat, 1 May 2021 19:49:27 -0400 Subject: [PATCH] * Created tools/striker-boot-machine to, well, boot machines. It uses host_ipmi or, failing that, other fence methods when available to boot a node. * Created Cluster->get_fence_methods() that parses all fence methods out of a recorded CIB and stores the in a hash for a given host_uuid. * Fixed a bug in ScanCore->post_scan_analysis_striker() where the short_host_name was not being stored correctly. Signed-off-by: Digimer --- Anvil/Tools/Cluster.pm | 164 ++++++++++++++++++++++ Anvil/Tools/ScanCore.pm | 10 +- share/words.xml | 13 ++ tools/Makefile.am | 1 + tools/scancore | 2 +- tools/striker-boot-machine | 272 +++++++++++++++++++++++++++++++++++++ 6 files changed, 456 insertions(+), 6 deletions(-) create mode 100755 tools/striker-boot-machine diff --git a/Anvil/Tools/Cluster.pm b/Anvil/Tools/Cluster.pm index b1d90f09..7db566f6 100644 --- a/Anvil/Tools/Cluster.pm +++ b/Anvil/Tools/Cluster.pm @@ -19,6 +19,7 @@ my $THIS_FILE = "Cluster.pm"; # boot_server # check_node_status # delete_server +# get_fence_methods # get_anvil_name # get_anvil_uuid # get_peers @@ -849,6 +850,169 @@ sub delete_server } +=head2 get_fence_methods + +This takes a host UUID, looks up which Anvil! it belongs to, and then load and parses the recorded CIB, if possible. If one is found for the Anvil!, it parses the fence methods and stores them in a hash. + +If the target host is not in an Anvil!, or there is no CIB recorded for the Anvi!, C<< 1 >> is returned. + +B<< Note >>: There is usually only one method, but if there are two or more, they must all be confirmed off before the fence action can be considered successful. + +* fence_method::::order::::method::::command + +Parameters; + +=head3 host_uuid (Optional, default Get->host_uuid) + +This is the host whose fence methods we're looking for. + +=cut +sub get_fence_methods +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Cluster->get_fence_methods()" }}); + + my $host_uuid = defined $parameter->{host_uuid} ? $parameter->{host_uuid} : $anvil->Get->host_uuid; + my $host_name = $anvil->Get->host_name_from_uuid({debug => $debug, host_uuid => $host_uuid}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + host_uuid => $host_uuid, + host_name => $host_name, + }}); + + my $short_host_name = $host_name; + $short_host_name =~ s/\..*$//; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { short_host_name => $short_host_name }}); + + # Find the Anvil! UUID. + my $anvil_uuid = $anvil->Cluster->get_anvil_uuid({host_uuid => $host_uuid}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); + if (not $anvil_uuid) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0295", variables => { host_name => $host_name }}); + return(1); + } + + # Get the Anvil! name now, for logging. + my $anvil_name = $anvil->Get->anvil_name_from_uuid({ + debug => $debug, + anvil_uuid => $anvil_uuid, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_name => $anvil_name }}); + + ### NOTE: This probably won't work with fence methods that require multiple calls be run in parallel. + ### As this is PDUs usually, and we skip them anyway, this shouldn't be an issue. + my $query = "SELECT scan_cluster_cib FROM scan_cluster WHERE scan_cluster_anvil_uuid = ".$anvil->Database->quote($anvil_uuid).";"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + my $scan_cluster_cib = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $scan_cluster_cib = "" if not defined $scan_cluster_cib; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { scan_cluster_cib => $scan_cluster_cib }}); + if (not $scan_cluster_cib) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0296", variables => { anvil_name => $anvil_name }}); + return(1); + } + + # Delete past data, if any. + if (exists $anvil->data->{fence_method}{$short_host_name}) + { + delete $anvil->data->{fence_method}{$short_host_name}; + } + + # Reading in fence data is expensive, so we only do it as needed. + my $update_fence_data = 1; + if ((exists $anvil->data->{fence_data}{updated}) && ($anvil->data->{fence_data}{updated})) + { + my $age = time - $anvil->data->{fence_data}{updated}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { age => $age }}); + if ($age < 86400) + { + # Only refresh daily. + $update_fence_data = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { update_fence_data => $update_fence_data }}); + } + } + if ($update_fence_data) + { + $anvil->Striker->get_fence_data({debug => ($debug + 1)}); + } + + # Parse out the fence methods for this host. + my $problem = $anvil->Cluster->parse_cib({ + debug => $debug, + cib => $scan_cluster_cib, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }}); + if (not $problem) + { + # Parsed! Do we have a fence method we can trust to check the power + # state of this node? + my $node_name = exists $anvil->data->{cib}{parsed}{data}{node}{$short_host_name} ? $short_host_name : $host_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { node_name => $node_name }}); + foreach my $order (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{order}}) + { + my $method = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{order}{$order}{devices}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:order' => $order, + 's2:method' => $method, + }}); + + foreach my $this_method (split/,/, $method) + { + my $agent = $anvil->data->{cib}{parsed}{data}{stonith}{primitive_id}{$this_method}{agent}; + + # We ignore the fake, delay method + next if $agent eq "fence_delay"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:this_method' => $this_method, + 's2:agent' => $agent, + }}); + + my $shell_call = $agent." "; + foreach my $stdin_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$this_method}{argument}}) + { + next if $stdin_name =~ /pcmk_o\w+_action/; + my $switch = ""; + my $value = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$this_method}{argument}{$stdin_name}{value}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:stdin_name' => $stdin_name, + 's2:value' => $value, + }}); + + foreach my $this_switch (sort {$a cmp $b} keys %{$anvil->data->{fence_data}{$agent}{switch}}) + { + my $this_name = $anvil->data->{fence_data}{$agent}{switch}{$this_switch}{name}; + if ($stdin_name eq $this_name) + { + $switch = $this_switch; + my $dashes = (length($switch) > 1) ? "--" : "-"; + $shell_call .= $dashes.$switch." \"".$value."\" "; + last; + } + } + if (not $switch) + { + if ($anvil->data->{fence_data}{$agent}{switch}{$stdin_name}{name}) + { + my $dashes = (length($stdin_name) > 1) ? "--" : "-"; + $shell_call .= $dashes.$stdin_name." \"".$value."\" "; + } + } + } + $anvil->data->{fence_method}{$short_host_name}{order}{$order}{method}{$this_method}{command} = $shell_call; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "fence_method::${short_host_name}::order::${order}::method::${this_method}::command" => $anvil->data->{fence_method}{$short_host_name}{order}{$order}{method}{$this_method}{command}, + }}); + } + } + } + + return(0); +} + + =head2 get_anvil_name This returns the C<< anvils >> -> C<< anvil_name >> for a given C<< anvil_uuid >>. If no C<< anvil_uuid >> is passed, a check is made to see if this host is in an Anvil! and, if so, the Anvil! name it's a member of is returned. diff --git a/Anvil/Tools/ScanCore.pm b/Anvil/Tools/ScanCore.pm index 4afa1ef2..3dd15294 100644 --- a/Anvil/Tools/ScanCore.pm +++ b/Anvil/Tools/ScanCore.pm @@ -2057,7 +2057,7 @@ sub post_scan_analysis_striker # Compile host's data. my $host_name = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_name}; - my $short_host_name = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_name}; + my $short_host_name = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{short_host_name}; my $host_type = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_type}; my $host_key = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_key}; my $host_ipmi = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_ipmi}; @@ -2219,6 +2219,7 @@ LIMIT 1;"; # Do we have IPMI info? if ((not $host_ipmi) && ($host_type eq "node") && ($anvil_uuid)) { + # No host IPMI (that we know of). Can we check using another (non PDU) fence method? my $query = "SELECT scan_cluster_cib FROM scan_cluster WHERE scan_cluster_anvil_uuid = ".$anvil->Database->quote($anvil_uuid).";"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); @@ -2249,8 +2250,9 @@ LIMIT 1;"; 's3:agent' => $agent }}); - # We can't trust a PDU's output, so skip them. + # We can't trust a PDU's output, so skip them. We also can't use the fake 'fence_delay' agent. next if $agent =~ /pdu/; + next if $agent eq "fence_delay"; my $shell_call = $agent." "; foreach my $stdin_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$method}{argument}}) @@ -2312,9 +2314,7 @@ LIMIT 1;"; } } - ### TODO: Add support for power-cycling a target using PDUs. Until this, this - ### will never be hit as we next on no host_ipmi, but will be useful - ### when PDU support is added. + ### TODO: Add support for power-cycling a target using PDUs. # Nothing we can do (for now) $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0559", variables => { host_name => $host_name }}); next; diff --git a/share/words.xml b/share/words.xml index 592513ad..bc10f8aa 100644 --- a/share/words.xml +++ b/share/words.xml @@ -401,6 +401,14 @@ The attempt to start the servers appears to have failed. The return code '0' was Failed to write the file: [#!variable!file!#] on the host: [#!variable!target!#]. Failed to add the server: [#!variable!server_name!#] to the cluster. The return code from the pcs command was: [#!variable!return_code!#]. The output, if any, was: [#!variable!output!#]. The server: [#!variable!server!#] already exists on this Anvil!. Please use a different new name. + ' or '--host-uuid UUID'.]]> + + The attempt to boot the machine failed! The output, if anything, was: [#!variable!output!#]. + The attempt to check the power status of the machine failed. The output, if anything, was: [#!variable!output!#]. + There is no IPMI information or fence options available to boot this machine, unable to proceed. + The host: [#!variable!host_name!#] is not in an Anvil!, unable to parse fence methods. + The Anvil!: [#!variable!anvil_name!#] does not have a recored CIB in the database, unable to parse fence methods. + Either we failed to find a fence method, or all fence methods failed to boot this machine, unable to proceed. @@ -845,6 +853,11 @@ It should be provisioned in the next minute or two. Waiting for the node to finish withdrawing from the cluster. Shutdown complete, powering off now. Done. This node is no longer in the cluster. + The machine: [#!variable!host_name!#] appears to have IPMI, trying to boot it using that... + The target machine is already on, nothing to do. + The target machine is confirmed off, will try to start now. + The target machine is now booting! + The machine: [#!variable!host_name!#] does not have a (known) IPMI BMC, but it is a member of the Anvil! [#!variable!anvil_name!#]. Searching for a fence method to boot it... Starting: [#!variable!program!#]. diff --git a/tools/Makefile.am b/tools/Makefile.am index cbb93175..e95ea9f9 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -32,6 +32,7 @@ dist_sbin_SCRIPTS = \ anvil-update-states \ anvil-update-system \ scancore \ + striker-boot-machine \ striker-get-peer-data \ striker-initialize-host \ striker-manage-install-target \ diff --git a/tools/scancore b/tools/scancore index 67d4018d..016d2542 100755 --- a/tools/scancore +++ b/tools/scancore @@ -112,7 +112,7 @@ while(1) }}); # If we're in maintenance mode, do nothing. - my $maintenance_mode = $anvil->System->maintenance_mode({debug => $debug}); + my $maintenance_mode = $anvil->System->maintenance_mode(); if ($maintenance_mode) { # Sleep and skip. diff --git a/tools/striker-boot-machine b/tools/striker-boot-machine new file mode 100755 index 00000000..2ffdc1dd --- /dev/null +++ b/tools/striker-boot-machine @@ -0,0 +1,272 @@ +#!/usr/bin/perl +# +# This program will boot a target machine using either it's IPMI interface, if available, or one of the +# (non-PDU) fence methods, if the target is in an Anvil! and we have a manifest for it. +# +# Exit codes; +# 0 = Normal exit. +# 1 = No database connection. +# +# TODO: +# + +use strict; +use warnings; +use Anvil::Tools; + +my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; +my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; +if (($running_directory =~ /^\./) && ($ENV{PWD})) +{ + $running_directory =~ s/^\./$ENV{PWD}/; +} + +# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete. +$| = 1; + +my $anvil = Anvil::Tools->new(); + +$anvil->data->{switches}{'job-uuid'} = ""; +$anvil->data->{switches}{'host-uuid'} = ""; +$anvil->data->{switches}{'host-name'} = ""; +$anvil->Get->switches; +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }}); +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'}, + 'switches::host-uuid' => $anvil->data->{switches}{'host-uuid'}, + 'switches::host-name' => $anvil->data->{switches}{'host-name'}, +}}); + +$anvil->Database->connect(); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); +if (not $anvil->data->{sys}{database}{connections}) +{ + # No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try + # again after we exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0075"}); + sleep 10; + $anvil->nice_exit({exit_code => 1}); +} + +# If we don't have a job UUID, try to find one. +if (not $anvil->data->{switches}{'job-uuid'}) +{ + # Load the job data. + $anvil->data->{switches}{'job-uuid'} = $anvil->Job->get_job_uuid({program => $THIS_FILE}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }}); +} + +if ($anvil->data->{switches}{'job-uuid'}) +{ + # Load the job data. + $anvil->Job->clear(); + $anvil->Job->get_job_details(); + $anvil->Job->update_progress({ + progress => 1, + job_picked_up_by => $$, + job_picked_up_at => time, + message => "job_0283", + }); + + # Pull out the job data. + foreach my $line (split/\n/, $anvil->data->{jobs}{job_data}) + { + if ($line =~ /host-uuid=(.*?)$/) + { + $anvil->data->{switches}{'host-uuid'} = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 'switches::host-uuid' => $anvil->data->{switches}{'host-uuid'}, + }}); + } + if ($line =~ /host-name=(.*?)$/) + { + $anvil->data->{switches}{'host-name'} = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 'switches::host-name' => $anvil->data->{switches}{'host-name'}, + }}); + } + } +} + +# If we have a host name, but not a host-uuid, look up the host UUID. +if ((not $anvil->data->{switches}{'host-uuid'}) && ($anvil->data->{switches}{'host-name'})) +{ + $anvil->data->{switches}{'host-uuid'} = $anvil->Get->host_uuid_from_name({host_name => $anvil->data->{switches}{'host-name'}}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 'switches::host-uuid' => $anvil->data->{switches}{'host-uuid'}, + }}); + # host name not found + if (not $anvil->data->{switches}{'host-uuid'}) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0291", variables => { host_name => $anvil->data->{switches}{'host-name'} }}); + $anvil->Job->update_progress({progress => 100, message => "error_0291,!!host_name!".$anvil->data->{switches}{'host-name'}."!!"}); + $anvil->nice_exit({exit_code => 1}); + } +} + +# If we still don't have a host_uuid, we can't proceed. +if (not $anvil->data->{switches}{'host-uuid'}) +{ + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0290"}); + $anvil->Job->update_progress({progress => 100, message => "error_0290"}); + $anvil->nice_exit({exit_code => 1}); +} + +find_boot_method($anvil); + +$anvil->nice_exit({exit_code => 0}); + + +############################################################################################################# +# Functions # +############################################################################################################# + +# This will try to boot the node with host_ipmi data, if available. If not, it will try to find a (non-PDU) +# fence method to try +sub find_boot_method +{ + my ($anvil) = @_; + + $anvil->Database->get_hosts_info({debug => 2}); + + my $host_uuid = $anvil->data->{switches}{'host-uuid'}; + my $host_name = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_name}; + my $host_ipmi = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_ipmi}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + host_uuid => $host_uuid, + host_name => $host_name, + host_ipmi => $anvil->Log->is_secure($host_ipmi), + }}); + + $anvil->data->{sys}{progress} = 10; + + # If we have IPMI, that's the easiest method. + if ($host_ipmi) + { + # Got it. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0327", variables => { host_name => $host_name }}); + $anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}+=10, message => "job_0327,!!host_name!".$host_name."!!"}); + + # First, is the node already on? + my $shell_call = $host_ipmi." -o status"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { shell_call => $shell_call }}); + + call_fence_agent($anvil, $shell_call); + } + + # If I am here, either there is no IPMI. Can we boot it using another fence method? + # Is the machine in an Anvil! system? + my $anvil_uuid = $anvil->Cluster->get_anvil_uuid({host_uuid => $host_uuid}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); + if ($anvil_uuid) + { + my $anvil_name = $anvil->Cluster->get_anvil_name({anvil_uuid => $anvil_uuid}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_name => $anvil_name }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0331", variables => { + host_name => $host_name, + anvil_name => $anvil_name, + }}); + $anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}+=10, message => "job_0331,!!host_name!".$host_name."!!,!!anvil_name!".$anvil_name."!!"}); + + $anvil->Cluster->get_fence_methods({host_uuid => $host_uuid}); + foreach my $target_host_name (sort {$a cmp $b} keys %{$anvil->data->{fence_method}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_name => $target_host_name }}); + foreach my $order (sort {$a cmp $b} keys %{$anvil->data->{fence_method}{$target_host_name}{order}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { order => $order }}); + foreach my $method (sort {$a cmp $b} keys %{$anvil->data->{fence_method}{$target_host_name}{order}{$order}{method}}) + { + next if $method =~ /pdu/; + my $shell_call = $anvil->data->{fence_method}{$target_host_name}{order}{$order}{method}{$method}{command}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + method => $method, + shell_call => $anvil->Log->is_secure($shell_call), + }}); + call_fence_agent($anvil, $shell_call); + } + } + } + + # If I hit here, we ran out of fence options and can't boot the machine. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0297"}); + $anvil->Job->update_progress({progress => 100, message => "error_0297"}); + $anvil->nice_exit({exit_code => 1}); + } + else + { + # Nothing we can do to boot this machine. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0294"}); + $anvil->Job->update_progress({progress => 100, message => "error_0294"}); + $anvil->nice_exit({exit_code => 1}); + } + + return(0); +} + +# This calls a fence agent and exits if it successfully boots the target +sub call_fence_agent +{ + my ($anvil, $shell_call) = @_; + + my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, secure => 1}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); + + if ($return_code eq "0") + { + # The machine is already on + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0328"}); + $anvil->Job->update_progress({progress => 100, message => "job_0328"}); + $anvil->nice_exit({exit_code => 0}); + } + elsif ($return_code eq "1") + { + # Unable to connect to the fence device. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0293", variables => { output => $output }}); + $anvil->Job->update_progress({progress => 100, message => "error_0293,!!output!".$output."!!"}); + $anvil->nice_exit({exit_code => 1}); + } + elsif ($return_code eq "2") + { + # The machine is off, try to start it. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0329"}); + $anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}+=10, message => "job_0329"}); + + # First, is the node already on? + $shell_call .= " -o on"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { shell_call => $shell_call }}); + + my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, secure => 1}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); + + if ($return_code eq "0") + { + # Success! + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0330"}); + $anvil->Job->update_progress({progress => 100, message => "job_0330"}); + + # Update the host's status to 'booting' and exit + $anvil->Database->update_host_status({ + debug => 2, + host_uuid => $anvil->data->{switches}{'host-uuid'}, + host_status => "booting", + }); + $anvil->nice_exit({exit_code => 0}); + } + else + { + # Failed. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0292", variables => { output => $output }}); + $anvil->Job->update_progress({progress => 100, message => "error_0292,!!output!".$output."!!"}); + $anvil->nice_exit({exit_code => 1}); + } + } + + return(0); +}