From d271ffec26a51d63d2a0ed84f96da88d331a6efe Mon Sep 17 00:00:00 2001 From: Digimer Date: Wed, 31 Aug 2022 12:57:01 -0400 Subject: [PATCH 1/2] * Updated Cluster->parse_crm_mon() to record the role of stonith resources. * Fixed a bug in System->parse_arguments() where a quoted password without spaces was returned without being recorded in the hash. Also updated logging to log 'suppressed' for passwords when secure logging is disabled. Signed-off-by: Digimer --- Anvil/Tools/Cluster.pm | 73 ++++++++++++++++++++-------- Anvil/Tools/Log.pm | 6 +-- Anvil/Tools/System.pm | 105 ++++++++++++++++++++++++++++++++++------- share/words.xml | 1 + 4 files changed, 147 insertions(+), 38 deletions(-) diff --git a/Anvil/Tools/Cluster.pm b/Anvil/Tools/Cluster.pm index f2a0f895..2ece435b 100644 --- a/Anvil/Tools/Cluster.pm +++ b/Anvil/Tools/Cluster.pm @@ -2976,7 +2976,7 @@ sub parse_cib if ($anvil->Network->is_local({host => $target})) { # Local call - ($cib_data, $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call}); + ($cib_data, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { cib_data => $cib_data, return_code => $return_code, @@ -3151,6 +3151,7 @@ sub parse_cib foreach my $node_state ($dom->findnodes('/cib/status/node_state')) { my $id = $node_state->{id}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { id => $id }}); foreach my $variable (sort {$a cmp $b} keys %{$node_state}) { next if $variable eq "id"; @@ -3162,6 +3163,7 @@ sub parse_cib foreach my $lrm ($node_state->findnodes('./lrm')) { my $lrm_id = $lrm->{id}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { lrm_id => $lrm_id }}); foreach my $lrm_resource ($lrm->findnodes('./lrm_resources/lrm_resource')) { my $lrm_resource_id = $lrm_resource->{id}; @@ -3642,6 +3644,8 @@ sub parse_cib 's5:active' => $active, }}); } + + return($problem); } @@ -3717,7 +3721,7 @@ sub parse_crm_mon if ($anvil->Network->is_local({host => $target})) { # Local call - ($crm_mon_data, $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call}); + ($crm_mon_data, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { crm_mon_data => $crm_mon_data, return_code => $return_code, @@ -3764,28 +3768,59 @@ sub parse_crm_mon $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }}); foreach my $resource ($dom->findnodes('/pacemaker-result/resources/resource')) { - next if $resource->{resource_agent} ne "ocf::alteeve:server"; - my $id = $resource->{id}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { id => $id }}); - foreach my $variable (sort {$a cmp $b} keys %{$resource}) + if ($resource->{resource_agent} eq "ocf::alteeve:server") { - next if $variable eq "id"; - $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$id}{variables}{$variable} = $resource->{$variable}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "crm_mon::parsed::pacemaker-result::resources::resource::${id}::variables::${variable}" => $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$id}{variables}{$variable}, - }}); + my $id = $resource->{id}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { id => $id }}); + foreach my $variable (sort {$a cmp $b} keys %{$resource}) + { + next if $variable eq "id"; + $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$id}{variables}{$variable} = $resource->{$variable}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "crm_mon::parsed::pacemaker-result::resources::resource::${id}::variables::${variable}" => $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$id}{variables}{$variable}, + }}); + } + foreach my $node ($resource->findnodes('./node')) + { + my $node_id = $node->{id}; + my $node_name = $node->{name}; + my $cached = $node->{cached}; + $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$id}{host}{node_name} = $node->{name}; + $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$id}{host}{node_id} = $node->{id}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "crm_mon::parsed::pacemaker-result::resources::resource::${id}::host::node_name" => $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$id}{host}{node_name}, + "crm_mon::parsed::pacemaker-result::resources::resource::${id}::host::node_id" => $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$id}{host}{node_id}, + }}); + } } - foreach my $node ($resource->findnodes('./node')) + if ($resource->{resource_agent} =~ /stonith:(.*)$/) { - my $node_id = $node->{id}; - my $node_name = $node->{name}; - my $cached = $node->{cached}; - $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$id}{host}{node_name} = $node->{name}; - $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$id}{host}{node_id} = $node->{id}; + my $fence_agent = $1; + my $id = $resource->{id}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "crm_mon::parsed::pacemaker-result::resources::resource::${id}::host::node_name" => $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$id}{host}{node_name}, - "crm_mon::parsed::pacemaker-result::resources::resource::${id}::host::node_id" => $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$id}{host}{node_id}, + 's1:fence_agent' => $fence_agent, + 's2:id' => $id, }}); + foreach my $variable (sort {$a cmp $b} keys %{$resource}) + { + next if $variable eq "id"; + $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{stonith}{$id}{variables}{$variable} = $resource->{$variable}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "crm_mon::parsed::pacemaker-result::resources::stonith::${id}::variables::${variable}" => $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{stonith}{$id}{variables}{$variable}, + }}); + } + foreach my $node ($resource->findnodes('./node')) + { + my $node_id = $node->{id}; + my $node_name = $node->{name}; + my $cached = $node->{cached}; + $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{stonith}{$id}{host}{node_name} = $node->{name}; + $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{stonith}{$id}{host}{node_id} = $node->{id}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "crm_mon::parsed::pacemaker-result::resources::stonith::${id}::host::node_name" => $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{stonith}{$id}{host}{node_name}, + "crm_mon::parsed::pacemaker-result::resources::stonith::${id}::host::node_id" => $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{stonith}{$id}{host}{node_id}, + }}); + } } } } diff --git a/Anvil/Tools/Log.pm b/Anvil/Tools/Log.pm index d93b6a22..fecc9c59 100644 --- a/Anvil/Tools/Log.pm +++ b/Anvil/Tools/Log.pm @@ -924,7 +924,7 @@ sub variables # Strip a leading 'sX:' in case the user is sorting the output. my $say_key = $key; $say_key =~ s/^s(\d+)://; - $raw .= "$say_key: [".$list->{$key}."], "; + $raw .= $say_key.": [".$list->{$key}."], "; print $THIS_FILE." ".__LINE__."; raw: [".$raw."]\n" if $test; } $raw =~ s/, $//; @@ -961,8 +961,8 @@ sub variables { $say_key = $prefix." - ".$say_key; } - $say_key .= ":"; - my $difference = $length - length($say_key); + $say_key .= ":"; + my $difference = $length - length($say_key); print $THIS_FILE." ".__LINE__."; say_key: [".$say_key."], difference: [".$difference."]\n" if $test; if ($difference) { diff --git a/Anvil/Tools/System.pm b/Anvil/Tools/System.pm index 5cd4adeb..9c43b7ce 100644 --- a/Anvil/Tools/System.pm +++ b/Anvil/Tools/System.pm @@ -3864,18 +3864,62 @@ sub parse_arguments arguments => $arguments, }}); + # Convert escaped quotes, we'll convert them back at the end + my $secure = $arguments =~ /-passw/ ? 1 : 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + ">> arguments" => $secure ? $anvil->Log->is_secure($arguments) : $arguments, + }}); + $arguments =~ s/\\\"/"/g; + $arguments =~ s/\\\'/'/g; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "<< arguments" => $secure ? $anvil->Log->is_secure($arguments) : $arguments, + }}); + my $hash = {}; my $quoted = ""; my $switch = ""; my $value = ""; foreach my $arg (split/ /, $arguments) { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { arg => $arg }}); + # We're likely processing a password, so we need to decide if we're logging securely or not + # right away. + my $secure = 0; + if (($switch) && ($switch =~ /passw/)) + { + $secure = 1; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { secure => $secure }}); + + # Now we can start logging. + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + arg => $secure ? $anvil->Log->is_secure($arg) : $arg, + }}); if (($arg =~ /^'/) or ($arg =~ /^"/)) { - # Store a quoted value. - $quoted .= $arg." "; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { quoted => $quoted }}); + # If the quoted value has no spaces, store it now. + if (($arg =~ /^'(.*?)'$/) or ($arg =~ /^"(.*?)"$/)) + { + $hash->{$switch} = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "hash->{$switch}" => $secure ? $anvil->Log->is_secure($hash->{$switch}) : $hash->{$switch}, + }}); + } + elsif (($arg =~ /'$/) or ($arg =~ /"$/)) + { + $hash->{$switch} = $quoted.$arg; + $quoted = ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "hash->{$switch}" => $secure ? $anvil->Log->is_secure($hash->{$switch}) : $hash->{$switch}, + }}); + } + else + { + # Store a quoted value. + $quoted .= $arg." "; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + quoted => $secure ? $anvil->Log->is_secure($quoted) : $quoted, + }}); + } } elsif ($quoted) { @@ -3883,42 +3927,54 @@ sub parse_arguments { # Done $quoted .= $arg; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { quoted => $quoted }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + quoted => $secure ? $anvil->Log->is_secure($quoted) : $quoted, + }}); if ($quoted =~ /^'(.*)'$/) { $value = $1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { value => $value }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + value => $secure ? $anvil->Log->is_secure($value) : $value, + }}); } elsif ($quoted =~ /^"(.*)"$/) { $value = $1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { value => $value }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + value => $secure ? $anvil->Log->is_secure($value) : $value, + }}); } $hash->{$switch} = $value; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "hash->{$switch}" => $hash->{$switch} }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "hash->{$switch}" => $secure ? $anvil->Log->is_secure($hash->{$switch}) : $hash->{$switch}, + }}); $quoted = ""; $switch = ""; $value = ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - quoted => $quoted, + quoted => $secure ? $anvil->Log->is_secure($quoted) : $quoted, switch => $switch, - value => $value, + value => $secure ? $anvil->Log->is_secure($value) : $value, }}); } else { $quoted .= $arg." "; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { quoted => $quoted }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + quoted => $secure ? $anvil->Log->is_secure($quoted) : $quoted, + }}); } } elsif ($arg =~ /^-/) { if ($switch) { - $value = "#!SET!#"; + $value = "#!SET!#"; $hash->{$switch} = $value; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "hash->{$switch}" => $hash->{$switch} }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "hash->{$switch}" => $secure ? $anvil->Log->is_secure($hash->{$switch}) : $hash->{$switch}, + }}); $switch = ""; $value = ""; @@ -3928,7 +3984,7 @@ sub parse_arguments }}); } - $quoted = ""; + $quoted = ""; if ($arg =~ /^(.*?)=(.*)$/) { $switch = $1; @@ -3942,7 +3998,9 @@ sub parse_arguments else { $hash->{$switch} = $value; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "hash->{$switch}" => $hash->{$switch} }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "hash->{$switch}" => $secure ? $anvil->Log->is_secure($hash->{$switch}) : $hash->{$switch}, + }}); $switch = ""; $value = ""; @@ -3962,7 +4020,9 @@ sub parse_arguments else { $hash->{$switch} = $arg; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "hash->{$switch}" => $hash->{$switch} }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "hash->{$switch}" => $secure ? $anvil->Log->is_secure($hash->{$switch}) : $hash->{$switch}, + }}); $switch = ""; $value = ""; @@ -3973,6 +4033,19 @@ sub parse_arguments } } + foreach my $switch (sort {$a cmp $b} keys %{$hash}) + { + my $secure = $switch =~ /^passw/ ? 1 : 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + ">> hash->{$switch}" => $secure ? $anvil->Log->is_secure($hash->{$switch}) : $hash->{$switch}, + }}); + $hash->{$switch} =~ s/"/\\"/g; + $hash->{$switch} =~ s/'/\\'/g; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "<< hash->{$switch}" => $secure ? $anvil->Log->is_secure($hash->{$switch}) : $hash->{$switch}, + }}); + } + return($hash); } diff --git a/share/words.xml b/share/words.xml index e36b0d84..2d29f214 100644 --- a/share/words.xml +++ b/share/words.xml @@ -3315,6 +3315,7 @@ The error was: ======== [ Warning ] - The interface: [#!variable!interface!#] is in a bond, but it is down. The system uptime is: [#!variable!uptime!#], so it might be a problem where the interface didn't start on boot as it should have. So we're going to bring the interface up. + [ Warning ] - The IPMI stonith resource: [#!variable!resource!#] is in the role: [#!variable!role!#] (should be 'Started'). Will check the IPMI config now. From e90dae96f7290006eb2764671d28862f518735be Mon Sep 17 00:00:00 2001 From: Digimer Date: Wed, 31 Aug 2022 18:12:07 -0400 Subject: [PATCH 2/2] * In Server->shutdown_virsh(), disabled trying to resume a paused VM. Also updated the logging around not waiting for a VM to stop. * Updated anvil-safe-stop to check for VMs running, even if the cluster is stopped, when --stop-servers is used. Signed-off-by: Digimer --- Anvil/Tools/Server.pm | 34 +++++++-------- share/words.xml | 3 ++ tools/anvil-safe-stop | 96 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 114 insertions(+), 19 deletions(-) diff --git a/Anvil/Tools/Server.pm b/Anvil/Tools/Server.pm index 41f65efc..fffcf80f 100644 --- a/Anvil/Tools/Server.pm +++ b/Anvil/Tools/Server.pm @@ -1959,21 +1959,22 @@ sub shutdown_virsh } elsif ($status eq "paused") { + ### TODO: No, don't do this! The server might be migrating # The server is paused. Resume it, wait a few, then proceed with the shutdown. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0314", variables => { server => $server }}); - my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." resume $server"}); - if ($return_code) - { - # Looks like virsh isn't running. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0315", variables => { - server => $server, - return_code => $return_code, - output => $output, - }}); - $anvil->nice_exit({exit_code => 1}); - } - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0316"}); - sleep 3; +# $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0314", variables => { server => $server }}); +# my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." resume $server"}); +# if ($return_code) +# { +# # Looks like virsh isn't running. +# $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0315", variables => { +# server => $server, +# return_code => $return_code, +# output => $output, +# }}); +# $anvil->nice_exit({exit_code => 1}); +# } +# $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0316"}); +# sleep 3; } elsif ($status eq "pmsuspended") { @@ -2147,9 +2148,10 @@ WHERE { # Give up waiting. $waiting = 0; - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0426", variables => { waiting => $waiting }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { waiting => $waiting }}); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0427", variables => { + my $key = $wait_time == 1 ? "log_0727" : "log_0427"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => $key, variables => { server => $server, 'wait' => $wait_time, }}); diff --git a/share/words.xml b/share/words.xml index 2d29f214..ac1751d8 100644 --- a/share/words.xml +++ b/share/words.xml @@ -1372,6 +1372,8 @@ Note: This is a permanent action! If you protect this server again later, a full Done! The server: [#!variable!server!#] is no longer being protected on DR! The resource config file: [#!variable!config_file!#] doesn't exist locally, pulling a copy over from: [#!variable!source!#]. Re-parsing the replicated storage configuration. + The server: [#!variable!server!#] was found to be running outside the cluster. Asking it to shut down now. + The server: [#!variable!server!#] is still running two minutes after asking it to stop. It might have woken up on the first press and ignored the shutdown request (Hi Windows). Pressing the poewr button again. Starting: [#!variable!program!#]. @@ -2200,6 +2202,7 @@ The file: [#!variable!file!#] needs to be updated. The difference is: The file: [#!variable!file!#] needs to be added to the database, but since the last scan it's size grew from: [#!variable!old_size_bytes!# (#!variable!old_size_hr!#)] to: [#!variable!new_size_bytes!# (#!variable!new_size_hr!#)]. A difference of: [#!variable!difference_bytes!# (#!variable!difference_hr!#)]. It might still be being uploaded, so we'll keep checking periodocally until the size stops changing. Found the missing file: [#!variable!file!#] in the directory: [#!variable!directory!#]. Updating the database now. Deleting the hash key: [#!variable!hash_key!#]. + [ Note ] - The server: [#!variable!server!#] is not yet off, but we've been told not to wait for it to stop. The host name: [#!variable!target!#] does not resolve to an IP address. diff --git a/tools/anvil-safe-stop b/tools/anvil-safe-stop index 3bf32ed2..53c280c0 100755 --- a/tools/anvil-safe-stop +++ b/tools/anvil-safe-stop @@ -277,7 +277,20 @@ sub process_servers $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0321"}); $anvil->Job->update_progress({progress => 10, message => "job_0321"}); } - my $waiting = 1; + + # Use virsh to check for servers, in case pacemaker lies to us. + $anvil->Server->find(); + my $progress = 10; + my $waiting = 1; + my $first_try = 0; + my $second_try = 0; + my $try_again = 0; + my $server_count = keys %{$anvil->data->{server}{location}}; + my $progress_steps = $server_count ? int(35 / $server_count) : 70; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:server_count' => $server_count, + 's2:progress_steps' => $progress_steps, + }}); while ($waiting) { # Is the cluster up? @@ -286,7 +299,84 @@ sub process_servers $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); if ($problem) { - # Nope. + # Nope. Are we stopping servers? + if ($anvil->data->{switches}{'stop-servers'}) + { + # Yes, are any servers running (check virsh) + foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{server}{location}}) + { + my $status = $anvil->data->{server}{location}{$server_name}{status}; + my $host_name = $anvil->data->{server}{location}{$server_name}{host_name}; + $progress += $progress_steps; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:server_name' => $server_name, + 's2:status' => $status, + 's3:host_name' => $host_name, + 's4:progress' => $progress, + }}); + + if ($host_name eq $anvil->Get->host_name) + { + # Server is still running. + if (($status eq "running") && (not $first_try)) + { + # It's running despite the cluster being own, stop it. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0419", variables => { server => $server_name }}); + $anvil->Job->update_progress({progress => $progress, message => "job_0419,!!server!".$server_name."!!"}); + $anvil->Server->shutdown_virsh({ + debug => 2, + server => $server_name, + wait_time => 1, + }); + + $waiting = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); + } + elsif (($status eq "in shutdown") && ($try_again)) + { + # Hit the power button again. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0420", variables => { server => $server_name }}); + $anvil->Job->update_progress({progress => $progress, message => "job_0420,!!server!".$server_name."!!"}); + $anvil->Server->shutdown_virsh({ + debug => 2, + server => $server_name, + wait_time => 1, + }); + + $waiting = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); + } + } + } + + if ($waiting) + { + if (not $first_try) + { + $first_try = time; + $second_try = $first_try + 120; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + first_try => $first_try, + second_try => $second_try, + }}); + } + elsif ($try_again) + { + $try_again = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { try_again => $try_again }}); + } + elsif (($second_try) && (time > $second_try)) + { + $try_again = 1; + $second_try = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + second_try => $second_try, + try_again => $try_again, + }}); + } + } + } + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0313"}); $anvil->Job->update_progress({progress => 80, message => "job_0313"}); } @@ -304,7 +394,7 @@ sub process_servers $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:server' => $server, 's2:status' => $status, - 's2:host_name' => $host_name, + 's3:host_name' => $host_name, 's4:role' => $role, 's5:active' => $active, }});