diff --git a/share/words.xml b/share/words.xml index cd618daf..8c20d726 100644 --- a/share/words.xml +++ b/share/words.xml @@ -1267,6 +1267,22 @@ It will take time for it to initialize, please be patient. The job has been recorded with the UUID: [#!variable!job_uuid!#], it will start in just a moment if anvil-daemon is running. Manage DR tasks for a given server This job can protect, remove (unprotect), connect, disconnect or update (connect, sync, disconnect) a given server. + +Do you want to connect the DR host for the server: [#!variable!server!#]? +Note: Depending on the disk write load and storage network speed to the DR host, + this could cause reduced disk write performance. + + About to connect the DR resource for the server: [#!variable!server!#]. + Brought up the connection locally. Now checking that the resource is up on the nodes. + Making sure the resource is up on: [#!variable!host_name!#]. + Waiting now for the our resource to connect. + Done! The server: [#!variable!server!#] is now connected. + +Do you want to disconnect the DR host for the server: [#!variable!server!#]? +Note: Once down, no further changes will be written to the DR host. + + About to disconnect the DR resource for the server: [#!variable!server!#]. + Done! The server: [#!variable!server!#] is now disconnected. Starting: [#!variable!program!#]. diff --git a/tools/anvil-manage-dr b/tools/anvil-manage-dr index b3f13af5..a669b3f6 100755 --- a/tools/anvil-manage-dr +++ b/tools/anvil-manage-dr @@ -126,16 +126,73 @@ sub sanity_check anvil_uuid => $anvil_uuid, }}); - if (($host_type ne "node") or (not $anvil_uuid)) + # If we're doing a --protect or --remove, make sure we're a node, the cluster is up, and both nodes + # are ready. + if (($anvil->data->{switches}{protect}) or ($anvil->data->{switches}{remove})) { - # This must be run on a node active in the cluster hosting the server being managed. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0332"}); - $anvil->Job->update_progress({ - progress => 100, - message => "error_0332", - job_status => "failed", - }); - $anvil->nice_exit({exit_code => 1}); + if (($host_type ne "node") or (not $anvil_uuid)) + { + # This must be run on a node active in the cluster hosting the server being managed. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0332"}); + $anvil->Job->update_progress({ + progress => 100, + message => "error_0332", + job_status => "failed", + }); + $anvil->nice_exit({exit_code => 1}); + } + + # Can we parse the CIB? + my ($problem) = $anvil->Cluster->parse_cib(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); + if ($problem) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0336"}); + $anvil->Job->update_progress({ + progress => 0, + message => "error_0336", + job_status => "failed", + }); + $anvil->nice_exit({exit_code => 1}); + $anvil->nice_exit({exit_code => 1}); + } + + # Both nodes need to be in the cluster, are they? + if (not $anvil->data->{cib}{parsed}{'local'}{ready}) + { + # We're not a full member of the cluster yet. Please try again once we're fully in. Exiting. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0337"}); + $anvil->Job->update_progress({ + progress => 0, + message => "error_0337", + job_status => "failed", + }); + $anvil->nice_exit({exit_code => 1}); + } + + if ($anvil->data->{switches}{protect}) + { + if (not $anvil->data->{switches}{protocol}) + { + $anvil->data->{switches}{protocol} = "async"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 'switches::protocol' => $anvil->data->{switches}{protocol}, + }}); + } + elsif (($anvil->data->{switches}{protocol} ne "sync") && + ($anvil->data->{switches}{protocol} ne "async") && + ($anvil->data->{switches}{protocol} ne "long-throw")) + { + # The protocol is invalid. Please use '--help' for more information. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0342", variables => { protocol => $anvil->data->{switches}{protocol} }}); + $anvil->Job->update_progress({ + progress => 100, + message => "error_0341,!!protocol!".$anvil->data->{switches}{protocol}."!!", + job_status => "failed", + }); + $anvil->nice_exit({exit_code => 1}); + } + } } # Get the Anvil! details. @@ -156,7 +213,7 @@ sub sanity_check $anvil->nice_exit({exit_code => 1}); } - # Can we access DR? + # Can we access DR, if we're not the DR host? my $password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password}; my $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; my $dr1_host_name = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name}; @@ -164,71 +221,46 @@ sub sanity_check debug => 2, host => $dr1_host_name, }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - password => $anvil->Log->is_secure($password), - dr1_host_uuid => $dr1_host_uuid, - dr1_host_name => $dr1_host_name, - dr_ip => $dr_ip, - }}); - if ((not $dr_ip) or ($dr_ip eq "!!error!!")) - { - # Failed to find an IP we can access the DR host. Has it been configured? Is it running? Exiting. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "error_0334", variables => { host_name => $dr1_host_name }}); - $anvil->Job->update_progress({ - progress => 0, - message => "error_0334,!!host_name!".$dr1_host_name."!!", - job_status => "failed", - }); - $anvil->nice_exit({exit_code => 1}); - } - - # Test access. - my $access = $anvil->Remote->test_access({ - target => $dr_ip, - password => $password, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }}); - if (not $access) + if ($dr1_host_uuid ne $anvil->Get->host_uuid) { - # Failed to access the DR host. Is it running? Exiting. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0335", variables => { - host_name => $dr1_host_name, - ip_address => $dr_ip, + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + password => $anvil->Log->is_secure($password), + dr1_host_uuid => $dr1_host_uuid, + dr1_host_name => $dr1_host_name, + dr_ip => $dr_ip, }}); - $anvil->Job->update_progress({ - progress => 0, - message => "error_0335,!!host_name!".$dr1_host_name."!!,!!ip_address!".$dr_ip."!!", - job_status => "failed", - }); - $anvil->nice_exit({exit_code => 1}); - } - - # Can we parse the CIB? - my ($problem) = $anvil->Cluster->parse_cib(); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); - if ($problem) - { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0336"}); - $anvil->Job->update_progress({ - progress => 0, - message => "error_0336", - job_status => "failed", - }); - $anvil->nice_exit({exit_code => 1}); - $anvil->nice_exit({exit_code => 1}); - } - - # Both nodes need to be in the cluster, are they? - if (not $anvil->data->{cib}{parsed}{'local'}{ready}) - { - # We're not a full member of the cluster yet. Please try again once we're fully in. Exiting. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0337"}); - $anvil->Job->update_progress({ - progress => 0, - message => "error_0337", - job_status => "failed", + if ((not $dr_ip) or ($dr_ip eq "!!error!!")) + { + # Failed to find an IP we can access the DR host. Has it been configured? Is it running? Exiting. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "error_0334", variables => { host_name => $dr1_host_name }}); + $anvil->Job->update_progress({ + progress => 0, + message => "error_0334,!!host_name!".$dr1_host_name."!!", + job_status => "failed", + }); + $anvil->nice_exit({exit_code => 1}); + } + + # Test access. + my $access = $anvil->Remote->test_access({ + target => $dr_ip, + password => $password, }); - $anvil->nice_exit({exit_code => 1}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }}); + if (not $access) + { + # Failed to access the DR host. Is it running? Exiting. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0335", variables => { + host_name => $dr1_host_name, + ip_address => $dr_ip, + }}); + $anvil->Job->update_progress({ + progress => 0, + message => "error_0335,!!host_name!".$dr1_host_name."!!,!!ip_address!".$dr_ip."!!", + job_status => "failed", + }); + $anvil->nice_exit({exit_code => 1}); + } } ### TODO: We can queue a job to update the peer later, there's no real need, in the long run, for the @@ -318,27 +350,6 @@ sub sanity_check $anvil->nice_exit({exit_code => 1}); } - if (not $anvil->data->{switches}{protocol}) - { - $anvil->data->{switches}{protocol} = "async"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - 'switches::protocol' => $anvil->data->{switches}{protocol}, - }}); - } - elsif (($anvil->data->{switches}{protocol} ne "sync") && - ($anvil->data->{switches}{protocol} ne "async") && - ($anvil->data->{switches}{protocol} ne "long-throw")) - { - # The protocol is invalid. Please use '--help' for more information. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0342", variables => { protocol => $anvil->data->{switches}{protocol} }}); - $anvil->Job->update_progress({ - progress => 100, - message => "error_0341,!!protocol!".$anvil->data->{switches}{protocol}."!!", - job_status => "failed", - }); - $anvil->nice_exit({exit_code => 1}); - } - # Are we being asked to actuall do something? if (((not $anvil->data->{switches}{'connect'}) && (not $anvil->data->{switches}{disconnect}) && @@ -431,6 +442,10 @@ Options (all require --server ); This removes the DR image from the DR host for the server, freeing up space on DR but removing the protection afforded by DR. + --server + + This is the name or UUID of the server being worked on. + --update This tells the DR to be connected and sync, Once the volume(s) on DR are 'UpToDate', the connection @@ -465,6 +480,364 @@ Exiting. { process_protect($anvil, $terminal); } + elsif ($anvil->data->{switches}{'connect'}) + { + process_connect($anvil, $terminal); + } + elsif ($anvil->data->{switches}{disconnect}) + { + process_disconnect($anvil, $terminal); + } + + return(0); +} + +sub process_disconnect +{ + my ($anvil, $terminal) = @_; + + # Parse out the DRBD resource's backing the server and get their LV sizes. + $anvil->Database->get_server_definitions(); + my $anvil_uuid = $anvil->Cluster->get_anvil_uuid(); + my $anvil_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password}; + my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; + my $node1_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name}; + my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name}; + my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; + my $node2_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name}; + my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name}; + my $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; + my $dr1_host_name = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name}; + my $dr1_short_host_name = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{short_host_name}; + my $server_name = $anvil->data->{server}{'server-name'}; + my $server_uuid = $anvil->data->{server}{'server-uuid'}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + anvil_uuid => $anvil_uuid, + anvil_password => $anvil->Log->is_secure($anvil_password), + node1_host_uuid => $node1_host_uuid, + node1_host_name => $node1_host_name, + node1_short_host_name => $node1_short_host_name, + node2_host_uuid => $node2_host_uuid, + node2_host_name => $node2_host_name, + node2_short_host_name => $node2_short_host_name, + dr1_host_uuid => $dr1_host_uuid, + dr1_host_name => $dr1_host_name, + dr1_short_host_name => $dr1_short_host_name, + server_name => $server_name, + server_uuid => $server_uuid, + }}); + + ### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job. + my $record_job = 0; + if (not $anvil->data->{switches}{Yes}) + { + my $variables = { + server => $server_name, + }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0392", variables => $variables}); + $anvil->Job->update_progress({ + progress => 25, + message => "job_0392", + variables => $variables, + }); + + # Ask the user to confirm. + print "\n".$anvil->Words->string({key => "message_0021"})."\n"; + my $answer = ; + chomp $answer; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }}); + + if ($answer =~ /^y/i) + { + print $anvil->Words->string({key => "message_0175"})."\n"; + $record_job = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }}); + } + else + { + print $anvil->Words->string({key => "message_0022"})."\n"; + $anvil->nice_exit({exit_code => 0}); + } + } + elsif (not $anvil->data->{switches}{'job-uuid'}) + { + $record_job = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }}); + } + + if ($record_job) + { + my $job_data = "server=".$anvil->data->{switches}{server}."\n"; + $job_data .= "disconnect=1\n"; + + # Register the job with the DR host. + my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ + debug => 2, + job_command => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, + job_data => $job_data, + job_name => "server::dr", + job_title => "job_0384", + job_description => "job_0385", + job_progress => 0, + job_host_uuid => $dr1_host_uuid, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); + + # Report the job UUID. + print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n"; + + $anvil->nice_exit({exit_code => 0}); + } + + # If the resource is down, bring it up. + my $variables = { server => $server_name }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0393", variables => $variables}); + $anvil->Job->update_progress({ + progress => 50, + message => "job_0393", + variables => $variables, + }); + + # Bring up the connection locally, and then also bring up the connection on the nodes, in case the + # server is down. + my $drbd_down_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." && ".$anvil->data->{path}{exe}{drbdadm}." down ".$server_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_down_call => $drbd_down_call }}); + my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_down_call}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); + + # Done! + $variables = { server => $server_name }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0394", variables => $variables}); + $anvil->Job->update_progress({ + progress => 100, + message => "job_0394", + variables => $variables, + }); + + return(0); +} + +sub process_connect +{ + my ($anvil, $terminal) = @_; + + # Parse out the DRBD resource's backing the server and get their LV sizes. + $anvil->Database->get_server_definitions(); + my $anvil_uuid = $anvil->Cluster->get_anvil_uuid(); + my $anvil_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password}; + my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; + my $node1_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name}; + my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name}; + my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; + my $node2_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name}; + my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name}; + my $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; + my $dr1_host_name = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name}; + my $dr1_short_host_name = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{short_host_name}; + my $server_name = $anvil->data->{server}{'server-name'}; + my $server_uuid = $anvil->data->{server}{'server-uuid'}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + anvil_uuid => $anvil_uuid, + anvil_password => $anvil->Log->is_secure($anvil_password), + node1_host_uuid => $node1_host_uuid, + node1_host_name => $node1_host_name, + node1_short_host_name => $node1_short_host_name, + node2_host_uuid => $node2_host_uuid, + node2_host_name => $node2_host_name, + node2_short_host_name => $node2_short_host_name, + dr1_host_uuid => $dr1_host_uuid, + dr1_host_name => $dr1_host_name, + dr1_short_host_name => $dr1_short_host_name, + server_name => $server_name, + server_uuid => $server_uuid, + }}); + + ### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job. + my $record_job = 0; + if (not $anvil->data->{switches}{Yes}) + { + my $variables = { + server => $server_name, + }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0386", variables => $variables}); + $anvil->Job->update_progress({ + progress => 25, + message => "job_0386", + variables => $variables, + }); + + # Ask the user to confirm. + print "\n".$anvil->Words->string({key => "message_0021"})."\n"; + my $answer = ; + chomp $answer; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }}); + + if ($answer =~ /^y/i) + { + print $anvil->Words->string({key => "message_0175"})."\n"; + $record_job = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }}); + } + else + { + print $anvil->Words->string({key => "message_0022"})."\n"; + $anvil->nice_exit({exit_code => 0}); + } + } + elsif (not $anvil->data->{switches}{'job-uuid'}) + { + $record_job = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }}); + } + + if ($record_job) + { + my $job_data = "server=".$anvil->data->{switches}{server}."\n"; + $job_data .= "connect=1\n"; + + # Register the job with the DR host. + my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ + debug => 2, + job_command => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, + job_data => $job_data, + job_name => "server::dr", + job_title => "job_0384", + job_description => "job_0385", + job_progress => 0, + job_host_uuid => $dr1_host_uuid, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); + + # Report the job UUID. + print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n"; + + $anvil->nice_exit({exit_code => 0}); + } + + # If the resource is down, bring it up. + my $variables = { server => $server_name }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0387", variables => $variables}); + $anvil->Job->update_progress({ + progress => 50, + message => "job_0387", + variables => $variables, + }); + + # Bring up the connection locally, and then also bring up the connection on the nodes, in case the + # server is down. + my $drbd_up_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." || ".$anvil->data->{path}{exe}{drbdadm}." up ".$server_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_up_call => $drbd_up_call }}); + my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_up_call}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0388", variables => $variables}); + $anvil->Job->update_progress({ + progress => 60, + message => "job_0388", + }); + foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid) + { + # "Peer" in this context is either/both nodes + next if $this_host_uuid eq $anvil->Get->host_uuid(); + my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name}; + my $peer_sn_ip = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{network}{sn1}{ip_address}; + $variables = { host_name => $peer_host_name }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0389", variables => $variables}); + $anvil->Job->update_progress({ + progress => 70, + message => "job_0389", + variables => $variables, + }); + my ($output, $error, $return_code) = $anvil->Remote->call({ + target => $peer_sn_ip, + password => $anvil_password, + shell_call => $drbd_up_call, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + error => $error, + output => $output, + return_code => $return_code, + }}); + } + + # Now watch until we connect to at least one peer. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0390"}); + $anvil->Job->update_progress({ + progress => 80, + message => "job_0390", + }); + my $waiting = 1; + while($waiting) + { + sleep 5; + $anvil->DRBD->gather_data({debug => 2}); + + $waiting = 0; + foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }}); + my $volume_up = 0; + foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}}) + { + my $replication_speed = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{replication_speed}; + my $peer_role = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_role}; + my $peer_disk_state = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_disk_state}; + my $estimated_time_to_sync = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{estimated_time_to_sync}; + my $local_disk_state = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_disk_state}; + my $connection_state = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{connection_state}; + my $local_role = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_role}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + peer => $peer, + replication_speed => $replication_speed, + peer_role => $peer_role, + peer_disk_state => $peer_disk_state, + estimated_time_to_sync => $estimated_time_to_sync, + local_disk_state => $local_disk_state, + connection_state => $connection_state, + local_role => $local_role, + }}); + + if ($connection_state eq "established") + { + $volume_up = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume_up => $volume_up }}); + } + last if $volume_up; + } + + if (not $volume_up) + { + $waiting = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); + } + } + + if (not $waiting) + { + # We're ready. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0379"}); + $anvil->Job->update_progress({ + progress => 95, + message => "job_0379", + }); + $waiting = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); + } + } + + # Done! + $variables = { server => $server_name }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0391", variables => $variables}); + $anvil->Job->update_progress({ + progress => 100, + message => "job_0391", + variables => $variables, + }); return(0); }