diff --git a/share/words.xml b/share/words.xml index 8c20d726..0fd2c4b4 100644 --- a/share/words.xml +++ b/share/words.xml @@ -1270,8 +1270,7 @@ It will take time for it to initialize, please be patient. Do you want to connect the DR host for the server: [#!variable!server!#]? Note: Depending on the disk write load and storage network speed to the DR host, - this could cause reduced disk write performance. - + this could cause reduced disk write performance. About to connect the DR resource for the server: [#!variable!server!#]. Brought up the connection locally. Now checking that the resource is up on the nodes. Making sure the resource is up on: [#!variable!host_name!#]. @@ -1279,10 +1278,17 @@ Note: Depending on the disk write load and storage network speed to the DR host, Done! The server: [#!variable!server!#] is now connected. Do you want to disconnect the DR host for the server: [#!variable!server!#]? -Note: Once down, no further changes will be written to the DR host. - +Note: Once down, no further changes will be written to the DR host. About to disconnect the DR resource for the server: [#!variable!server!#]. Done! The server: [#!variable!server!#] is now disconnected. + +Do you want to update the DR host for the server: [#!variable!server!#]? +Note: This will connect the DR host until the disk(s) on DR are (all) UpToDate. + Depending on the disk write load and storage network speed to the DR host, + this could cause reduced disk write performance. + Still sync'ing from: [#!variable!sync_source!#] at a rate of: [#!variable!sync_speed!#/sec]. Estimated time remaining is: [#!variable!time_to_sync!#]. + Sync'ed! Bringing the resource back down now. + Waiting for the connection to come up... Starting: [#!variable!program!#]. diff --git a/tools/anvil-manage-dr b/tools/anvil-manage-dr index a669b3f6..77d51519 100755 --- a/tools/anvil-manage-dr +++ b/tools/anvil-manage-dr @@ -488,6 +488,277 @@ Exiting. { process_disconnect($anvil, $terminal); } + elsif ($anvil->data->{switches}{update}) + { + process_update($anvil, $terminal); + } + + return(0); +} + +sub process_update +{ + my ($anvil, $terminal) = @_; + + # Parse out the DRBD resource's backing the server and get their LV sizes. + $anvil->Database->get_server_definitions(); + my $anvil_uuid = $anvil->Cluster->get_anvil_uuid(); + my $anvil_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password}; + my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; + my $node1_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name}; + my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name}; + my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; + my $node2_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name}; + my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name}; + my $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; + my $dr1_host_name = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name}; + my $dr1_short_host_name = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{short_host_name}; + my $server_name = $anvil->data->{server}{'server-name'}; + my $server_uuid = $anvil->data->{server}{'server-uuid'}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + anvil_uuid => $anvil_uuid, + anvil_password => $anvil->Log->is_secure($anvil_password), + node1_host_uuid => $node1_host_uuid, + node1_host_name => $node1_host_name, + node1_short_host_name => $node1_short_host_name, + node2_host_uuid => $node2_host_uuid, + node2_host_name => $node2_host_name, + node2_short_host_name => $node2_short_host_name, + dr1_host_uuid => $dr1_host_uuid, + dr1_host_name => $dr1_host_name, + dr1_short_host_name => $dr1_short_host_name, + server_name => $server_name, + server_uuid => $server_uuid, + }}); + + ### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job. + my $record_job = 0; + if (not $anvil->data->{switches}{Yes}) + { + my $variables = { + server => $server_name, + }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0395", variables => $variables}); + $anvil->Job->update_progress({ + progress => 25, + message => "job_0395", + variables => $variables, + }); + + # Ask the user to confirm. + print "\n".$anvil->Words->string({key => "message_0021"})."\n"; + my $answer = ; + chomp $answer; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }}); + + if ($answer =~ /^y/i) + { + print $anvil->Words->string({key => "message_0175"})."\n"; + $record_job = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }}); + } + else + { + print $anvil->Words->string({key => "message_0022"})."\n"; + $anvil->nice_exit({exit_code => 0}); + } + } + elsif (not $anvil->data->{switches}{'job-uuid'}) + { + $record_job = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }}); + } + + if ($record_job) + { + my $job_data = "server=".$anvil->data->{switches}{server}."\n"; + $job_data .= "update=1\n"; + + # Register the job with the DR host. + my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ + debug => 2, + job_command => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, + job_data => $job_data, + job_name => "server::dr", + job_title => "job_0384", + job_description => "job_0385", + job_progress => 0, + job_host_uuid => $dr1_host_uuid, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); + + # Report the job UUID. + print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n"; + + $anvil->nice_exit({exit_code => 0}); + } + + # If the resource is down, bring it up. + my $variables = { server => $server_name }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0387", variables => $variables}); + $anvil->Job->update_progress({ + progress => 50, + message => "job_0387", + variables => $variables, + }); + + # Bring up the connection locally, and then also bring up the connection on the nodes, in case the + # server is down. + my $drbd_up_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." || ".$anvil->data->{path}{exe}{drbdadm}." up ".$server_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_up_call => $drbd_up_call }}); + my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_up_call}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0388", variables => $variables}); + $anvil->Job->update_progress({ + progress => 60, + message => "job_0388", + }); + foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid) + { + # "Peer" in this context is either/both nodes + next if $this_host_uuid eq $anvil->Get->host_uuid(); + my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name}; + my $peer_sn_ip = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{network}{sn1}{ip_address}; + my $variables = { host_name => $peer_host_name }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0389", variables => $variables}); + $anvil->Job->update_progress({ + progress => 70, + message => "job_0389", + variables => $variables, + }); + my ($output, $error, $return_code) = $anvil->Remote->call({ + target => $peer_sn_ip, + password => $anvil_password, + shell_call => $drbd_up_call, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + error => $error, + output => $output, + return_code => $return_code, + }}); + } + + # Now watch until out volume(s) are UpToDate + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0390"}); + $anvil->Job->update_progress({ + progress => 70, + message => "job_0390", + }); + + my $waiting = 1; + while ($waiting) + { + $anvil->DRBD->gather_data({debug => 2}); + $waiting = 0; + my $time_to_sync = ""; + my $sync_speed = ""; + my $sync_source = ""; + foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }}); + foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}}) + { + my $replication_speed = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{replication_speed}; + my $peer_role = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_role}; + my $peer_disk_state = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_disk_state}; + my $estimated_time_to_sync = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{estimated_time_to_sync}; + my $local_disk_state = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_disk_state}; + my $connection_state = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{connection_state}; + my $local_role = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_role}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + peer => $peer, + replication_speed => $replication_speed, + peer_role => $peer_role, + peer_disk_state => $peer_disk_state, + estimated_time_to_sync => $estimated_time_to_sync, + local_disk_state => $local_disk_state, + connection_state => $connection_state, + local_role => $local_role, + }}); + + if ($connection_state eq "synctarget") + { + $sync_source = $peer; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sync_source => $sync_source }}); + } + if ($replication_speed) + { + $sync_speed = $anvil->Convert->bytes_to_human_readable({'bytes' => $replication_speed}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sync_speed => $sync_speed }}); + } + if ($estimated_time_to_sync) + { + $time_to_sync = $anvil->Convert->time({'time' => $estimated_time_to_sync, translate => 1}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { time_to_sync => $time_to_sync }}); + } + + if ($local_disk_state ne "uptodate") + { + $waiting = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); + } + } + } + if ($waiting) + { + if ($sync_source) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + sync_source => $sync_source, + sync_speed => $sync_speed, + time_to_sync => $time_to_sync, + }}); + my $variables = { + sync_source => $sync_source, + sync_speed => $sync_speed, + time_to_sync => $time_to_sync, + }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0396", variables => $variables}); + $anvil->Job->update_progress({ + progress => 80, + message => "job_0396", + variables => $variables, + }); + } + else + { + $anvil->Job->update_progress({ + progress => 75, + message => "job_0398", + variables => $variables, + }); + } + sleep 5; + } + } + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0397", variables => $variables}); + $anvil->Job->update_progress({ + progress => 90, + message => "job_0397", + variables => $variables, + }); + + # Bring up the connection locally, and then also bring up the connection on the nodes, in case the + # server is down. + my $drbd_down_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." && ".$anvil->data->{path}{exe}{drbdadm}." down ".$server_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_down_call => $drbd_down_call }}); + ($output, $return_code) = $anvil->System->call({shell_call => $drbd_down_call}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); + + # Done! + $variables = { server => $server_name }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0391", variables => $variables}); + $anvil->Job->update_progress({ + progress => 100, + message => "job_0391", + variables => $variables, + }); return(0); }