From c8ee75420dff87573cbc34cf8313b3c7443bbacd Mon Sep 17 00:00:00 2001 From: Digimer Date: Thu, 1 Sep 2022 16:09:37 -0400 Subject: [PATCH 1/3] * Updated anvil-manage-dr to check if a server is protected before processing a --connect or --disconnect request. Also made it smarter if an attempt to connect a resource fails. Signed-off-by: Digimer --- share/words.xml | 7 ++++- tools/anvil-manage-dr | 63 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/share/words.xml b/share/words.xml index ac1751d8..f870eb18 100644 --- a/share/words.xml +++ b/share/words.xml @@ -521,6 +521,11 @@ The definition data passed in was: ]]> [ Error ] - Failed to wipe and delete the logical volume: [#!variable!local_lv!#] that was volume number: [#!variable!volume!#] under the server: [#!variable!server!#]. There was a problem deleting: [#!variable!config_file!#]. The rest of the process completed successfully. Please manually remove this file if it still exists. + [ Error ] - Failed to connect the DRBD resource. Expected return code '0', but got: [#!variable!return_code!#]. The error output, if anything, was +==== +#!variable!output!# +==== + Can not (dis)connect the server: [#!variable!server!#] as the resource config file: [#!variable!config_file!#] doesn't exist. Do you need to '--protect' it? @@ -1320,7 +1325,7 @@ Note: Depending on the disk write load and storage network speed to the DR host, About to connect the DR resource for the server: [#!variable!server!#]. Brought up the connection locally. Now checking that the resource is up on the nodes. Making sure the resource is up on: [#!variable!host_name!#]. - Waiting now for the our resource to connect. + Waiting now for the resource to connect. Done! The server: [#!variable!server!#] is now connected. Do you want to disconnect the DR host for the server: [#!variable!server!#]? diff --git a/tools/anvil-manage-dr b/tools/anvil-manage-dr index 1bcf4267..15b469be 100755 --- a/tools/anvil-manage-dr +++ b/tools/anvil-manage-dr @@ -59,7 +59,7 @@ if (not $anvil->data->{switches}{'job-uuid'}) my $termios = new POSIX::Termios; $termios->getattr; my $ospeed = $termios->getospeed; - $terminal = Tgetent Term::Cap { TERM => undef, OSPEED => $ospeed }; + $terminal = Tgetent Term::Cap { TERM => undef, OSPEED => $ospeed }; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { terminal => $terminal }}); $terminal->Trequire(qw/ce ku kd/); @@ -94,6 +94,31 @@ sub sanity_check anvil_uuid => $anvil_uuid, }}); + # If we're (dis}connecting, is the server being protected in the first place? + if (($anvil->data->{switches}{'connect'}) or ($anvil->data->{switches}{'disconnect'})) + { + # Is this server configured to be protected? + my $server_name = $anvil->data->{switches}{server}; + my $config_file = $anvil->data->{path}{directories}{drbd_resources}."/".$server_name.".res"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { config_file => $config_file }}); + if (not -e $config_file) + { + # Can't connect an unprotected server. + my $variables = { + server => $server_name, + config_file => $config_file, + }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0371", variables => $variables}); + $anvil->Job->update_progress({ + progress => 100, + message => "error_0371", + variables => $variables, + job_status => "failed", + }); + } + $anvil->nice_exit({exit_code => 1}); + } + # If we're doing a --protect or --remove, make sure we're a node, the cluster is up, and both nodes # are ready. if (($anvil->data->{switches}{protect}) or ($anvil->data->{switches}{remove})) @@ -240,9 +265,9 @@ sub sanity_check ### peer to be online. # If we're protecting or removing a server from DR, the peer needs to be up. if ((($anvil->data->{switches}{protect}) or - ($anvil->data->{switches}{remove}) or - ($anvil->data->{switches}{protocol})) && - (not $anvil->data->{cib}{parsed}{peer}{ready})) + ($anvil->data->{switches}{remove}) or + ($anvil->data->{switches}{protocol})) && + (not $anvil->data->{cib}{parsed}{peer}{ready})) { if ($anvil->data->{switches}{protect}) { @@ -250,8 +275,8 @@ sub sanity_check # isn't at this time. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0338"}); $anvil->Job->update_progress({ - progress => 0, - message => "error_0338", + progress => 0, + message => "error_0338", job_status => "failed", }); } @@ -261,8 +286,8 @@ sub sanity_check # this time. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0339"}); $anvil->Job->update_progress({ - progress => 0, - message => "error_0339", + progress => 0, + message => "error_0339", job_status => "failed", }); } @@ -279,8 +304,8 @@ sub sanity_check # Please specify the server to manager using '--server '. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0340"}); $anvil->Job->update_progress({ - progress => 100, - message => "error_0340", + progress => 100, + message => "error_0340", job_status => "failed", }); $anvil->nice_exit({exit_code => 1}); @@ -986,6 +1011,24 @@ sub process_connect output => $output, return_code => $return_code, }}); + + # If this was an error, abort. + if ($return_code) + { + # Failed. + my $variables = { + return_code => $return_code, + output => $output, + }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0370", variables => $variables}); + $anvil->Job->update_progress({ + progress => 100, + message => "error_0370", + variables => $variables, + }); + $anvil->nice_exit({exit_code => 1}); + } + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0388", variables => $variables}); $anvil->Job->update_progress({ progress => 60, From 2fab7bc1b7af5f56c6ddef1686b6be1cd6e8385d Mon Sep 17 00:00:00 2001 From: Digimer Date: Wed, 21 Sep 2022 23:35:06 -0400 Subject: [PATCH 2/3] This adds support (testing needed) for "Long-Throw" DR; which is a wrapper for using 'drbd-proxy' to provide larger transmit buffers so slow/high-latency DR hosts. * Created DRBD->check_proxy_license() to do (some level of) sanity checks on the DRBD proxy license file. * Updated DRBD->gather_data() to parse out the inside and outside ports for resource configs using proxy. * Reworked DRBD->get_next_resource() to return 1, 3 or 7 TCP ports depending, with the new long_throw_ports parameter triggering the 7 ports. * Added 'tcpdump' to the anvil-core requires list. * Reworked scan-drbd to record the ports used in proxy configs. This required adding a check to change the 'scan_drbd_peer_tcp_port' column type to 'text' to support CSVs. * Reworked anvil-manage-dr (needs testing!) to support "long-throw" DR configs. * Updated anvil-safe-stop to check if the nodes are in the cluster before trying to migrate. Signed-off-by: Digimer --- Anvil/Tools.pm | 1 + Anvil/Tools/DRBD.pm | 433 ++++++++++++++++++++---- anvil.spec.in | 1 + scancore-agents/scan-drbd/scan-drbd | 64 +++- scancore-agents/scan-drbd/scan-drbd.sql | 4 +- share/words.xml | 40 ++- tools/anvil-manage-dr | 235 ++++++++++--- tools/anvil-provision-server | 2 + tools/anvil-safe-stop | 61 +++- 9 files changed, 712 insertions(+), 129 deletions(-) diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index c82b803f..a5670f61 100644 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -1043,6 +1043,7 @@ sub _set_paths 'corosync.conf' => "/etc/corosync/corosync.conf", 'dhcpd.conf' => "/etc/dhcp/dhcpd.conf", 'dnf.conf' => "/etc/dnf/dnf.conf", + 'drbd-proxy.license' => "/etc/drbd-proxy.license", 'firewalld.conf' => "/etc/firewalld/firewalld.conf", 'global-common.conf' => "/etc/drbd.d/global_common.conf", hostname => "/etc/hostname", diff --git a/Anvil/Tools/DRBD.pm b/Anvil/Tools/DRBD.pm index 9ac13120..ebcddd8e 100644 --- a/Anvil/Tools/DRBD.pm +++ b/Anvil/Tools/DRBD.pm @@ -17,6 +17,7 @@ my $THIS_FILE = "DRBD.pm"; # allow_two_primaries # check_if_syncsource # check_if_synctarget +# check_proxy_license # delete_resource # gather_data # get_devices @@ -347,6 +348,144 @@ sub check_if_synctarget } +=head2 check_proxy_license + +This method checks to see if the DRBD Proxy license file exists and _appears_ correct. If things look good, C<< 0 >> is returned. If there is a problem, C<< 1 >> is returned. + +=cut +sub check_proxy_license +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "DRBD->check_proxy_license()" }}); + + if (not -e $anvil->data->{path}{configs}{'drbd-proxy.license'}) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0728"}); + return(1); + } + + # Read in the file. + my $wildcard_mac = 0; + my $problem = 0; + my $owner = ""; + my $expiry_date = 0; + my $mac_addresses = []; + my $features = ""; + my $signature = ""; + my $license_body = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'drbd-proxy.license'}}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { license_body => $license_body }}); + foreach my $line (split/\n/, $license_body) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }}); + + if ($line =~ /^owner: (.*)$/) + { + $owner = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { owner => $owner }}); + next; + } + if ($line =~ /^expiry-date: (.*)$/) + { + $expiry_date = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { expiry_date => $expiry_date }}); + next; + } + if ($line =~ /^mac-address: (.*)$/) + { + my $this_mac = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { this_mac => $this_mac }}); + + if ($this_mac eq "00:00:00:00:00:00") + { + $wildcard_mac = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { wildcard_mac => $wildcard_mac }}); + } + + push @{$mac_addresses}, $this_mac; + next; + } + if ($line =~ /^features: (.*)$/) + { + $features = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { features => $features }}); + next; + } + if ($line =~ /^signature: (.*)$/) + { + $signature = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { signature => $signature }}); + next; + } + } + + if ((not $owner) or + (not $expiry_date) or + (not $signature)) + { + # Appears to not be a valid license file. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0731"}); + return(1); + } + + if (time >= $expiry_date) + { + # The license has expired. + $problem = 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0729"}); + } + + if (not $wildcard_mac) + { + # Loop through all MACs on this system and see if one matches the license. + my $match = 0; + my $host = $anvil->Get->short_host_name(); + my $mac_count = @{$mac_addresses}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + host => $host, + mac_count => $mac_count, + }}); + + if (not $mac_count) + { + } + + foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{$host}{interface}}) + { + my $mac_address = $anvil->data->{network}{$host}{interface}{$in_iface}{mac_address}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:in_iface' => $in_iface, + 's2:mac_address' => $mac_address, + }}); + + foreach my $licensed_mac (@{$mac_addresses}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { licensed_mac => $licensed_mac }}); + + if (lc($mac_address) eq lc($licensed_mac)) + { + $match = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { match => $match }}); + last; + } + } + last if $match; + } + + if (not $match) + { + # MACs don't match. + $problem = 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0730"}); + } + } + + return($problem); +} + + =head2 delete_resource This method deletes an entire resource. It does this by looping through the volumes configured in a resource and deleting them one after the other (even if there is only one volume). @@ -578,6 +717,13 @@ sub gather_data } } + my $local_host_name = $anvil->Get->host_name; + my $local_short_host_name = $anvil->Get->short_host_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + local_host_name => $local_host_name, + local_short_host_name => $local_short_host_name, + }}); + local $@; my $dom = eval { XML::LibXML->load_xml(string => $xml); }; if ($@) @@ -598,13 +744,6 @@ sub gather_data $anvil->data->{new}{scan_drbd}{scan_drbd_flush_md} = 1; $anvil->data->{new}{scan_drbd}{scan_drbd_timeout} = 6; # Default is '60', 6 seconds $anvil->data->{new}{scan_drbd}{scan_drbd_total_sync_speed} = 0; - - my $local_host_name = $anvil->Get->host_name; - my $local_short_host_name = $anvil->Get->short_host_name; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - local_host_name => $local_host_name, - local_short_host_name => $local_short_host_name, - }}); foreach my $name ($dom->findnodes('/config/common/section')) { my $section = $name->{name}; @@ -736,7 +875,7 @@ sub gather_data } else { - $host2_name = $this_host_name; + $host2_name = $this_host_name; $host2_ip_address = $host->findvalue('./address'); $host2_tcp_port = $host->findvalue('./address/@port'); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { @@ -755,6 +894,37 @@ sub gather_data "s3:new::resource::${resource}::host1_to_host2::${host1_name}::${host2_name}::host2_ip_address" => $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_ip_address}, "s4:new::resource::${resource}::host1_to_host2::${host1_name}::${host2_name}::host2_tcp_port" => $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_tcp_port}, }}); + + foreach my $proxy ($host->findnodes('./proxy')) + { + my $host_name = $proxy->{hostname}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_name => $host_name }}); + + # This should always be the target, but lets be safe/careful + next if $host_name ne $host2_name; + + $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_ip_address} = $proxy->findvalue('./inside'); + $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port} = $proxy->findvalue('./inside/@port'); + $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_ip_address} = $proxy->findvalue('./outside'); + $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_tcp_port} = $proxy->findvalue('./outside/@port'); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:new::resource::${resource}::host1_to_host2::${host1_name}::${host2_name}::host2_inside_ip_address" => $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_ip_address}, + "s2:new::resource::${resource}::host1_to_host2::${host1_name}::${host2_name}::host2_inside_tcp_port" => $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port}, + "s3:new::resource::${resource}::host1_to_host2::${host1_name}::${host2_name}::host2_outside_ip_address" => $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_ip_address}, + "s4:new::resource::${resource}::host1_to_host2::${host1_name}::${host2_name}::host2_outside_tcp_port" => $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_tcp_port}, + }}); + + $anvil->data->{new}{resource}{$resource}{proxy}{$host_name}{inside}{ip_address} = $proxy->findvalue('./inside'); + $anvil->data->{new}{resource}{$resource}{proxy}{$host_name}{inside}{tcp_port} = $proxy->findvalue('./inside/@port'); + $anvil->data->{new}{resource}{$resource}{proxy}{$host_name}{outside}{ip_address} = $proxy->findvalue('./outside'); + $anvil->data->{new}{resource}{$resource}{proxy}{$host_name}{outside}{tcp_port} = $proxy->findvalue('./outside/@port'); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "new::resource::${resource}::proxy::${host_name}::inside::ip_address" => $anvil->data->{new}{resource}{$resource}{proxy}{$host_name}{inside}{ip_address}, + "new::resource::${resource}::proxy::${host_name}::inside::tcp_port" => $anvil->data->{new}{resource}{$resource}{proxy}{$host_name}{inside}{tcp_port}, + "new::resource::${resource}::proxy::${host_name}::outside::ip_address" => $anvil->data->{new}{resource}{$resource}{proxy}{$host_name}{outside}{ip_address}, + "new::resource::${resource}::proxy::${host_name}::outside::tcp_port" => $anvil->data->{new}{resource}{$resource}{proxy}{$host_name}{outside}{tcp_port}, + }}); + } } # $peer = $this_host_name; @@ -1017,6 +1187,127 @@ sub gather_data "new::scan_drbd::scan_drbd_total_sync_speed" => $anvil->data->{new}{scan_drbd}{scan_drbd_total_sync_speed}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{new}{scan_drbd}{scan_drbd_total_sync_speed}}).")", }}); + # For resources using drbd-proxy, the host1_to_host2 will be using the internal IP, which we want to + # switch to the 'outside' IP. Also, the ports will need to be concatenated together as a CSV list. + foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { resource => $resource }}); + foreach my $host1_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{host1_to_host2}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host1_name => $host1_name }}); + foreach my $host2_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host2_name => $host2_name }}); + + my $host1_ip_address = $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host1_ip_address}; + my $host1_tcp_port = $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host1_tcp_port}; + my $host2_ip_address = $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_ip_address}; + my $host2_tcp_port = $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_tcp_port}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:host1_ip_address" => $host1_ip_address, + "s2:host1_tcp_port" => $host1_tcp_port, + "s3:host2_ip_address" => $host2_ip_address, + "s4:host2_tcp_port" => $host2_tcp_port, + }}); + + my $host1_tcp_ports = $host1_tcp_port.","; + my $host2_tcp_ports = $host2_tcp_port.","; + my $proxy_found = 0; + if (exists $anvil->data->{new}{resource}{$resource}{proxy}{$host1_name}) + { + $proxy_found = 1; + my $host1_inside_ip_address = $anvil->data->{new}{resource}{$resource}{proxy}{$host1_name}{inside}{ip_address}; + my $host1_inside_tcp_port = $anvil->data->{new}{resource}{$resource}{proxy}{$host1_name}{inside}{tcp_port}; + my $host1_outside_ip_address = $anvil->data->{new}{resource}{$resource}{proxy}{$host1_name}{outside}{ip_address}; + my $host1_outside_tcp_port = $anvil->data->{new}{resource}{$resource}{proxy}{$host1_name}{outside}{tcp_port}; + $host1_tcp_ports .= $host1_inside_tcp_port.",".$host1_outside_tcp_port; + $host1_ip_address = $host1_outside_ip_address; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:proxy_found" => $proxy_found, + "s2:host1_inside_ip_address" => $host1_inside_ip_address, + "s3:host1_inside_tcp_port" => $host1_inside_tcp_port, + "s4:host1_outside_ip_address" => $host1_outside_ip_address, + "s5:host1_outside_tcp_port" => $host1_outside_tcp_port, + "s6:host1_tcp_ports" => $host1_tcp_ports, + "s7:host1_ip_address" => $host1_ip_address, + }}); + } + if (exists $anvil->data->{new}{resource}{$resource}{proxy}{$host2_name}) + { + $proxy_found = 1; + my $host2_inside_ip_address = $anvil->data->{new}{resource}{$resource}{proxy}{$host2_name}{inside}{ip_address}; + my $host2_inside_tcp_port = $anvil->data->{new}{resource}{$resource}{proxy}{$host2_name}{inside}{tcp_port}; + my $host2_outside_ip_address = $anvil->data->{new}{resource}{$resource}{proxy}{$host2_name}{outside}{ip_address}; + my $host2_outside_tcp_port = $anvil->data->{new}{resource}{$resource}{proxy}{$host2_name}{outside}{tcp_port}; + $host2_tcp_ports .= $host2_inside_tcp_port.",".$host2_outside_tcp_port; + $host2_ip_address = $host2_outside_ip_address; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:proxy_found" => $proxy_found, + "s2:host2_inside_ip_address" => $host2_inside_ip_address, + "s3:host2_inside_tcp_port" => $host2_inside_tcp_port, + "s4:host2_outside_ip_address" => $host2_outside_ip_address, + "s5:host2_outside_tcp_port" => $host2_outside_tcp_port, + "s6:host2_tcp_ports" => $host2_tcp_ports, + "s7:host2_ip_address" => $host2_ip_address, + }}); + } + next if not $proxy_found; + + # Save the new info. + $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host1_ip_address} = $host1_ip_address; + $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host1_tcp_port} = $host1_tcp_ports; + $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_ip_address} = $host2_ip_address; + $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_tcp_port} = $host2_tcp_ports; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:new::resource::${resource}::host1_to_host2::${host1_name}::${host2_name}::host1_ip_address" => $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host1_ip_address}, + "s2:new::resource::${resource}::host1_to_host2::${host1_name}::${host2_name}::host1_tcp_port" => $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host1_tcp_port}, + "s3:new::resource::${resource}::host1_to_host2::${host1_name}::${host2_name}::host2_ip_address" => $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_ip_address}, + "s4:new::resource::${resource}::host1_to_host2::${host1_name}::${host2_name}::host2_tcp_port" => $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_tcp_port}, + }}); + + my $peer = ""; + if (($host1_name eq $local_short_host_name) or ($host1_name eq $local_host_name)) + { + # Our peer is host2. Is the protocol C? If so, this can't be proxy. + $peer = $host2_name; + my $peer_protocol = $anvil->data->{new}{resource}{$resource}{peer}{$peer}{protocol}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:peer" => $peer, + "s2:peer_protocol" => $peer_protocol, + }}); + next if uc($peer_protocol) eq "C"; + + # Still here? Then this is a proxy connection, update the ports and IP + $anvil->data->{new}{resource}{$resource}{peer}{$peer}{peer_ip_address} = $host2_ip_address; + $anvil->data->{new}{resource}{$resource}{peer}{$peer}{tcp_port} = $host1_tcp_ports; # Store our ports. + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:new::resource::${resource}::peer::${peer}::peer_ip_address" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{peer_ip_address}, + "s2:new::resource::${resource}::peer::${peer}::tcp_port" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{tcp_port}, + }}); + } + else + { + # Our peer is host1, Is the protocol C? If so, this can't be proxy. + $peer = $host1_name; + my $peer_protocol = $anvil->data->{new}{resource}{$resource}{peer}{$peer}{protocol}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:peer" => $peer, + "s2:peer_protocol" => $peer_protocol, + }}); + next if uc($peer_protocol) eq "C"; + + # Still here? Then this is a proxy connection, update the ports and IP + $anvil->data->{new}{resource}{$resource}{peer}{$peer}{peer_ip_address} = $host1_ip_address; + $anvil->data->{new}{resource}{$resource}{peer}{$peer}{tcp_port} = $host2_tcp_ports; # Store out ports + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:new::resource::${resource}::peer::${peer}::peer_ip_address" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{peer_ip_address}, + "s2:new::resource::${resource}::peer::${peer}::tcp_port" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{tcp_port}, + }}); + } + } + } + } + return(0); } @@ -1399,6 +1690,10 @@ This is the Anvil! in which we're looking for the next free resources. It's requ If set, the 'free_port' returned will be a comma-separated pair of TCP ports. This is meant to help find two TCP ports needed to connect a resource from both nodes to a DR host. +=head3 long_throw_ports (optional, default '0') + +If set, the 'free_port' returned will be a comma-separated list of seven TCP ports needed for a full B<< Long Throw >> configuration. + =head3 resource_name (optional) If this is set, and the resource is found to already exist, the first DRBD minor number and first used TCP port are returned. Alternatively, if C<< force_unique >> is set to C<< 1 >>, and the resource is found to exist, empty strings are returned. @@ -1416,15 +1711,20 @@ sub get_next_resource my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "DRBD->get_next_resource()" }}); - my $anvil_uuid = defined $parameter->{anvil_uuid} ? $parameter->{anvil_uuid} : ""; - my $dr_tcp_ports = defined $parameter->{dr_tcp_ports} ? $parameter->{dr_tcp_ports} : ""; - my $resource_name = defined $parameter->{resource_name} ? $parameter->{resource_name} : ""; - my $force_unique = defined $parameter->{force_unique} ? $parameter->{force_unique} : 0; + ### TODO: Cache results in the states or variables table and don't reuse ports given out for five + ### minutes. If the user batches a series of calls, TCP ports / minor numbers could be offered + ### multiple times. + my $anvil_uuid = defined $parameter->{anvil_uuid} ? $parameter->{anvil_uuid} : ""; + my $dr_tcp_ports = defined $parameter->{dr_tcp_ports} ? $parameter->{dr_tcp_ports} : ""; + my $long_throw_ports = defined $parameter->{long_throw_ports} ? $parameter->{long_throw_ports} : ""; + my $resource_name = defined $parameter->{resource_name} ? $parameter->{resource_name} : ""; + my $force_unique = defined $parameter->{force_unique} ? $parameter->{force_unique} : 0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - anvil_uuid => $anvil_uuid, - dr_tcp_ports => $dr_tcp_ports, - resource_name => $resource_name, - force_unique => $force_unique, + anvil_uuid => $anvil_uuid, + dr_tcp_ports => $dr_tcp_ports, + long_throw_ports => $long_throw_ports, + resource_name => $resource_name, + force_unique => $force_unique, }}); # If we weren't passed an anvil_uuid, see if we can find one locally @@ -1449,8 +1749,6 @@ sub get_next_resource # Read in the resource information from both nodes. They _should_ be identical, but that's not 100% # certain. - my $free_minor = ""; - my $free_port = ""; my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; my $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; @@ -1545,12 +1843,20 @@ ORDER BY }}); $anvil->data->{drbd}{used_resources}{minor}{$scan_drbd_volume_device_minor}{used} = 1; - $anvil->data->{drbd}{used_resources}{tcp_port}{$scan_drbd_peer_tcp_port}{used} = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "drbd::used_resources::minor::${scan_drbd_volume_device_minor}::used" => $anvil->data->{drbd}{used_resources}{minor}{$scan_drbd_volume_device_minor}{used}, - "drbd::used_resources::tcp_port::${scan_drbd_peer_tcp_port}::used" => $anvil->data->{drbd}{used_resources}{tcp_port}{$scan_drbd_peer_tcp_port}{used}, }}); + # DRBD proxy uses three ports per connection. This handles that, and still works fine for + # single TCP ports. + foreach my $tcp_port (split/,/, $scan_drbd_peer_tcp_port) + { + $anvil->data->{drbd}{used_resources}{tcp_port}{$tcp_port}{used} = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "drbd::used_resources::tcp_port::${tcp_port}::used" => $anvil->data->{drbd}{used_resources}{tcp_port}{$tcp_port}{used}, + }}); + } + if (($resource_name) && ($scan_drbd_resource_name eq $resource_name)) { # Found the resource the user was asking for. @@ -1568,10 +1874,9 @@ ORDER BY } } - # If I'm here, I need to find the next free TCP port. We'll look for the next minor number for this - # host. + # If I'm here, We'll look for the next minor number for this host. my $looking = 1; - $free_minor = 0; + my $free_minor = 0; while($looking) { if (exists $anvil->data->{drbd}{used_resources}{minor}{$free_minor}) @@ -1586,60 +1891,62 @@ ORDER BY } } - $looking = 1; - $free_port = 7788; - my $tcp_pair = ""; + # I need to find the next free TCP port. + $looking = 1; + my $check_port = 7788; + my $free_ports = ""; + my $tcp_pair = ""; + my $proxy_list = ""; + my $port_count = 0; + my $neeed_ports = 1; + if ($long_throw_ports) + { + $neeed_ports = 7; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { neeed_ports => $neeed_ports }}); + } + elsif ($dr_tcp_ports) + { + $neeed_ports = 3; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { neeed_ports => $neeed_ports }}); + } while($looking) { - if ((exists $anvil->data->{drbd}{used_resources}{tcp_port}{$free_port}) && - ($anvil->data->{drbd}{used_resources}{tcp_port}{$free_port}{used})) + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { check_port => $check_port }}); + if ((exists $anvil->data->{drbd}{used_resources}{tcp_port}{$check_port}) && + ($anvil->data->{drbd}{used_resources}{tcp_port}{$check_port}{used})) { - $free_port++; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { free_port => $free_port }}); + $check_port++; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { check_port => $check_port }}); + next; } else { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { free_port => $free_port }}); - if ($dr_tcp_ports) - { - if (not $tcp_pair) - { - $tcp_pair = $free_port; - $free_port++; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - tcp_pair => $tcp_pair, - free_port => $free_port, - }}); - } - elsif ($tcp_pair !~ /,/) - { - $tcp_pair .= ",".$free_port; - $looking = 0; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - tcp_pair => $tcp_pair, - looking => $looking, - }}); - } - } - else + # This is a free port. + $free_ports .= $check_port.","; + $port_count++; + $check_port++; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + free_ports => $free_ports, + port_count => $port_count, + }}); + + if ($port_count >= $neeed_ports) { - $looking = 0; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { looking => $looking }}); + $looking = 0; + $free_ports =~ s/,$//; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + looking => $looking, + free_ports => $free_ports, + }}); } } } - if ($dr_tcp_ports) - { - $free_port = $tcp_pair; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { free_port => $free_port }}); - } - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { free_minor => $free_minor, - free_port => $free_port, + free_ports => $free_ports, }}); - return($free_minor, $free_port); + return($free_minor, $free_ports); } diff --git a/anvil.spec.in b/anvil.spec.in index 21d89bc4..5fa0c50f 100644 --- a/anvil.spec.in +++ b/anvil.spec.in @@ -101,6 +101,7 @@ Requires: screen Requires: smartmontools Requires: syslinux Requires: tar +Requires: tcpdump Requires: tmux Requires: unzip Requires: usbutils diff --git a/scancore-agents/scan-drbd/scan-drbd b/scancore-agents/scan-drbd/scan-drbd index 9776b3fc..e070e094 100755 --- a/scancore-agents/scan-drbd/scan-drbd +++ b/scancore-agents/scan-drbd/scan-drbd @@ -14,7 +14,8 @@ # TODO: # - Create a background script that is invoked when we see a resync is running that loops every few seconds # and updates the progress in the database, exiting when the last resync is complete. -# +# - If the DRBD proxy file exists, see how long until it expires. May want to suppress alerts if no resources +# use it. use strict; use warnings; @@ -90,6 +91,9 @@ if ($anvil->DRBD->gather_data({debug => 2})) $problem = $anvil->Storage->manage_lvm_conf(); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); +# TODO: Remove this eventually. +check_schema($anvil); + read_last_scan($anvil); find_changes($anvil); @@ -104,6 +108,41 @@ $anvil->ScanCore->agent_shutdown({agent => $THIS_FILE}); # Functions # ############################################################################################################# +# In the early days, scan_drbd_peers -> scan_drbd_peer_tcp_port was type numeric. This wasn't compatible with +# drbd-proxy and had to be changed to type text to support csv port lists. +sub check_schema +{ + my ($anvil) = @_; + + my $query = "SELECT table_schema, data_type FROM information_schema.columns WHERE column_name = 'scan_drbd_peer_tcp_port';"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $schema = $row->[0]; + my $type = $row->[1]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + schema => $schema, + type => $type, + }}); + + if ($type ne "text") + { + my $query = "ALTER TABLE ".$schema.".scan_drbd_peers ALTER COLUMN scan_drbd_peer_tcp_port TYPE text;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { query => $query }}); + $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); + } + } + + return(0); +} + # This looks at the global-common.conf file and updates it, if needed. sub check_config { @@ -465,7 +504,7 @@ INSERT INTO sub process_peers { my ($anvil, $resource, $scan_drbd_resource_uuid) = @_; - + foreach my $volume_number (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{volume}}) { foreach my $peer_host_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{volume}{$volume_number}{peer}}) @@ -505,7 +544,26 @@ sub process_peers "s13:new_scan_drbd_peer_protocol" => $new_scan_drbd_peer_protocol, "s14:new_scan_drbd_peer_fencing" => $new_scan_drbd_peer_fencing, }}); - + + # Is there a proxy config? +# my $new_scan_drbd_peer_inside_ip_address = ""; +# my $new_scan_drbd_peer_inside_tcp_port = ""; +# my $new_scan_drbd_peer_outside_ip_address = ""; +# my $new_scan_drbd_peer_outside_tcp_port = ""; +# if (exists $anvil->data->{new}{resource}{$resource}{proxy}{$peer_host_name}) +# { +# $new_scan_drbd_peer_inside_ip_address = $anvil->data->{new}{resource}{$resource}{proxy}{$peer_host_name}{inside}{ip_address}; +# $new_scan_drbd_peer_inside_tcp_port = $anvil->data->{new}{resource}{$resource}{proxy}{$peer_host_name}{inside}{ip_port}; +# $new_scan_drbd_peer_outside_ip_address = $anvil->data->{new}{resource}{$resource}{proxy}{$peer_host_name}{outside}{ip_address}; +# $new_scan_drbd_peer_outside_tcp_port = $anvil->data->{new}{resource}{$resource}{proxy}{$peer_host_name}{outside}{ip_port}; +# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { +# "s1:new_scan_drbd_peer_inside_ip_address" => $new_scan_drbd_peer_inside_ip_address, +# "s2:new_scan_drbd_peer_inside_tcp_port" => $new_scan_drbd_peer_inside_tcp_port, +# "s3:new_scan_drbd_peer_outside_ip_address" => $new_scan_drbd_peer_outside_ip_address, +# "s4:new_scan_drbd_peer_outside_tcp_port" => $new_scan_drbd_peer_outside_tcp_port, +# }}); +# } + if (exists $anvil->data->{old}{resource_to_uuid}{$resource}{volume}{$volume_number}{peer}{$peer_host_name}) { # Look for changes diff --git a/scancore-agents/scan-drbd/scan-drbd.sql b/scancore-agents/scan-drbd/scan-drbd.sql index a2412081..858a1aa7 100644 --- a/scancore-agents/scan-drbd/scan-drbd.sql +++ b/scancore-agents/scan-drbd/scan-drbd.sql @@ -196,7 +196,7 @@ CREATE TABLE scan_drbd_peers ( scan_drbd_peer_replication_speed numeric not null, -- This is how many bytes per second are being copied. Set to '0' when not synchronizing. scan_drbd_peer_estimated_time_to_sync numeric not null, -- This is the number of second that is *estimated* remaining in the resync. Set to '0' when both sides are UpToDate. scan_drbd_peer_ip_address text not null, -- The (SN) IP address used for this peer. - scan_drbd_peer_tcp_port numeric not null, -- This is the port number used for this peer. + scan_drbd_peer_tcp_port text not null, -- This is the port number used for this peer. It can be a CSV for drbd-proxy connections, hence being type text scan_drbd_peer_protocol text not null, -- This is 'A' for async peers (to DR, usually) or 'C' to sync peers (node peer and sometimes DR) scan_drbd_peer_fencing text not null, -- Set to 'resource-and-stonith' for node peers and 'dont-care' for DR hosts. modified_date timestamp with time zone not null, @@ -221,7 +221,7 @@ CREATE TABLE history.scan_drbd_peers ( scan_drbd_peer_replication_speed numeric, scan_drbd_peer_estimated_time_to_sync numeric, scan_drbd_peer_ip_address text, - scan_drbd_peer_tcp_port numeric, + scan_drbd_peer_tcp_port text, scan_drbd_peer_protocol text, scan_drbd_peer_fencing text, modified_date timestamp with time zone not null diff --git a/share/words.xml b/share/words.xml index f870eb18..a0de2ad7 100644 --- a/share/words.xml +++ b/share/words.xml @@ -526,6 +526,10 @@ The definition data passed in was: #!variable!output!# ==== Can not (dis)connect the server: [#!variable!server!#] as the resource config file: [#!variable!config_file!#] doesn't exist. Do you need to '--protect' it? + We're set to migrate servers (--stop-servers not used) but both nodes are not in the cluster, so migrations would fail. Aborting. + Long-throw requires a license, and the license file is not installed, and '--license-file /path/to/drbd-proxy.license' was not passed. + The long-throw license file: [#!variable!file!#] was not found, so unable to install it. + There was a problem with the "Long-throw" lincense file. This will prevent Long-Throw DR from working. Details of the error will be recorded in the log file. @@ -802,9 +806,38 @@ sys::manage::firewall = 1 ]]> + + = 16MB per proxy connection to DRBD proxy for it to bring up a connection + memlimit #!variable!memlimiin!#M; + } + ]]> @@ -1379,6 +1412,7 @@ Note: This is a permanent action! If you protect this server again later, a full Re-parsing the replicated storage configuration. The server: [#!variable!server!#] was found to be running outside the cluster. Asking it to shut down now. The server: [#!variable!server!#] is still running two minutes after asking it to stop. It might have woken up on the first press and ignored the shutdown request (Hi Windows). Pressing the poewr button again. + Copying the Long-throw (drbd proxy) license file: [#!variable!file!#] into place. Starting: [#!variable!program!#]. @@ -1511,7 +1545,7 @@ The database connection error was: - Record Locator: [#!variable!record_locator!#] - Timestamp: .... [#!variable!modified_date!#] - [ Warning ] - There is no #!string!brand_0002!# database user set for the local machine. Please check: [#!data!path::config::anvil.conf!#]'s DB entry: [#!variable!uuid!#]. Using 'admin'. + [ Warning ] - There is no #!string!brand_0002!# database user set for the local machine. Please check: [#!data!path::configs::anvil.conf!#]'s DB entry: [#!variable!uuid!#]. Using 'admin'. Database user: [#!variable!user!#] password has been set/updated. Failed to connect to: [#!variable!target!#:#!variable!port!#], sleeping for a second and then trying again. I am not recording the alert with message_key: [#!variable!message_key!#] to the database because its log level was lower than any recipients. @@ -2208,6 +2242,10 @@ The file: [#!variable!file!#] needs to be updated. The difference is: Found the missing file: [#!variable!file!#] in the directory: [#!variable!directory!#]. Updating the database now. Deleting the hash key: [#!variable!hash_key!#]. [ Note ] - The server: [#!variable!server!#] is not yet off, but we've been told not to wait for it to stop. + The DRBD Proxy license file: [#!data!path::configs::drbd-proxy.license!#] doesn't exist. + The DRBD Proxy license file has expired. + None of the MAC sddresses in the The DRBD Proxy license file match any of the MAC addresses on this system. + The DRBD Proxy license file: [#!data!path::configs::drbd-proxy.license!#] is missing expected data or is malformed. The host name: [#!variable!target!#] does not resolve to an IP address. diff --git a/tools/anvil-manage-dr b/tools/anvil-manage-dr index 15b469be..bd218fb7 100755 --- a/tools/anvil-manage-dr +++ b/tools/anvil-manage-dr @@ -32,7 +32,7 @@ $| = 1; my $anvil = Anvil::Tools->new(); -$anvil->Get->switches({list => ["connect", "disconnect", "job-uuid", "protect", "protocol", "remove", "server", "update", "Yes"], man => $THIS_FILE}); +$anvil->Get->switches({list => ["connect", "disconnect", "job-uuid", "license-file", "protect", "protocol", "remove", "server", "update", "Yes"], man => $THIS_FILE}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }}); @@ -129,8 +129,8 @@ sub sanity_check # This must be run on a node active in the cluster hosting the server being managed. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0332"}); $anvil->Job->update_progress({ - progress => 100, - message => "error_0332", + progress => 100, + message => "error_0332", job_status => "failed", }); $anvil->nice_exit({exit_code => 1}); @@ -146,8 +146,8 @@ sub sanity_check { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0336"}); $anvil->Job->update_progress({ - progress => 0, - message => "error_0336", + progress => 0, + message => "error_0336", job_status => "failed", }); $anvil->nice_exit({exit_code => 1}); @@ -160,8 +160,8 @@ sub sanity_check # We're not a full member of the cluster yet. Please try again once we're fully in. Exiting. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0337"}); $anvil->Job->update_progress({ - progress => 0, - message => "error_0337", + progress => 0, + message => "error_0337", job_status => "failed", }); $anvil->nice_exit({exit_code => 1}); @@ -172,24 +172,79 @@ sub sanity_check { if (not $anvil->data->{switches}{protocol}) { - $anvil->data->{switches}{protocol} = "async"; + $anvil->data->{switches}{protocol} = "short-throw"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'switches::protocol' => $anvil->data->{switches}{protocol}, }}); } - elsif (($anvil->data->{switches}{protocol} ne "sync") && - ($anvil->data->{switches}{protocol} ne "async") && + elsif (($anvil->data->{switches}{protocol} ne "sync") && + ($anvil->data->{switches}{protocol} ne "short-throw") && ($anvil->data->{switches}{protocol} ne "long-throw")) { # The protocol is invalid. Please use '--help' for more information. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0342", variables => { protocol => $anvil->data->{switches}{protocol} }}); $anvil->Job->update_progress({ - progress => 100, - message => "error_0341,!!protocol!".$anvil->data->{switches}{protocol}."!!", + progress => 100, + message => "error_0341,!!protocol!".$anvil->data->{switches}{protocol}."!!", job_status => "failed", }); $anvil->nice_exit({exit_code => 1}); } + + if ($anvil->data->{switches}{protocol} eq "long-throw")) + { + # If there isn't a license file, make sure it's being provided. + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 'path::config::drbd-proxy.license' => $anvil->data->{path}{configs}{'drbd-proxy.license'}, + }}); + if (not -e $anvil->data->{path}{configs}{'drbd-proxy.license'}) + { + if (not $anvil->data->{switches}{'license-file'}) + { + # Proxy wouldn't work. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0373"}); + $anvil->Job->update_progress({ + progress => 100, + message => "error_0373", + job_status => "failed", + }); + $anvil->nice_exit({exit_code => 1}); + } + elsif (not -f $anvil->data->{switches}{'license-file'}) + { + # It was passed, but doesn't point to the file. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0374", variables => { file => $anvil->data->{switches}{'license-file'} }}); + $anvil->Job->update_progress({ + progress => 100, + message => "error_0374,!!file!".$anvil->data->{switches}{'license-file'}."!!", + job_status => "failed", + }); + $anvil->nice_exit({exit_code => 1}); + } + + # Still here? Copy the license file. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0429", variables => { file => $anvil->data->{switches}{'license-file'} }}); + my $failed = $anvil->Storage->copy_file({ + source_file => $anvil->data->{switches}{'license-file'}, + target_file => $anvil->data->{path}{configs}{'drbd-proxy.license'}, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }}); + } + + # Lastly, read in the license file to make sure it _looks_ ok. + my $problem = $anvil->DRBD->check_proxy_license({debug => 2}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); + if ($problem) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0375"}); + $anvil->Job->update_progress({ + progress => 100, + message => "error_0375", + job_status => "failed", + }); + $anvil->nice_exit({exit_code => 1}); + } + } } } @@ -204,8 +259,8 @@ sub sanity_check # This Anvil! does not seem to have a DR host. Exiting. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "error_0333"}); $anvil->Job->update_progress({ - progress => 100, - message => "error_0333", + progress => 100, + message => "error_0333", job_status => "failed", }); $anvil->nice_exit({exit_code => 1}); @@ -233,8 +288,8 @@ sub sanity_check # Failed to find an IP we can access the DR host. Has it been configured? Is it running? Exiting. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "error_0334", variables => { host_name => $dr1_host_name }}); $anvil->Job->update_progress({ - progress => 0, - message => "error_0334,!!host_name!".$dr1_host_name."!!", + progress => 0, + message => "error_0334,!!host_name!".$dr1_host_name."!!", job_status => "failed", }); $anvil->nice_exit({exit_code => 1}); @@ -254,15 +309,15 @@ sub sanity_check ip_address => $dr_ip, }}); $anvil->Job->update_progress({ - progress => 0, - message => "error_0335,!!host_name!".$dr1_host_name."!!,!!ip_address!".$dr_ip."!!", + progress => 0, + message => "error_0335,!!host_name!".$dr1_host_name."!!,!!ip_address!".$dr_ip."!!", job_status => "failed", }); $anvil->nice_exit({exit_code => 1}); } - ### TODO: We can queue a job to update the peer later, there's no real need, in the long run, for the - ### peer to be online. + ### TODO: We can queue a job to update the peer later, there's no real need, in the long run, + ### for the peer to be online. # If we're protecting or removing a server from DR, the peer needs to be up. if ((($anvil->data->{switches}{protect}) or ($anvil->data->{switches}{remove}) or @@ -271,8 +326,8 @@ sub sanity_check { if ($anvil->data->{switches}{protect}) { - # We can't setup a server to be protected unless both nodes are up, and the peer - # isn't at this time. + # We can't setup a server to be protected unless both nodes are up, and the + # peer isn't at this time. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0338"}); $anvil->Job->update_progress({ progress => 0, @@ -282,8 +337,8 @@ sub sanity_check } else { - # We can't remove a server from DR unless both nodes are up, and the peer isn't at - # this time. + # We can't remove a server from DR unless both nodes are up, and the peer + # isn't at this time. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0339"}); $anvil->Job->update_progress({ progress => 0, @@ -342,8 +397,8 @@ sub sanity_check # Failed to find the server by name or UUID. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0341", variables => { server => $anvil->data->{switches}{server} }}); $anvil->Job->update_progress({ - progress => 100, - message => "error_0341,!!server!".$anvil->data->{switches}{server}."!!", + progress => 100, + message => "error_0341,!!server!".$anvil->data->{switches}{server}."!!", job_status => "failed", }); $anvil->nice_exit({exit_code => 1}); @@ -1956,8 +2011,12 @@ sub process_protect } # Have we already configured the DR? If so, what ports are already allocated. - my $node1_to_dr_port = ""; - my $node2_to_dr_port = ""; + my $node1_to_dr_port = ""; + my $node1_to_dr_port_inside = ""; + my $node1_to_dr_port_outside = ""; + my $node2_to_dr_port = ""; + my $node2_to_dr_port_inside = ""; + my $node2_to_dr_port_outside = ""; foreach my $host1_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{host1_to_host2}}) { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host1_name => $host1_name }}); @@ -1965,46 +2024,99 @@ sub process_protect { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host2_name => $host2_name }}); next if (($host1_name ne $dr1_short_host_name) && ($host2_name ne $dr1_short_host_name)); + if (($host1_name eq $node1_short_host_name) or ($host2_name eq $node1_short_host_name)) { $node1_to_dr_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host1_tcp_port}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port => $node1_to_dr_port }}); + + if (exists $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port}) + { + $node1_to_dr_port_inside = $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port}; + $node1_to_dr_port_outside = $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_tcp_port}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + node1_to_dr_port_inside => $node1_to_dr_port_inside, + node1_to_dr_port_outside => $node1_to_dr_port_outside, + }}); + } } else { $node2_to_dr_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host1_tcp_port}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port => $node2_to_dr_port }}); + + if (exists $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port}) + { + $node2_to_dr_port_inside = $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port}; + $node2_to_dr_port_outside = $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_tcp_port}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + node2_to_dr_port_inside => $node2_to_dr_port_inside, + node2_to_dr_port_outside => $node2_to_dr_port_outside, + }}); + } } } } # Get net next pair of TCP ports, if needed. + my $dr_tcp_ports = 1; + my $long_throw_ports = 0; + if ($anvil->data->{switches}{protocol} eq "long-throw") + { + $dr_tcp_ports = 0; + $long_throw_ports = 1; + } my (undef, $tcp_ports) = $anvil->DRBD->get_next_resource({ - debug => 2, - dr_tcp_ports => 1, + debug => 2, + dr_tcp_ports => $dr_tcp_ports, + long_throw_ports => $long_throw_ports, }); - my ($first_port, $second_port) = split/,/, $tcp_ports; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "s01:first_port" => $first_port, - "s02:second_port" => $second_port, - }}); + my @free_ports = split/,/, $tcp_ports; + my $i = 0; + if ($node1_to_dr_port eq "") { - $node1_to_dr_port = $first_port; + $node1_to_dr_port = $free_ports[$i++]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port => $node1_to_dr_port }}); } + + if (($long_throw_ports) && (not $node1_to_dr_port_inside)) + { + $node1_to_dr_port_inside = $free_ports[$i++]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port_inside => $node1_to_dr_port_inside }}); + } + if (($long_throw_ports) && (not $node1_to_dr_port_outside)) + { + $node1_to_dr_port_outside = $free_ports[$i++]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port_outside => $node1_to_dr_port_outside }}); + } + if ($node2_to_dr_port eq "") { - $node2_to_dr_port = $second_port; + $node2_to_dr_port = $free_ports[$i]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port => $node2_to_dr_port }}); } + if (($long_throw_ports) && (not $node2_to_dr_port_inside)) + { + $node2_to_dr_port_inside = $free_ports[$i++]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port_inside => $node2_to_dr_port_inside }}); + } + if (($long_throw_ports) && (not $node2_to_dr_port_outside)) + { + $node2_to_dr_port_outside = $free_ports[$i++]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port_outside => $node2_to_dr_port_outside }}); + } # Show what we're doing my $variables = { - protocol => $anvil->data->{switches}{protocol}, - node1_to_dr_port => $node1_to_dr_port, - node2_to_dr_port => $node2_to_dr_port, - config_file => $config_file, + protocol => $anvil->data->{switches}{protocol}, + node1_to_dr_port => $node1_to_dr_port, + node1_to_dr_port_inside => $node1_to_dr_port_inside, + node1_to_dr_port_outside => $node1_to_dr_port_outside, + node2_to_dr_port => $node2_to_dr_port, + node2_to_dr_port_inside => $node2_to_dr_port_inside, + node2_to_dr_port_outside => $node2_to_dr_port_outside, + config_file => $config_file, }; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0361", variables => $variables}); $anvil->Job->update_progress({ @@ -2282,33 +2394,58 @@ sub process_protect }}); # Choose the DR protocol - my $dr_protocol = "A"; + my $use_drbd_proxy = 0; + my $dr_protocol = "A"; if ($anvil->data->{switches}{protocol} eq "sync") { $dr_protocol = "C"; } - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_protocol => $dr_protocol }}); + elsif ($anvil->data->{switches}{protocol} eq "long-throw") + { + $use_drbd_proxy = 1; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + dr_protocol => $dr_protocol, + use_drbd_proxy => $use_drbd_proxy, + }}); # Node 1 to Node 2 first + my $proxy = ""; + my $max_c_rate = 500; + my $file_key = "file_0005"; + if ($use_drbd_proxy) + { + $proxy = $anvil->Words->string({key => "file_0008", variables => { memlimit => 256 }}); + $max_c_rate = 100; + $file_key = "file_0007"; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + proxy => $proxy, + max_c_rate => $max_c_rate, + file_key => $file_key, + }}); + my $connections = $anvil->Words->string({key => "file_0005", variables => { host1_short_name => $node1_short_host_name, host1_ip => $node1_sn_ip, host2_short_name => $node2_short_host_name, host2_ip => $node2_sn_ip, tcp_port => $nodes_tcp_port, - 'c-rate-maximum' => 500, + 'c-rate-maximum' => $max_c_rate, protocol => "C", fencing => "resource-and-stonith" }}); # Node 1 to DR - $connections .= $anvil->Words->string({key => "file_0005", variables => { + $connections .= $anvil->Words->string({key => "file_0005", variables => { host1_short_name => $node1_short_host_name, host1_ip => $node1_dr_ip, host2_short_name => $dr1_short_host_name, host2_ip => $dr1_ip, tcp_port => $node1_to_dr_port, - 'c-rate-maximum' => 500, + inside_tcp_port => $node1_to_dr_port_inside, + outside_tcp_port => $node1_to_dr_port_outside, + 'c-rate-maximum' => $max_c_rate, protocol => $dr_protocol, fencing => "dont-care" }}); @@ -2320,13 +2457,17 @@ sub process_protect host2_short_name => $dr1_short_host_name, host2_ip => $dr1_ip, tcp_port => $node2_to_dr_port, - 'c-rate-maximum' => 500, + inside_tcp_port => $node2_to_dr_port_inside, + outside_tcp_port => $node2_to_dr_port_outside, + 'c-rate-maximum' => $max_c_rate, protocol => $dr_protocol, fencing => "dont-care" }}); + my $new_resource_config = $anvil->Words->string({key => "file_0006", variables => { server => $server_name, + proxy => $proxy, hosts => $hosts, connections => $connections, }}); diff --git a/tools/anvil-provision-server b/tools/anvil-provision-server index 2b572de6..4a6b241c 100755 --- a/tools/anvil-provision-server +++ b/tools/anvil-provision-server @@ -759,6 +759,8 @@ sub startup_resource task => "up", }); + # If both sides are Inconsistent, for node 1 to primary + my $waiting = 1; while($waiting) { diff --git a/tools/anvil-safe-stop b/tools/anvil-safe-stop index 53c280c0..7b217eed 100755 --- a/tools/anvil-safe-stop +++ b/tools/anvil-safe-stop @@ -8,6 +8,9 @@ # # TODO: # +# BUG: +# - --poweroff when the peer is offline tries to migrate anyway. +# - use strict; use warnings; @@ -265,19 +268,6 @@ sub process_servers { my ($anvil) = @_; - if ($anvil->data->{switches}{'stop-servers'}) - { - # Tell the user we're about to shut down servers. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0320"}); - $anvil->Job->update_progress({progress => 10, message => "job_0320"}); - } - else - { - # Tell the user we're about to migrate servers. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0321"}); - $anvil->Job->update_progress({progress => 10, message => "job_0321"}); - } - # Use virsh to check for servers, in case pacemaker lies to us. $anvil->Server->find(); my $progress = 10; @@ -291,6 +281,51 @@ sub process_servers 's1:server_count' => $server_count, 's2:progress_steps' => $progress_steps, }}); + + # If we have one or more local servers, we need to know if both of us are in the cluster. If we're + # not, or the peer isn't, we can't migrate. + my $can_migrate = 0; + if ($server_count) + { + my $problem = $anvil->Cluster->parse_cib({debug => 2}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:problem' => $problem, + 's2:cib::parsed::local::ready' => $anvil->data->{cib}{parsed}{'local'}{ready}, + 's3:cib::parsed::peer::ready' => $anvil->data->{cib}{parsed}{peer}{ready}, + }}); + if ($problem) + { + $can_migrate = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { can_migrate => $can_migrate }}); + } + elsif ((not $anvil->data->{cib}{parsed}{'local'}{ready}) or (not $anvil->data->{cib}{parsed}{peer}{ready})) + { + $can_migrate = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { can_migrate => $can_migrate }}); + } + + if ((not $anvil->data->{switches}{'stop-servers'}) && (not $can_migrate)) + { + # Abort. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0372"}); + $anvil->Job->update_progress({progress => 100, message => "error_0372"}); + $anvil->nice_exit({exit_code => 1}); + } + } + + if ($anvil->data->{switches}{'stop-servers'}) + { + # Tell the user we're about to shut down servers. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0320"}); + $anvil->Job->update_progress({progress => 10, message => "job_0320"}); + } + else + { + # Tell the user we're about to migrate servers. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0321"}); + $anvil->Job->update_progress({progress => 10, message => "job_0321"}); + } + while ($waiting) { # Is the cluster up? From 599373816f9bce72ce325d563324dcc51158552c Mon Sep 17 00:00:00 2001 From: Digimer Date: Thu, 22 Sep 2022 16:40:40 -0400 Subject: [PATCH 3/3] * Fixed bugs that came up in testing. Was now able to setup long-throw DR! Signed-off-by: Digimer --- share/words.xml | 7 ++----- tools/anvil-manage-dr | 26 +++++++++++++------------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/share/words.xml b/share/words.xml index a0de2ad7..3e127296 100644 --- a/share/words.xml +++ b/share/words.xml @@ -832,12 +832,11 @@ resource #!variable!server!# { } } ]]> - = 16MB per proxy connection to DRBD proxy for it to bring up a connection - memlimit #!variable!memlimiin!#M; + memlimit #!variable!memlimit!#M; } - ]]> @@ -1320,8 +1319,6 @@ It should be provisioned in the next minute or two. Beginning to protect the server: [#!variable!server!#]! Verified that there is enough space on DR to proceed. * The connection protocol will be: ..... [#!variable!protocol!#] -* Node 1 to DR will use TCP port: ...... [#!variable!node1_to_dr_port!#] -* Node 2 to DR will use TCP port: ...... [#!variable!node2_to_dr_port!#] * We will update the DRBD resource file: [#!variable!config_file!#] The following LV(s) will be created: diff --git a/tools/anvil-manage-dr b/tools/anvil-manage-dr index bd218fb7..705a3cd8 100755 --- a/tools/anvil-manage-dr +++ b/tools/anvil-manage-dr @@ -191,7 +191,7 @@ sub sanity_check $anvil->nice_exit({exit_code => 1}); } - if ($anvil->data->{switches}{protocol} eq "long-throw")) + if ($anvil->data->{switches}{protocol} eq "long-throw") { # If there isn't a license file, make sure it's being provided. $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { @@ -2030,10 +2030,10 @@ sub process_protect $node1_to_dr_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host1_tcp_port}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port => $node1_to_dr_port }}); - if (exists $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port}) + if (exists $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port}) { - $node1_to_dr_port_inside = $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port}; - $node1_to_dr_port_outside = $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_tcp_port}; + $node1_to_dr_port_inside = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port}; + $node1_to_dr_port_outside = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_tcp_port}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port_inside => $node1_to_dr_port_inside, node1_to_dr_port_outside => $node1_to_dr_port_outside, @@ -2045,10 +2045,10 @@ sub process_protect $node2_to_dr_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host1_tcp_port}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port => $node2_to_dr_port }}); - if (exists $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port}) + if (exists $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port}) { - $node2_to_dr_port_inside = $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port}; - $node2_to_dr_port_outside = $anvil->data->{new}{resource}{$resource}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_tcp_port}; + $node2_to_dr_port_inside = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port}; + $node2_to_dr_port_outside = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_tcp_port}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port_inside => $node2_to_dr_port_inside, node2_to_dr_port_outside => $node2_to_dr_port_outside, @@ -2425,6 +2425,7 @@ sub process_protect file_key => $file_key, }}); + # Node 1 to 2 my $connections = $anvil->Words->string({key => "file_0005", variables => { host1_short_name => $node1_short_host_name, host1_ip => $node1_sn_ip, @@ -2433,11 +2434,11 @@ sub process_protect tcp_port => $nodes_tcp_port, 'c-rate-maximum' => $max_c_rate, protocol => "C", - fencing => "resource-and-stonith" + fencing => "resource-and-stonith" }}); # Node 1 to DR - $connections .= $anvil->Words->string({key => "file_0005", variables => { + $connections .= $anvil->Words->string({key => $file_key, variables => { host1_short_name => $node1_short_host_name, host1_ip => $node1_dr_ip, host2_short_name => $dr1_short_host_name, @@ -2447,11 +2448,11 @@ sub process_protect outside_tcp_port => $node1_to_dr_port_outside, 'c-rate-maximum' => $max_c_rate, protocol => $dr_protocol, - fencing => "dont-care" + fencing => "dont-care" }}); # Node 2 to DR - $connections .= $anvil->Words->string({key => "file_0005", variables => { + $connections .= $anvil->Words->string({key => $file_key, variables => { host1_short_name => $node2_short_host_name, host1_ip => $node2_dr_ip, host2_short_name => $dr1_short_host_name, @@ -2461,10 +2462,9 @@ sub process_protect outside_tcp_port => $node2_to_dr_port_outside, 'c-rate-maximum' => $max_c_rate, protocol => $dr_protocol, - fencing => "dont-care" + fencing => "dont-care" }}); - my $new_resource_config = $anvil->Words->string({key => "file_0006", variables => { server => $server_name, proxy => $proxy,