From 6664c5b77f4daa3b2f13e27bba6a93d5f0bb0444 Mon Sep 17 00:00:00 2001 From: Digimer Date: Sun, 12 Sep 2021 23:34:25 -0400 Subject: [PATCH] * Fixed a bug where scan-drbd, with DR configured, was not recording TCP ports assigned to connections properly. * More bugs fixed in anvil-manage-dr, tested repeatedly as a job and so far, so good. Other functionality still to come. Signed-off-by: Digimer --- Anvil/Tools/DRBD.pm | 181 ++++++++++++++++++++-------- scancore-agents/scan-drbd/scan-drbd | 2 +- share/words.xml | 7 +- tools/anvil-manage-dr | 91 +++++++++++--- 4 files changed, 208 insertions(+), 73 deletions(-) diff --git a/Anvil/Tools/DRBD.pm b/Anvil/Tools/DRBD.pm index 0bf5a152..c34a687d 100644 --- a/Anvil/Tools/DRBD.pm +++ b/Anvil/Tools/DRBD.pm @@ -618,6 +618,12 @@ sub gather_data $anvil->data->{new}{scan_drbd}{scan_drbd_timeout} = 6; # Default is '60', 6 seconds $anvil->data->{new}{scan_drbd}{scan_drbd_total_sync_speed} = 0; + my $local_host_name = $anvil->Get->host_name; + my $local_short_host_name = $anvil->Get->short_host_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + local_host_name => $local_host_name, + local_short_host_name => $local_short_host_name, + }}); foreach my $name ($dom->findnodes('/config/common/section')) { my $section = $name->{name}; @@ -706,7 +712,7 @@ sub gather_data }}); # Record the local data only. - if (($this_host_name eq $anvil->Get->host_name) or ($this_host_name eq $anvil->Get->short_host_name)) + if (($this_host_name eq $local_host_name) or ($this_host_name eq $local_short_host_name)) { $anvil->data->{new}{resource}{$resource}{volume}{$volume}{device_path} = $volume_vnr->findvalue('./device'); $anvil->data->{new}{resource}{$resource}{volume}{$volume}{backing_disk} = $volume_vnr->findvalue('./disk'); @@ -723,32 +729,135 @@ sub gather_data foreach my $connection ($name->findnodes('./connection')) { - my $peer = ""; + my $host1_name = ""; + my $host1_ip_address = ""; + my $host1_tcp_port = ""; + my $host2_name = ""; + my $host2_ip_address = ""; + my $host2_tcp_port = ""; + my $peer = ""; foreach my $host ($connection->findnodes('./host')) { my $this_host_name = $host->{name}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { this_host_name => $this_host_name }}); + if (not $host1_name) + { + $host1_name = $this_host_name; + $host1_ip_address = $host->findvalue('./address'); + $host1_tcp_port = $host->findvalue('./address/@port'); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + host1_name => $host1_name, + host1_ip_address => $host1_ip_address, + host1_tcp_port => $host1_tcp_port, + }}); + } + else + { + $host2_name = $this_host_name; + $host2_ip_address = $host->findvalue('./address'); + $host2_tcp_port = $host->findvalue('./address/@port'); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + host2_name => $host2_name, + host2_ip_address => $host2_ip_address, + host2_tcp_port => $host2_tcp_port, + }}); + } - next if (($this_host_name eq $anvil->Get->host_name) or ($this_host_name eq $anvil->Get->short_host_name)); - - $peer = $this_host_name; - $anvil->data->{new}{resource}{$resource}{peer}{$peer}{peer_ip_address} = $host->findvalue('./address'); - $anvil->data->{new}{resource}{$resource}{peer}{$peer}{tcp_port} = $host->findvalue('./address/@port');; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "s1:new::resource::${resource}::peer::${peer}::peer_ip_address" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{peer_ip_address}, - "s2:new::resource::${resource}::peer::${peer}::tcp_port" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{tcp_port}, - }}); +# $peer = $this_host_name; +# $anvil->data->{new}{resource}{$resource}{peer}{$peer}{peer_ip_address} = $host->findvalue('./address'); +# $anvil->data->{new}{resource}{$resource}{peer}{$peer}{tcp_port} = $host->findvalue('./address/@port'); +# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { +# "s1:new::resource::${resource}::peer::${peer}::peer_ip_address" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{peer_ip_address}, +# "s2:new::resource::${resource}::peer::${peer}::tcp_port" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{tcp_port}." (".$host->findvalue('./address/@port').")", +# }}); - if (not exists $anvil->data->{new}{resource}{$resource}{peer}{$peer}{protocol}) +# if (not exists $anvil->data->{new}{resource}{$resource}{peer}{$peer}{protocol}) +# { +# $anvil->data->{new}{resource}{$resource}{peer}{$peer}{protocol} = "unknown"; +# $anvil->data->{new}{resource}{$resource}{peer}{$peer}{fencing} = "unknown"; +# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { +# "s1:new::resource::${resource}::peer::${peer}::protocol" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{protocol}, +# "s2:new::resource::${resource}::peer::${peer}::fencing" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{fencing}, +# }}); +# } +# +# foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{volume}}) +# { +# $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{connection_state} = "disconnected"; +# $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{local_disk_state} = "down"; +# $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{peer_disk_state} = "unknown"; +# $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{local_role} = "down"; +# $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{peer_role} = "unknown"; +# $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{out_of_sync_size} = -1; +# $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{replication_speed} = 0; +# $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{estimated_time_to_sync} = 0; +# } + } + + if (($host1_name eq $local_short_host_name) or + ($host1_name eq $local_host_name) or + ($host2_name eq $local_short_host_name) or + ($host2_name eq $local_host_name)) + { + # This is one of our connections. + my $peer = ""; + if (($host1_name eq $local_short_host_name) or ($host1_name eq $local_host_name)) + { + # Our peer is host2 + $peer = $host2_name; + $anvil->data->{new}{resource}{$resource}{peer}{$peer}{peer_ip_address} = $host2_ip_address; + $anvil->data->{new}{resource}{$resource}{peer}{$peer}{tcp_port} = $host2_tcp_port; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:new::resource::${resource}::peer::${peer}::peer_ip_address" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{peer_ip_address}, + "s2:new::resource::${resource}::peer::${peer}::tcp_port" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{tcp_port}, + }}); + } + else { - $anvil->data->{new}{resource}{$resource}{peer}{$peer}{protocol} = "unknown"; - $anvil->data->{new}{resource}{$resource}{peer}{$peer}{fencing} = "unknown"; + # Our peer is host1 + $peer = $host1_name; + $anvil->data->{new}{resource}{$resource}{peer}{$peer}{peer_ip_address} = $host1_ip_address; + $anvil->data->{new}{resource}{$resource}{peer}{$peer}{tcp_port} = $host1_tcp_port; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "s1:new::resource::${resource}::peer::${peer}::protocol" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{protocol}, - "s2:new::resource::${resource}::peer::${peer}::fencing" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{fencing}, + "s1:new::resource::${resource}::peer::${peer}::peer_ip_address" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{peer_ip_address}, + "s2:new::resource::${resource}::peer::${peer}::tcp_port" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{tcp_port}, }}); } + foreach my $name ($connection->findnodes('./section')) + { + my $section = $name->{name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { section => $section }}); + + foreach my $option_name ($name->findnodes('./option')) + { + my $variable = $option_name->{name}; + my $value = $option_name->{value}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:variable' => $variable, + 's2:value' => $value, + }}); + + if ($section eq "net") + { + if ($variable eq "protocol") + { + $anvil->data->{new}{resource}{$resource}{peer}{$peer}{protocol} = $value; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "new::resource::${resource}::peer::${peer}::protocol" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{protocol}, + }}); + } + if ($variable eq "fencing") + { + $anvil->data->{new}{resource}{$resource}{peer}{$peer}{fencing} = $value; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "new::resource::${resource}::peer::${peer}::fencing" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{fencing}, + }}); + } + } + } + } + foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{volume}}) { $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{connection_state} = "disconnected"; @@ -761,40 +870,6 @@ sub gather_data $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{estimated_time_to_sync} = 0; } } - - foreach my $name ($connection->findnodes('./section')) - { - my $section = $name->{name}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { section => $section }}); - - foreach my $option_name ($name->findnodes('./option')) - { - my $variable = $option_name->{name}; - my $value = $option_name->{value}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - 's1:variable' => $variable, - 's2:value' => $value, - }}); - - if ($section eq "net") - { - if ($variable eq "protocol") - { - $anvil->data->{new}{resource}{$resource}{peer}{$peer}{protocol} = $value; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "new::resource::${resource}::peer::${peer}::protocol" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{protocol}, - }}); - } - if ($variable eq "fencing") - { - $anvil->data->{new}{resource}{$resource}{peer}{$peer}{fencing} = $value; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "new::resource::${resource}::peer::${peer}::fencing" => $anvil->data->{new}{resource}{$resource}{peer}{$peer}{fencing}, - }}); - } - } - } - } } } } @@ -1394,7 +1469,7 @@ ORDER BY 's6:scan_drbd_volume_device_minor' => $scan_drbd_volume_device_minor, 's7:scan_drbd_peer_host_name' => $scan_drbd_peer_host_name, 's8:scan_drbd_peer_ip_address' => $scan_drbd_peer_ip_address, - 's9:scan_drbd_peer_protocol' => $scan_drbd_peer_protocol, + 's9:scan_drbd_peer_protocol' => $scan_drbd_peer_protocol, 's10:scan_drbd_peer_fencing' => $scan_drbd_peer_fencing, 's11:scan_drbd_peer_tcp_port' => $scan_drbd_peer_tcp_port, }}); @@ -1446,13 +1521,15 @@ ORDER BY my $tcp_pair = ""; while($looking) { - if (exists $anvil->data->{drbd}{used_resources}{tcp_port}{$free_port}) + if ((exists $anvil->data->{drbd}{used_resources}{tcp_port}{$free_port}) && + ($anvil->data->{drbd}{used_resources}{tcp_port}{$free_port}{used})) { $free_port++; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { free_port => $free_port }}); } else { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { free_port => $free_port }}); if ($dr_tcp_ports) { if (not $tcp_pair) diff --git a/scancore-agents/scan-drbd/scan-drbd b/scancore-agents/scan-drbd/scan-drbd index 09b27b99..5e8b9e5a 100755 --- a/scancore-agents/scan-drbd/scan-drbd +++ b/scancore-agents/scan-drbd/scan-drbd @@ -79,7 +79,7 @@ if ($anvil->data->{switches}{purge}) $anvil->nice_exit({exit_code => 0}); } -if ($anvil->DRBD->gather_data()) +if ($anvil->DRBD->gather_data({debug => 2})) { # DRBD not found or configured. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "scan_drbd_error_0001"}); diff --git a/share/words.xml b/share/words.xml index 7e9e7d25..94948381 100644 --- a/share/words.xml +++ b/share/words.xml @@ -1225,8 +1225,8 @@ The following LV(s) will be created: - The new config looks good! - Updating the peers now... - Updating the resource file: [#!variable!file!#] on the host: [#!variable!host_name!#] via IP: [#!variable!ip_address!#]. - Creating logical volumes on DR, if needed. New LVs will have metadata created. - - Volume: [#!variable!volume!#]. + - Creating logical volumes on DR, if needed. New LVs will have metadata created. + - Volume: [#!variable!volume!#], logical volume: [#!variable!lv_path!#]. - The logical volume: [#!variable!lv_path!#] already exists, skipping it, and NOT create DRBD meta data. - Reloading the local DRBD resource config. - Reloading the resource: [#!variable!server!#] on the host: [#!variable!host_name!#]. @@ -1238,6 +1238,9 @@ The following LV(s) will be created: - Up! Done! The server: [#!variable!server!#] is now being protected on DR! It will take time for it to initialize, please be patient. + - Running the scan agent 'scan-drbd' locally to record the newly used TCP ports. + - Running the scan agent 'scan-drbd' on: [#!variable!host_name!#] to record the newly used TCP ports. + The job has been recorded with the UUID: [#!variable!job_uuid!#], it will start in just a moment if anvil-daemon is running. Starting: [#!variable!program!#]. diff --git a/tools/anvil-manage-dr b/tools/anvil-manage-dr index ab23e8ef..b9c999a6 100755 --- a/tools/anvil-manage-dr +++ b/tools/anvil-manage-dr @@ -54,7 +54,7 @@ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list }}); $anvil->Database->connect(); -$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0132"}); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"}); if (not $anvil->data->{sys}{database}{connections}) { # No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try @@ -64,17 +64,23 @@ if (not $anvil->data->{sys}{database}{connections}) $anvil->nice_exit({exit_code => 1}); } -my $termios = new POSIX::Termios; -$termios->getattr; -my $ospeed = $termios->getospeed; -my $terminal = Tgetent Term::Cap { TERM => undef, OSPEED => $ospeed }; -$terminal->Trequire(qw/ce ku kd/); -print $terminal->Tputs('cl'); - # If we've got a job UUID, load the job details. if ($anvil->data->{switches}{'job-uuid'}) { - load_job($anvil, $terminal); + load_job($anvil); +} + +my $terminal = ""; +if (not $anvil->data->{switches}{'job-uuid'}) +{ + my $termios = new POSIX::Termios; + $termios->getattr; + my $ospeed = $termios->getospeed; + $terminal = Tgetent Term::Cap { TERM => undef, OSPEED => $ospeed }; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { terminal => $terminal }}); + + $terminal->Trequire(qw/ce ku kd/); + print $terminal->Tputs('cl'); } sanity_check($anvil, $terminal); @@ -850,6 +856,11 @@ sub process_protect job_host_uuid => $anvil->Get->host_uuid, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); + + # Report the job UUID. + print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n"; + + $anvil->nice_exit({exit_code => 0}); } @@ -1222,6 +1233,46 @@ sub process_protect $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); } + # Immediately call scan-drbd on all machines to ensure that if another run is about to happen for a + # different server, it knows the used ports list is updated. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0381"}); + $anvil->Job->update_progress({ + progress => 76, + message => "job_0381", + }); + + my $scan_drbd_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd --force ".$anvil->Log->switches(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { scan_drbd_call => $scan_drbd_call }}); + my ($output, $return_code) = $anvil->System->call({shell_call => $scan_drbd_call}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); + foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid) + { + # "Peer" in this context is either a node or a DR host + next if $this_host_uuid eq $anvil->Get->host_uuid(); + my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name}; + my $peer_sn_ip = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{network}{sn1}{ip_address}; + my $variables = { host_name => $peer_host_name }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0382", variables => $variables}); + $anvil->Job->update_progress({ + progress => 77, + message => "job_0382", + variables => $variables, + }); + my ($output, $error, $return_code) = $anvil->Remote->call({ + target => $peer_sn_ip, + password => $anvil_password, + shell_call => $scan_drbd_call, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + error => $error, + output => $output, + return_code => $return_code, + }}); + } + # Create the LV and MD on DR. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0370"}); $anvil->Job->update_progress({ @@ -1231,13 +1282,6 @@ sub process_protect my $create_md = 0; foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}{$server_name}}) { - my $variables = { volume => $volume }; - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0371", variables => $variables}); - $anvil->Job->update_progress({ - progress => 80, - message => "job_0371", - variables => $variables, - }); my $dr1_sn1_ip = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{network}{sn1}{ip_address}; my $lv_path = $anvil->data->{server}{dr}{volumes}{$server_name}{$volume}{lv_path}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { @@ -1246,6 +1290,17 @@ sub process_protect lv_path => $lv_path, }}); + my $variables = { + volume => $volume, + lv_path => $lv_path, + }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0371", variables => $variables}); + $anvil->Job->update_progress({ + progress => 80, + message => "job_0371", + variables => $variables, + }); + my $lv_check_call = " if [ -e '".$lv_path."' ]; then @@ -1347,7 +1402,7 @@ fi"; }); my $shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$server_name; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); - my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); + ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code, @@ -1493,7 +1548,7 @@ fi"; sub load_job { - my ($anvil, $terminal) = @_; + my ($anvil) = @_; $anvil->Job->clear(); $anvil->Job->get_job_details();