From 218934bec82e7e6dfabc08c4e4167d2c1ea7e212 Mon Sep 17 00:00:00 2001 From: Digimer Date: Sat, 30 Jan 2021 19:06:22 -0500 Subject: [PATCH 1/4] * Fixed a bug with the path to anvil-provision-server. Signed-off-by: Digimer --- Anvil/Tools.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index 2d9d2404..e64783f9 100644 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -1096,7 +1096,7 @@ sub _set_paths 'anvil-manage-keys' => "/usr/sbin/anvil-manage-keys", 'anvil-manage-power' => "/usr/sbin/anvil-manage-power", 'anvil-parse-fence-agents' => "/usr/sbin/anvil-parse-fence-agents", - 'anvil-provision-server' => "/usr/sbib/anvil-provision-server", + 'anvil-provision-server' => "/usr/sbin/anvil-provision-server", 'anvil-report-memory' => "/usr/sbin/anvil-report-memory", 'anvil-sync-shared' => "/usr/sbin/anvil-sync-shared", 'anvil-update-files' => "/usr/sbin/anvil-update-files", From 3d7ce84c388a0eba069b7ad646dc9fd552881f5d Mon Sep 17 00:00:00 2001 From: Digimer Date: Sun, 31 Jan 2021 00:00:23 -0500 Subject: [PATCH 2/4] * Fixed a bug in Get->host_from_ip_address() where hosts that are no longer used are returned, meaning 2+ results could be returned after a node was replaced, meaning no host name was returned. * Fixed a bug in anvil-provision-server where forcing initialization of a new DRBD resource when running on node 2 would fail because the node ID in the drbdsetup command was hard-coded to be run from node 1. Signed-off-by: Digimer --- Anvil/Tools/Get.pm | 6 ++++-- tools/anvil-provision-server | 6 +++++- tools/test.pl | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Anvil/Tools/Get.pm b/Anvil/Tools/Get.pm index 340187ba..f29f2819 100644 --- a/Anvil/Tools/Get.pm +++ b/Anvil/Tools/Get.pm @@ -1255,11 +1255,13 @@ FROM ip_addresses b WHERE a.host_uuid = b.ip_address_host_uuid +AND + a.host_key != 'DELETED' AND b.ip_address_note != 'DELETED' AND - b.ip_address_address = ".$anvil->Database->quote($ip_address)."; -"; + b.ip_address_address = ".$anvil->Database->quote($ip_address)." +;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; diff --git a/tools/anvil-provision-server b/tools/anvil-provision-server index 0b034790..1da6ddd7 100755 --- a/tools/anvil-provision-server +++ b/tools/anvil-provision-server @@ -162,12 +162,16 @@ sub run_jobs $anvil->data->{job}{short_host_name} = $anvil->Get->short_host_name; $anvil->data->{job}{node1_short_host_name} = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name}; $anvil->data->{job}{node2_short_host_name} = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name}; + $anvil->data->{job}{drbd_local_node_id} = $host_uuid eq $node1_host_uuid ? 0 : 1; + $anvil->data->{job}{drbd_peer_node_id} = $host_uuid eq $node1_host_uuid ? 1 : 0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::node1_host_uuid' => $anvil->data->{job}{node1_host_uuid}, 'job::node2_host_uuid' => $anvil->data->{job}{node2_host_uuid}, 'job::node1_short_host_name' => $anvil->data->{job}{node1_short_host_name}, 'job::node2_short_host_name' => $anvil->data->{job}{node2_short_host_name}, 'job::short_host_name' => $anvil->data->{job}{short_host_name}, + 'job::drbd_local_node_id' => $anvil->data->{job}{drbd_local_node_id}, + 'job::drbd_peer_node_id' => $anvil->data->{job}{drbd_peer_node_id}, }}); # We convert to extents as it ensure clean boundaries and, being based on bytes in both cases, gets @@ -676,7 +680,7 @@ sub startup_resource }); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0195"}); - my $shell_call = $anvil->data->{path}{exe}{drbdsetup}." net-options ".$anvil->data->{job}{server_name}." 1 --set-defaults --_name=".$anvil->data->{job}{peer_short_name}." --protocol=C --fencing=dont-care"; + my $shell_call = $anvil->data->{path}{exe}{drbdsetup}." net-options ".$anvil->data->{job}{server_name}." ".$anvil->data->{job}{drbd_peer_node_id}." --set-defaults --_name=".$anvil->data->{job}{peer_short_name}." --protocol=C --fencing=dont-care"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); diff --git a/tools/test.pl b/tools/test.pl index 77f8141b..37a210f4 100755 --- a/tools/test.pl +++ b/tools/test.pl @@ -26,6 +26,6 @@ $anvil->Get->switches; $anvil->Database->connect({debug => 3}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"}); -$anvil->Cluster->assemble_storage_groups({debug => 2, anvil_uuid => '1aded871-fcb1-4473-9b97-6e9c246fc568'}); +$anvil->DRBD->gather_data({debug => 2}); $anvil->nice_exit({exit_code => 0}); From f4bf1fd54a8a6d471669694857a1c092053b2fd6 Mon Sep 17 00:00:00 2001 From: Digimer Date: Sun, 31 Jan 2021 01:39:52 -0500 Subject: [PATCH 3/4] * Removed some XML insertions into strings as the break inserting into strings. Note: These changes below shouldn't have been in this branch... *sigh* * Fixed an issue with tools/anvil-provision-server where a VM would be created but didn't boot. When this happens, an explicit boot is sent via virsh. Also bumped up the time it waits for a new server to start up. * Added an explicit call to scan-drbd after a new resource is created to ensure that if any calls come after looking for the next free DRBD minor or port, they don't use the ones just used. Signed-off-by: Digimer --- scancore-agents/scan-drbd/scan-drbd | 1 - scancore-agents/scan-drbd/scan-drbd.xml | 15 ----------- scancore-agents/scan-server/scan-server | 5 ++-- scancore-agents/scan-server/scan-server.xml | 5 +--- share/words.xml | 1 + tools/anvil-provision-server | 29 +++++++++++++++++++-- tools/test.pl | 6 ++++- 7 files changed, 36 insertions(+), 26 deletions(-) diff --git a/scancore-agents/scan-drbd/scan-drbd b/scancore-agents/scan-drbd/scan-drbd index 4a95e41c..cc4cca92 100755 --- a/scancore-agents/scan-drbd/scan-drbd +++ b/scancore-agents/scan-drbd/scan-drbd @@ -403,7 +403,6 @@ WHERE my $variables = { resource_name => $scan_drbd_resource_name, resource_state => $new_scan_drbd_resource_up ? "#!string!scan_drbd_unit_0004!#" : "#!string!scan_drbd_unit_0005!#", - resource_xml => $new_scan_drbd_resource_xml, }; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_drbd_message_0007", variables => $variables}); $anvil->Alert->register({alert_level => "warning", message => "scan_drbd_message_0007", variables => $variables, set_by => $THIS_FILE, sort_position => $anvil->data->{'scan-drbd'}{alert_sort}++}); diff --git a/scancore-agents/scan-drbd/scan-drbd.xml b/scancore-agents/scan-drbd/scan-drbd.xml index 722bd49d..4fc8c057 100644 --- a/scancore-agents/scan-drbd/scan-drbd.xml +++ b/scancore-agents/scan-drbd/scan-drbd.xml @@ -69,21 +69,11 @@ The full new config is: A new DRBD resource has been found on this host. - Resource Name: ...... [#!variable!resource_name!#] - Resource State: ..... [#!variable!resource_state!#] - -Raw resource XML: -======== -#!variable!resource_xml!# -======== A resource was found with a resource UUID that isn't valid on this host. An attempt to find a valid database entry was made but no candidate was found. Adding the resource to the database as if it were new, and generating a new resource UUID for the resource configuration file. - Resource Name: ...... [#!variable!resource_name!#] - Resource State: ..... [#!variable!resource_state!#] - -Raw resource XML: -======== -#!variable!resource_xml!# -======== The resource config: [#!variable!resource_name!#] has been deleted. The backing storage may or may not have been removed. The resource: [#!variable!old_value!#] has been renamed to: [#!variable!new_value!#]. @@ -201,11 +191,6 @@ Note: The fencing policy 'resource-and-stonith' must always be used between node The DRBD resource was not found in the database, but appears to have been in the past. Re-adding it. - Resource Name: ...... [#!variable!resource_name!#] - Resource State: ..... [#!variable!resource_state!#] - -Raw resource XML: -======== -#!variable!resource_xml!# -======== diff --git a/scancore-agents/scan-server/scan-server b/scancore-agents/scan-server/scan-server index 50b48d80..656e2ea6 100755 --- a/scancore-agents/scan-server/scan-server +++ b/scancore-agents/scan-server/scan-server @@ -146,9 +146,8 @@ sub collect_data if ($server_definition) { # Register an alert. - my $variables = { - server => $server_name, - definition => $server_definition, + my $variables = { + server => $server_name, }; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0016", variables => $variables}); $anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0016", variables => $variables, set_by => $THIS_FILE}); diff --git a/scancore-agents/scan-server/scan-server.xml b/scancore-agents/scan-server/scan-server.xml index 61b91313..e3fec658 100644 --- a/scancore-agents/scan-server/scan-server.xml +++ b/scancore-agents/scan-server/scan-server.xml @@ -84,10 +84,7 @@ The amount of RAM configured for the server: [#!variable!server!#] has changed f - NOTE: This RAM will not be used until the server is power cycled! -A server named: [#!variable!server!#] was found in the database, but there was no definition file on disk. Saving the definition file now. The definition XML is: -==== -#!variable!definition!# -==== +A server named: [#!variable!server!#] was found in the database, but there was no definition file on disk. Saving the definition file now. diff --git a/share/words.xml b/share/words.xml index fb3a2a4d..9c909832 100644 --- a/share/words.xml +++ b/share/words.xml @@ -654,6 +654,7 @@ It should be provisioned in the next minute or two. The server has been flagged as deleted now. The server delete is complete on this host! It looks like ScanCore has not yet run on one or both nodes in this Anvil! system. Missing resource data, so unable to proceed. + Manually calling 'scan-drbd' to ensure that the new agent is recorded. Starting: [#!variable!program!#]. diff --git a/tools/anvil-provision-server b/tools/anvil-provision-server index 1da6ddd7..85b41c9f 100755 --- a/tools/anvil-provision-server +++ b/tools/anvil-provision-server @@ -439,7 +439,7 @@ sub provision_server $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0200"}); # Loop for up to 10 seconds waiting to see the server start running. - my $wait_until = time + 10; + my $wait_until = time + 60; my $waiting = 1; my $status = ""; while($waiting) @@ -462,6 +462,15 @@ sub provision_server $waiting = 0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); } + elsif ($status eq "shut off") + { + # Try to boot it. + my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." start ".$server}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); + } } if (($waiting) && (time > $wait_until)) { @@ -923,6 +932,22 @@ sub create_resource_file $anvil->nice_exit({exit_code => 1}); } + # Manually call 'scan-drbd' to make sure that the new data is in scan_drbd_resources so that any + my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd"; + if (-f $shell_call) + { + $anvil->Job->update_progress({ + progress => 35, + message => "job_0218", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0218"}); + my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); + } + $anvil->Job->update_progress({ progress => 40, message => "job_0190,!!resource!".$anvil->data->{job}{server_name}."!!", @@ -1044,7 +1069,7 @@ sub check_drbd_minor_and_port if ((($anvil->data->{job}{drbd_minor} eq "") or ($anvil->data->{job}{drbd_tcp_port} eq "")) && (not $anvil->data->{job}{peer_mode})) { - my ($free_minor, $free_port) = $anvil->DRBD->get_next_resource({anvil_uuid => $anvil->data->{job}{anvil_uuid}}); + my ($free_minor, $free_port) = $anvil->DRBD->get_next_resource({debug => 2, anvil_uuid => $anvil->data->{job}{anvil_uuid}}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { free_minor => $free_minor, free_port => $free_port, diff --git a/tools/test.pl b/tools/test.pl index 37a210f4..0b314e82 100755 --- a/tools/test.pl +++ b/tools/test.pl @@ -26,6 +26,10 @@ $anvil->Get->switches; $anvil->Database->connect({debug => 3}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"}); -$anvil->DRBD->gather_data({debug => 2}); +my ($free_minor, $free_port) = $anvil->DRBD->get_next_resource({debug => 2, anvil_uuid => "1aded871-fcb1-4473-9b97-6e9c246fc568"}); +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + free_minor => $free_minor, + free_port => $free_port, +}}); $anvil->nice_exit({exit_code => 0}); From 108164589351bce69d668033f43121826cc49e06 Mon Sep 17 00:00:00 2001 From: Digimer Date: Sun, 31 Jan 2021 02:32:12 -0500 Subject: [PATCH 4/4] * Added parameters to DRBD->get_next_resource to allow for a resource to be searched and either error out if a resource is found, or return the first DRBD minor and tcp port if found. Signed-off-by: Digimer --- Anvil/Tools/DRBD.pm | 55 +++++++++++++++++++++++++++++++++--- tools/anvil-provision-server | 6 +++- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/Anvil/Tools/DRBD.pm b/Anvil/Tools/DRBD.pm index 2637b179..097631b4 100644 --- a/Anvil/Tools/DRBD.pm +++ b/Anvil/Tools/DRBD.pm @@ -1114,6 +1114,14 @@ Parameters; This is the Anvil! in which we're looking for the next free resources. +=head3 resource_name (optional) + +If this is set, and the resource is found to already exist, the first DRBD minor number and first used TCP port are returned. Alternatively, if C<< force_unique >> is set to C<< 1 >>, and the resource is found to exist, C<< !!error!! >> is returned. + +=head3 force_unique (optional, default '0') + +This can be used to cause this method to return an error if C<< resource_name >> is also set and the resource is found to already exist. + =cut sub get_next_resource { @@ -1123,11 +1131,15 @@ sub get_next_resource my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "DRBD->get_next_resource()" }}); - my $free_minor = ""; - my $free_port = ""; - my $anvil_uuid = defined $parameter->{anvil_uuid} ? $parameter->{anvil_uuid} : ""; + my $free_minor = ""; + my $free_port = ""; + my $anvil_uuid = defined $parameter->{anvil_uuid} ? $parameter->{anvil_uuid} : ""; + my $resource_name = defined $parameter->{resource_name} ? $parameter->{resource_name} : ""; + my $force_unique = defined $parameter->{force_unique} ? $parameter->{force_unique} : 0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - anvil_uuid => $anvil_uuid, + anvil_uuid => $anvil_uuid, + resource_name => $resource_name, + force_unique => $force_unique, }}); if (not $anvil_uuid) @@ -1208,6 +1220,8 @@ ORDER BY } # Successful parse! + my $local_minor = ""; + my $local_port = ""; foreach my $name ($dom->findnodes('/resource')) { my $resource = $name->{name}; @@ -1231,6 +1245,12 @@ ORDER BY $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "drbd::used_resources::minor::${minor}::used" => $anvil->data->{drbd}{used_resources}{minor}{$minor}{used}, }}); + + if (not $local_minor) + { + $local_minor = $minor; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { local_minor => $local_minor }}); + } } } @@ -1249,9 +1269,36 @@ ORDER BY $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "drbd::used_resources::tcp_port::${tcp_port}::used" => $anvil->data->{drbd}{used_resources}{tcp_port}{$tcp_port}{used}, }}); + + if (not $local_port) + { + $local_port = $tcp_port; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { local_port => $local_port }}); + } } } } + + # Is the user looking for this resource? + if (($resource_name) && ($resource_name eq $scan_drbd_resource_name)) + { + # If we're force_unique, error. + if ($force_unique) + { + # Error out. + return('!!error!!'); + } + else + { + $free_minor = $local_minor; + $free_port = $local_port; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + free_minor => $free_minor, + free_port => $free_port, + }}); + return($free_minor, $free_port); + } + } } my $looking = 1; diff --git a/tools/anvil-provision-server b/tools/anvil-provision-server index 85b41c9f..813852d2 100755 --- a/tools/anvil-provision-server +++ b/tools/anvil-provision-server @@ -1069,7 +1069,11 @@ sub check_drbd_minor_and_port if ((($anvil->data->{job}{drbd_minor} eq "") or ($anvil->data->{job}{drbd_tcp_port} eq "")) && (not $anvil->data->{job}{peer_mode})) { - my ($free_minor, $free_port) = $anvil->DRBD->get_next_resource({debug => 2, anvil_uuid => $anvil->data->{job}{anvil_uuid}}); + my ($free_minor, $free_port) = $anvil->DRBD->get_next_resource({ + debug => 2, + anvil_uuid => $anvil->data->{job}{anvil_uuid}, + resource_name => $anvil->data->{job}{server_name}, + }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { free_minor => $free_minor, free_port => $free_port,