From b36093671b48c790be55815596817f6923011fcf Mon Sep 17 00:00:00 2001 From: Digimer Date: Thu, 15 Apr 2021 02:35:58 -0400 Subject: [PATCH] * Updated Database queries that were passing 'debug => $debug' to not do that, as it was causing far too much (useless) noise in the logs. * Turned on print to console for logging in anvil-provision-server. Also updated it to check if the cluster is running and hold until it is. * Cleaned up some code in Get->available_resources() that proved hard to debug. Signed-off-by: Digimer --- Anvil/Tools/Cluster.pm | 14 ++- Anvil/Tools/Database.pm | 36 +++--- Anvil/Tools/Get.pm | 59 +++++++--- Anvil/Tools/Validate.pm | 2 +- share/words.xml | 4 +- tools/anvil-provision-server | 207 +++++++++++++++++++++++------------ 6 files changed, 213 insertions(+), 109 deletions(-) diff --git a/Anvil/Tools/Cluster.pm b/Anvil/Tools/Cluster.pm index dc0cd45e..67bdae14 100644 --- a/Anvil/Tools/Cluster.pm +++ b/Anvil/Tools/Cluster.pm @@ -329,10 +329,18 @@ WHERE $anvil->Database->get_storage_group_data({debug => $debug}); # Look for ungrouped VGs and see if we can group them by matching identical sizes together. - foreach my $host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid) + my $hosts = [$node1_host_uuid, $node2_host_uuid]; + if ($dr1_host_uuid) + { + push @{$hosts}, $dr1_host_uuid; + } + else + { + # No DR. + $anvil->data->{ungrouped_vg_count}{dr1} = 0; + } + foreach my $host_uuid (@{$hosts}) { - # If DR isn't defined, it'll be blank. - next if not $host_uuid; my $this_is = "node1"; if ($host_uuid eq $node2_host_uuid) { $this_is = "node2"; } elsif ($host_uuid eq $dr1_host_uuid) { $this_is = "dr1"; } diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 1ba7446f..b1c25b66 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -652,7 +652,7 @@ sub check_for_schema foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}}) { my $host_name = $anvil->Database->get_host_from_uuid({debug => $debug, short => 1, host_uuid => $uuid}); - my $count = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 's1:count' => $count, 's2:host_name' => $host_name, @@ -1485,7 +1485,7 @@ sub connect # Read the DB identifier and then check that we've not already connected to this DB. my $query = "SELECT system_identifier FROM pg_control_system();"; - my $identifier = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $identifier = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query, identifier => $identifier, @@ -1513,7 +1513,7 @@ sub connect my $query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename=".$anvil->Database->quote($anvil->data->{defaults}{sql}{test_table})." AND schemaname='public';"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - my $count = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); if ($count < 1) @@ -1529,7 +1529,7 @@ sub connect $query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename=".$anvil->Database->quote($test_table)." AND schemaname='public';"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - my $count = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); if ($count < 1) @@ -1673,7 +1673,7 @@ sub connect # my $query = "SELECT COUNT(*) FROM hosts WHERE host_name = ".$anvil->Database->quote($anvil->data->{database}{$uuid}{host}).";"; # $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); # -# my $count = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; +# my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; # $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); # # if ($count > 0) @@ -4263,7 +4263,7 @@ sub get_storage_group_data my $query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename='scan_lvm_vgs' AND schemaname='public';"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - my $count = $anvil->Database->query({debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); if ($count) @@ -13546,7 +13546,7 @@ sub log_connections }}); my $query = "SELECT system_identifier FROM pg_control_system();"; - my $identifier = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $identifier = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0540", variables => { uuid => $uuid, identifier => $identifier, @@ -14518,7 +14518,7 @@ sub refresh_timestamp my $query = "SELECT cast(now() AS timestamp with time zone);"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - $anvil->data->{sys}{database}{timestamp} = $anvil->Database->query({debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $anvil->data->{sys}{database}{timestamp} = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::timestamp" => $anvil->data->{sys}{database}{timestamp} }}); return($anvil->data->{sys}{database}{timestamp}); @@ -14877,7 +14877,7 @@ sub resync_databases ### INSERT) my $count_query = "SELECT COUNT(*) FROM public.".$table." WHERE ".$uuid_column." = ".$anvil->Database->quote($row_uuid).";"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count_query => $count_query }}); - my $count = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $count_query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $count = $anvil->Database->query({uuid => $uuid, query => $count_query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); if ($count) { @@ -15379,7 +15379,7 @@ sub _archive_table my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'history' AND table_name = ".$anvil->Database->quote($table).";"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - my $count = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); if (not $count) { @@ -15391,7 +15391,7 @@ sub _archive_table $query = "SELECT COUNT(*) FROM history.".$table.";"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - $count = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "s1:uuid" => $uuid, "s2:count" => $count, @@ -15423,7 +15423,7 @@ sub _archive_table $query = "SELECT column_name FROM information_schema.columns WHERE table_schema = 'history' AND table_name = ".$anvil->Database->quote($table)." AND column_name != 'history_id' AND column_name != 'modified_date';"; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0124", variables => { query => $query }}); - my $columns = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); + my $columns = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); my $column_count = @{$columns}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { columns => $columns, @@ -15456,7 +15456,7 @@ COPY history.".$table." ("; "s3:sql_file" => $sql_file, }}); - my $modified_date = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $modified_date = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { modified_date => $modified_date }}); # Build the query. @@ -15472,7 +15472,7 @@ COPY history.".$table." ("; sql_file => $sql_file, query => $query, }}); - my $results = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); + my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { results => $results, @@ -15672,7 +15672,7 @@ sub _find_behind_databases my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'public' AND table_name = ".$anvil->Database->quote($table).";"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - my $count = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); if ($count == 1) @@ -15703,7 +15703,7 @@ sub _find_behind_databases # See if there is a column that ends in '_host_uuid'. If there is, we'll use # it later to restrict resync activity to these columns with the local # 'sys::host_uuid'. - my $count = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); if ($count) @@ -15719,7 +15719,7 @@ sub _find_behind_databases $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'history' AND table_name = ".$anvil->Database->quote($table).";"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - my $count = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); my $schema = $count ? "history" : "public"; @@ -15744,7 +15744,7 @@ ORDER BY }}); # Get the count of columns as well as the most recent one. - my $results = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); + my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); my $row_count = @{$results}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { results => $results, diff --git a/Anvil/Tools/Get.pm b/Anvil/Tools/Get.pm index ba5c488b..34c144db 100644 --- a/Anvil/Tools/Get.pm +++ b/Anvil/Tools/Get.pm @@ -683,29 +683,56 @@ ORDER BY foreach my $storage_group_uuid (keys %{$anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}}) { - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{group_name} = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name}; - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size} = 0; - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size} = 0; - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr} = 0; - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr} = 0; + $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{group_name} = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "anvil_resources::${anvil_uuid}::storage_group::${storage_group_uuid}::group_name" => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{group_name}, + }}); - if ((exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node1_host_uuid}) && - (exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node2_host_uuid})) + my $node1_vg_size = 0; + my $node1_vg_free = 0; + my $node2_vg_size = 0; + my $node2_vg_free = 0; + my $dr1_vg_size = 0; + my $dr1_vg_free = 0; + if (exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node1_host_uuid}) { - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size} = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node1_host_uuid}{vg_size}; - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size} = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node1_host_uuid}{vg_free}; - if ($anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node2_host_uuid}{vg_free} < $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node1_host_uuid}{vg_free}) - { - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size} = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node2_host_uuid}{vg_size}; - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size} = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node2_host_uuid}{vg_free}; - } + $node1_vg_size = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node1_host_uuid}{vg_size}; + $node1_vg_free = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node1_host_uuid}{vg_free}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + node1_vg_size => $node1_vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $node1_vg_size}).")", + node1_vg_free => $node1_vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $node1_vg_free}).")", + }}); + } + if (exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node2_host_uuid}) + { + $node2_vg_size = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node2_host_uuid}{vg_size}; + $node2_vg_free = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node2_host_uuid}{vg_free}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + node2_vg_size => $node2_vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $node2_vg_size}).")", + node2_vg_free => $node2_vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $node2_vg_free}).")", + }}); } if (exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr1_host_uuid}) { - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr} = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr1_host_uuid}{vg_size}; - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr} = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr1_host_uuid}{vg_free}; + $dr1_vg_size = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr1_host_uuid}{vg_size}; + $dr1_vg_free = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr1_host_uuid}{vg_free}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + dr1_vg_size => $dr1_vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr1_vg_size}).")", + dr1_vg_free => $dr1_vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr1_vg_free}).")", + }}); } + $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size} = $node2_vg_size < $node1_vg_size ? $node2_vg_size : $node1_vg_size; + $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size} = $node2_vg_free < $node1_vg_free ? $node2_vg_free : $node1_vg_free; + $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr} = $dr1_vg_size; + $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr} = $dr1_vg_free; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "anvil_resources::${anvil_uuid}::storage_group::${storage_group_uuid}::vg_size" => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size}}).")", + "anvil_resources::${anvil_uuid}::storage_group::${storage_group_uuid}::free_size" => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}}).")", + "anvil_resources::${anvil_uuid}::storage_group::${storage_group_uuid}::vg_size_on_dr" => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr}}).")", + "anvil_resources::${anvil_uuid}::storage_group::${storage_group_uuid}::available_on_dr" => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr}}).")", + }}); + # Make it easy to sort by group name my $storage_group_name = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{group_name}; $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid} = $storage_group_uuid; diff --git a/Anvil/Tools/Validate.pm b/Anvil/Tools/Validate.pm index dd4a1ccb..8069b3a8 100644 --- a/Anvil/Tools/Validate.pm +++ b/Anvil/Tools/Validate.pm @@ -834,13 +834,13 @@ Parameters; This is the UUID to verify. =cut +### NOTE: Don't call Log->entry from here, causes a deep recursion. sub uuid { my $self = shift; my $parameter = shift; my $anvil = $self->parent; my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Validate->uuid()" }}); my $uuid = defined $parameter->{uuid} ? $parameter->{uuid} : 0; my $valid = 0; diff --git a/share/words.xml b/share/words.xml index d1e38cca..5db55ede 100644 --- a/share/words.xml +++ b/share/words.xml @@ -670,7 +670,7 @@ It should be provisioned in the next minute or two. The DRBD resource: [#!variable!resource!#] configuration has been created and loaded. The DRBD resource: [#!variable!resource!#] metadata has been created. Bringing up the new resource. - Waiting for the disk state to be ready. The current volume: [#!variable!volume!#] disk state is: [#!variale!disk_state!#], waiting for it to become 'UpToDate', 'Consistent', 'Outdated' or 'Inconsistent'. + Waiting for the disk state to be ready. The current volume: [#!variable!volume!#] disk state is: [#!variable!disk_state!#], waiting for it to become 'UpToDate', 'Consistent', 'Outdated' or 'Inconsistent'. The LV(s) behind the resource: [#!variable!resource!#] already existed, and the DRBD resource is not in the disk state 'UpToDate'. As such, we'll keep waiting before provisioning the server. The resource needs to be forced to UpToDate as it is brand now, doing that now. -=] OS Short List @@ -755,6 +755,8 @@ It should be provisioned in the next minute or two. The cluster still hasn't started. Calling startup again (will try once per minute). Successfully added/confirmed the filter in lvm.conf. Failed to add/confirmed the filter in lvm.conf! This should be corrected later by 'scan-drbd' though. + The cluster isn't up. Provisioning the server will hold until it is. Will check every 10 seconds. + The cluster is up. Starting: [#!variable!program!#]. diff --git a/tools/anvil-provision-server b/tools/anvil-provision-server index d3685500..3c9e6403 100755 --- a/tools/anvil-provision-server +++ b/tools/anvil-provision-server @@ -32,6 +32,7 @@ my $anvil = Anvil::Tools->new(); # Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is # passed directly, it will be used. Otherwise, the password will be read from the database. +$anvil->data->{switches}{'anvil-name'} = ""; $anvil->data->{switches}{'anvil-uuid'} = ""; $anvil->data->{switches}{os} = ""; $anvil->data->{switches}{cpu} = ""; @@ -42,8 +43,9 @@ $anvil->data->{switches}{ram} = ""; $anvil->data->{switches}{'storage-group'} = ""; $anvil->data->{switches}{'storage-size'} = ""; $anvil->Get->switches; -$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }}); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 'switches::anvil-name' => $anvil->data->{switches}{'anvil-name'}, 'switches::anvil-uuid' => $anvil->data->{switches}{'anvil-uuid'}, 'switches::os' => $anvil->data->{switches}{os}, 'switches::cpu' => $anvil->data->{switches}{cpu}, @@ -94,7 +96,6 @@ else { # Interactive! interactive_question($anvil); - } $anvil->nice_exit({exit_code => 0}); @@ -123,12 +124,44 @@ sub run_jobs # This parses the jobs::job_data intp variables. parse_job_data($anvil); + # Wait for the cluster to be up, if it's not. + my $waiting = 1; + my $waiting_reported = 0; + while($waiting) + { + my $problem = $anvil->Cluster->parse_cib({debug => 3}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); + if ($problem) + { + # Cluster isn't up yet, wait. + if (not $waiting_reported) + { + $anvil->Job->update_progress({ + progress => 5, + message => "job_0275", + }); + $waiting_reported = 1; + } + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0275"}); + sleep 10; + } + else + { + $waiting = 0; + $anvil->Job->update_progress({ + progress => 8, + message => "job_0276", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0276"}); + } + } + # Sanity checks passed $anvil->Job->update_progress({ progress => 10, message => "job_0185", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0185"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0185"}); # This picks out the next free DRBD minor number and TCP port if it's not set in the job data. check_drbd_minor_and_port($anvil); @@ -138,7 +171,7 @@ sub run_jobs progress => 20, message => "job_0186,!!minor!".$anvil->data->{job}{drbd_minor}."!!,!!port!".$anvil->data->{job}{drbd_tcp_port}."!!", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0186", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0186", variables => { minor => $anvil->data->{job}{drbd_minor}, port => $anvil->data->{job}{drbd_tcp_port}, }}); @@ -238,7 +271,7 @@ sub run_jobs progress => 25, message => "job_0188,!!job_uuid!".$job_uuid."!!,!!peer_name!".$peer_name."!!", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0188", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0188", variables => { job_uuid => $job_uuid, peer_name => $peer_name, }}); @@ -282,7 +315,7 @@ sub run_jobs progress => 100, message => "job_0202", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0202"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0202"}); return(0); } @@ -296,7 +329,7 @@ sub add_server_to_cluster progress => 90, message => "job_0207", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0207"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0207"}); # Is our peer in the cluster? For that matter, are we? my $problem = $anvil->Cluster->add_server({server_name => $anvil->data->{job}{server_name}}); @@ -309,7 +342,7 @@ sub add_server_to_cluster message => "error_0215", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0215"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0215"}); $anvil->nice_exit({exit_code => 1}); } @@ -332,7 +365,7 @@ sub write_definition progress => 70, message => "job_0205", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0205"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0205"}); while (not -f $xml_file) { @@ -340,7 +373,7 @@ sub write_definition $anvil->Database->get_server_definitions(); my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid}; - my $server_definition = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml}; + my $server_definition = exists $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid} ? $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition => $server_definition }}); if ($server_definition) { @@ -358,7 +391,7 @@ sub write_definition progress => 80, message => "job_0206,!!file!".$xml_file."!!", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0206", variables => { file => $xml_file }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0206", variables => { file => $xml_file }}); } } else @@ -373,7 +406,7 @@ sub write_definition progress => 100, message => "job_0204", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0204"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0204"}); return(0); } @@ -424,7 +457,7 @@ sub provision_server progress => 70, message => "job_0199,!!shell_call!".$shell_call."!!", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0199", variables => { shell_call => $shell_call }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0199", variables => { shell_call => $shell_call }}); # Call as a background process. my ($handle, $return_code) = $anvil->System->call({ @@ -443,7 +476,7 @@ sub provision_server progress => 80, message => "job_0200", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0200"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0200"}); # Loop for up to 10 seconds waiting to see the server start running. my $wait_until = time + 60; @@ -488,7 +521,7 @@ sub provision_server message => "error_0210,!!status!".$status."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0210", variables => { status => $status }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0210", variables => { status => $status }}); $anvil->nice_exit({exit_code => 1}); } sleep 1; @@ -499,7 +532,7 @@ sub provision_server progress => 85, message => "job_0201", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0201"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0201"}); # Dump the server's XML. (my $server_definition_xml, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." dumpxml ".$server}); @@ -522,7 +555,7 @@ sub provision_server message => "error_0209,!!shell_call!".$shell_call."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0209", variables => { shell_call => $shell_call }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0209", variables => { shell_call => $shell_call }}); $anvil->nice_exit({exit_code => 1}); } @@ -594,7 +627,7 @@ sub startup_resource } else { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0434", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0434", variables => { resource => $anvil->data->{job}{server_name}, role => $role, }}); @@ -603,7 +636,7 @@ sub startup_resource $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { startup_needed => $startup_needed }}); if (not $startup_needed) { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0431"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0431"}); } else { @@ -612,7 +645,7 @@ sub startup_resource progress => 52, message => "job_0192", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0192"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0192"}); $anvil->DRBD->manage_resource({ debug => 2, resource => $anvil->data->{job}{server_name}, @@ -651,7 +684,7 @@ sub startup_resource progress => 54, message => "job_0193,!!volume!".$volume."!!,!!disk_state!".$disk_state."!!", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0193", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0193", variables => { volume => $volume, disk_state => $disk_state, }}); @@ -668,7 +701,7 @@ sub startup_resource message => "error_0206,!!resource!".$anvil->data->{job}{server_name}."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0206", variables => { resource => $anvil->data->{job}{server_name} }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0206", variables => { resource => $anvil->data->{job}{server_name} }}); $anvil->nice_exit({exit_code => 1}); } if (lc($disk_state) ne "uptodate") @@ -681,7 +714,7 @@ sub startup_resource progress => 56, message => "job_0194,!!resource!".$anvil->data->{job}{server_name}."!!", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0194", variables => { resource => $anvil->data->{job}{server_name} }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0194", variables => { resource => $anvil->data->{job}{server_name} }}); } } @@ -694,7 +727,7 @@ sub startup_resource progress => 58, message => "job_0195", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0195"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0195"}); my $shell_call = $anvil->data->{path}{exe}{drbdsetup}." net-options ".$anvil->data->{job}{server_name}." ".$anvil->data->{job}{drbd_peer_node_id}." --set-defaults --_name=".$anvil->data->{job}{peer_short_name}." --protocol=C --fencing=dont-care"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); @@ -723,13 +756,13 @@ sub startup_resource message => "error_0205,!!return_code!".$return_code."!!,!!output!".$output."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0205", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0205", variables => { return_code => $return_code, output => $output, }}); $anvil->nice_exit({exit_code => 1}); } - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0580", variables => { resource => $anvil->data->{job}{server_name} }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0580", variables => { resource => $anvil->data->{job}{server_name} }}); # set the fencing back $shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$anvil->data->{job}{server_name}; @@ -760,7 +793,7 @@ sub startup_resource message => "error_0207,!!resource!".$anvil->data->{job}{server_name}."!!,!!disk_state!".$disk_state."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0207", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0207", variables => { resource => $anvil->data->{job}{server_name}, disk_state => $disk_state, }}); @@ -787,7 +820,7 @@ sub startup_resource progress => 60, message => "job_0203,!!resource!".$anvil->data->{job}{server_name}."!!", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0203", variables => { resource => $anvil->data->{job}{server_name} }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0203", variables => { resource => $anvil->data->{job}{server_name} }}); return(0); } @@ -799,7 +832,7 @@ sub create_md ### NOTE: The '--max-peers=3' is needed to make space for future DR additions # Create the DRBD metadata - my $shell_call = $anvil->data->{path}{exe}{drbdadm}." -- --force create-md --max-peers=3 ".$anvil->data->{job}{server_name}; + my $shell_call = $anvil->data->{path}{exe}{drbdadm}." --force create-md --max-peers=3 ".$anvil->data->{job}{server_name}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); @@ -810,8 +843,9 @@ sub create_md ### Return codes # 0 == Success + # 1 == ? # 3 == Configuration not found. - if ($return_code eq "3") + if ($return_code) { # Metadata creation failed. $anvil->Job->update_progress({ @@ -819,19 +853,19 @@ sub create_md message => "error_0204,!!return_code!".$return_code."!!,!!output!".$output."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0204", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0204", variables => { return_code => $return_code, output => $output, }}); $anvil->nice_exit({exit_code => 1}); } - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0579", variables => { resource => $anvil->data->{job}{server_name} }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0579", variables => { resource => $anvil->data->{job}{server_name} }}); $anvil->Job->update_progress({ progress => 50, message => "job_0191,!!resource!".$anvil->data->{job}{server_name}."!!", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0191", variables => { resource => $anvil->data->{job}{server_name} }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0191", variables => { resource => $anvil->data->{job}{server_name} }}); return(0); } @@ -918,7 +952,7 @@ sub create_resource_file message => "error_0202,!!drbd_res_file!".$drbd_res_file."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0202", variables => { drbd_res_file => $drbd_res_file }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0202", variables => { drbd_res_file => $drbd_res_file }}); $anvil->nice_exit({exit_code => 1}); } @@ -936,7 +970,7 @@ sub create_resource_file message => "error_0203,!!drbd_res_file!".$drbd_res_file."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0203", variables => { drbd_res_file => $drbd_res_file }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0203", variables => { drbd_res_file => $drbd_res_file }}); $anvil->nice_exit({exit_code => 1}); } @@ -948,7 +982,7 @@ sub create_resource_file progress => 35, message => "job_0218", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0218"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0218"}); my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, @@ -960,7 +994,7 @@ sub create_resource_file progress => 40, message => "job_0190,!!resource!".$anvil->data->{job}{server_name}."!!", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "job_0190", variables => { resource => $anvil->data->{job}{server_name} }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0190", variables => { resource => $anvil->data->{job}{server_name} }}); return(0); } @@ -982,7 +1016,7 @@ sub create_lv progress => 30, message => "job_0187,!!lv_path!".$lv_path."!!", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0187", variables => { lv_path => $lv_path }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0187", variables => { lv_path => $lv_path }}); } else { @@ -999,7 +1033,7 @@ sub create_lv message => "error_0194,!!server_name!".$anvil->data->{job}{server_name}."!!,!!available_size!".$say_available_size."!!,!!requested_size!".$say_requested_size."!!,!!storage_group!".$say_storage_group."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0194", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0194", variables => { server_name => $anvil->data->{job}{server_name}, available_size => $say_available_size, requested_size => $say_requested_size, @@ -1035,7 +1069,7 @@ sub create_lv message => "error_0201,!!lv_path!".$lv_path."!!,!!lv_create!".$shell_call."!!,!!return_code!".$return_code."!!,!!output!".$output."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0201", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0201", variables => { lv_path => $lv_path, lv_create => $shell_call, return_code => $return_code, @@ -1049,7 +1083,7 @@ sub create_lv progress => 30, message => "job_0189,!!lv_path!".$lv_path."!!", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0189", variables => { lv_path => $lv_path }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0189", variables => { lv_path => $lv_path }}); } return(0); @@ -1071,7 +1105,7 @@ sub check_drbd_minor_and_port message => "error_0199,!!storage_group_uuid!".$anvil->data->{job}{storage_group_uuid}."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0199", variables => { storage_group_uuid => $anvil->data->{job}{storage_group_uuid} }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0199", variables => { storage_group_uuid => $anvil->data->{job}{storage_group_uuid} }}); $anvil->nice_exit({exit_code => 1}); } @@ -1098,7 +1132,7 @@ sub check_drbd_minor_and_port message => "error_0200", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0200"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0200"}); $anvil->nice_exit({exit_code => 1}); } @@ -1194,7 +1228,7 @@ sub parse_job_data message => "error_0187,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0187", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0187", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }}); $anvil->nice_exit({exit_code => 1}); } if (not $anvil->data->{job}{storage_group_uuid}) @@ -1204,7 +1238,7 @@ sub parse_job_data message => "error_0192,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0192", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0192", variables => { server_name => $anvil->data->{job}{server_name}, job_uuid => $anvil->data->{switches}{'job-uuid'}, }}); @@ -1217,7 +1251,7 @@ sub parse_job_data message => "error_0193,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0193", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0193", variables => { server_name => $anvil->data->{job}{server_name}, job_uuid => $anvil->data->{switches}{'job-uuid'}, }}); @@ -1255,7 +1289,7 @@ sub parse_job_data message => "error_0198,!!server_name!".$anvil->data->{job}{server_name}."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0198", variables => { server_name => $anvil->data->{job}{server_name} }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0198", variables => { server_name => $anvil->data->{job}{server_name} }}); $anvil->nice_exit({exit_code => 1}); } } @@ -1268,7 +1302,7 @@ sub parse_job_data message => "error_0188,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0188", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0188", variables => { server_name => $anvil->data->{job}{server_name}, job_uuid => $anvil->data->{switches}{'job-uuid'}, }}); @@ -1282,7 +1316,7 @@ sub parse_job_data message => "error_0189,!!server_name!".$anvil->data->{job}{server_name}."!!,!!available_cores!".$anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}."!!,!!requested_cores!".$anvil->data->{job}{cpu_cores}."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0189", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0189", variables => { server_name => $anvil->data->{job}{server_name}, available_cores => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}, requested_cores => $anvil->data->{job}{server_name}, @@ -1297,7 +1331,7 @@ sub parse_job_data message => "error_0190,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0190", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0190", variables => { server_name => $anvil->data->{job}{server_name}, job_uuid => $anvil->data->{switches}{'job-uuid'}, }}); @@ -1313,7 +1347,7 @@ sub parse_job_data message => "error_0191,!!server_name!".$anvil->data->{job}{server_name}."!!,!!available_ram!".$say_available_ram."!!,!!requested_ram!".$say_requested_ram."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0191", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0191", variables => { server_name => $anvil->data->{job}{server_name}, available_ram => $say_available_ram, requested_ram => $say_requested_ram, @@ -1328,7 +1362,7 @@ sub parse_job_data message => "error_0195,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0195", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0195", variables => { server_name => $anvil->data->{job}{server_name}, job_uuid => $anvil->data->{switches}{'job-uuid'}, }}); @@ -1354,7 +1388,7 @@ sub parse_job_data message => "error_0196,!!server_name!".$anvil->data->{job}{server_name}."!!,!!install_iso!".$install_iso."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0196", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0196", variables => { server_name => $anvil->data->{job}{server_name}, install_iso => $install_iso, }}); @@ -1380,7 +1414,7 @@ sub parse_job_data message => "error_0197,!!server_name!".$anvil->data->{job}{server_name}."!!,!!driver_iso!".$driver_iso."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0197", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0197", variables => { server_name => $anvil->data->{job}{server_name}, driver_iso => $driver_iso, }}); @@ -1395,7 +1429,7 @@ sub parse_job_data message => "error_0208,!!server_name!".$anvil->data->{job}{server_name}."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0208", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0208", variables => { server_name => $anvil->data->{job}{server_name}, job_uuid => $anvil->data->{switches}{'job-uuid'}, }}); @@ -1882,10 +1916,7 @@ sub interactive_ask_server_storage_group { my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid}; $anvil->Database->get_anvils(); - $anvil->Get->available_resources({anvil_uuid => $anvil_uuid}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "anvil_resources::${anvil_uuid}::ram::available" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")", - }}); + $anvil->Get->available_resources({debug => 2, anvil_uuid => $anvil_uuid}); # I need a list of Storage groups, my $say_ram = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}}); @@ -1899,13 +1930,16 @@ sub interactive_ask_server_storage_group my $storage_group_uuid = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid}; my $vg_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size}; my $vg_free = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}; - my $dr_size = 0; - my $dr_free = 0; - if ($anvil->data->{anvil_resources}{$anvil_uuid}{has_dr}) - { - $dr_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr}; - $dr_free = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr}; - } + my $dr_size = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr} : 0; + my $dr_free = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr} : 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:storage_group_name' => $storage_group_name, + 's2:storage_group_uuid' => $storage_group_uuid, + 's3:vg_size' => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")", + 's4:vg_free' => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")", + 's5:dr_size' => $dr_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_size}).")", + 's6:dr_free' => $dr_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_free}).")", + }}); if ($anvil->data->{switches}{'storage-group'}) { @@ -1984,21 +2018,54 @@ sub interactive_ask_server_storage_size my $retry = 0; while(1) { - my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid}; + my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid}; + my $storage_group_uuid = $anvil->data->{new_server}{storage_group}; + my $storage_group_name = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name}; + my $vg_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size}; + my $vg_free = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}; + my $dr_size = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr} : 0; + my $dr_free = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr} : 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:anvil_uuid' => $anvil_uuid, + 's2:storage_group_name' => $storage_group_name, + 's3:storage_group_uuid' => $storage_group_uuid, + 's4:vg_size' => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")", + 's5:vg_free' => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")", + 's6:dr_size' => $dr_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_size}).")", + 's7:dr_free' => $dr_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_free}).")", + }}); + $anvil->Database->get_anvils(); - $anvil->Get->available_resources({anvil_uuid => $anvil_uuid}); + $anvil->Database->get_storage_group_data(); + $anvil->Get->available_resources({debug => 2, anvil_uuid => $anvil_uuid}); + + $vg_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size}; + $vg_free = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}; + $dr_size = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr} : 0; + $dr_free = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr} : 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:vg_size' => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")", + 's2:vg_free' => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")", + 's3:dr_size' => $dr_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_size}).")", + 's4:dr_free' => $dr_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_free}).")", + }}); # I need a list of Storage groups, my $default_storage_size = 0; - my $storage_group_uuid = $anvil->data->{new_server}{storage_group}; - my $vg_free = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}; + $vg_free = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}; $default_storage_size = $anvil->data->{switches}{'storage-size'}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + vg_free => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free})." (".$anvil->Convert->add_commas({number => $vg_free}).")", + default_storage_size => $default_storage_size, + }}); + if (not $default_storage_size) { if ($vg_free < (80 * (2**30))) { # Too small for default $default_storage_size = $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_storage_size => $default_storage_size }}); } else {