diff --git a/Anvil/Tools/Cluster.pm b/Anvil/Tools/Cluster.pm index 3b9fe02a..16c39af1 100644 --- a/Anvil/Tools/Cluster.pm +++ b/Anvil/Tools/Cluster.pm @@ -35,6 +35,7 @@ my $THIS_FILE = "Cluster.pm"; # parse_cib # parse_crm_mon # parse_quorum +# recover_server # shutdown_server # start_cluster # which_node @@ -863,9 +864,10 @@ sub check_node_status =head2 check_server_constraints -This method checks to see if the peer node is offline, and the local node is only. If this is the case, the location constraints for servers are checked to ensure that they favour the current host. If not, the location constraint is updated. +This checks to see if the constraints on a server are sane. Specifically; -This is meant to be used to prevent servers from automatically migrating back to a node after it was fenced. +* If the server is on a sub-node and the peer is offline, ensure that the location constraints prefer the current host. This prevents migrations back to the old host. +* Check to see if a DRBD resource constriant was applied against a node, and the node's DRBD resource is UpToDate. If so, remove the constraint. This method takes no parameters. @@ -2204,17 +2206,14 @@ sub get_anvil_uuid my $anvil_name = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name}; my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - my $anvil_dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_name => $anvil_name, anvil_node1_host_uuid => $anvil_node1_host_uuid, anvil_node2_host_uuid => $anvil_node2_host_uuid, - anvil_dr1_host_uuid => $anvil_dr1_host_uuid, }}); if (($host_uuid eq $anvil_node1_host_uuid) or - ($host_uuid eq $anvil_node2_host_uuid) or - ($host_uuid eq $anvil_dr1_host_uuid)) + ($host_uuid eq $anvil_node2_host_uuid)) { # Found ot! $member_anvil_uuid = $anvil_uuid; @@ -2237,13 +2236,11 @@ The data is stored as; sys::anvil::node1::host_name sys::anvil::node2::host_uuid sys::anvil::node2::host_name - sys::anvil::dr1::host_uuid - sys::anvil::dr1::host_name To assist with lookup, the following are also set; - sys::anvil::i_am = {node1,node2,dr1} - sys::anvil::peer_is = {node1,node2} # Not set if this host is 'dr1' + sys::anvil::i_am = {node1,node2} + sys::anvil::peer_is = {node1,node2} This method takes no parameters. @@ -2260,8 +2257,6 @@ sub get_peers $anvil->data->{sys}{anvil}{node1}{host_name} = ""; $anvil->data->{sys}{anvil}{node2}{host_uuid} = ""; $anvil->data->{sys}{anvil}{node2}{host_name} = ""; - $anvil->data->{sys}{anvil}{dr1}{host_uuid} = ""; - $anvil->data->{sys}{anvil}{dr1}{host_name} = ""; $anvil->data->{sys}{anvil}{i_am} = ""; $anvil->data->{sys}{anvil}{peer_is} = ""; @@ -2279,11 +2274,9 @@ sub get_peers { my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - my $anvil_dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_node1_host_uuid => $anvil_node1_host_uuid, anvil_node2_host_uuid => $anvil_node2_host_uuid, - anvil_dr1_host_uuid => $anvil_dr1_host_uuid, }}); if ($host_uuid eq $anvil_node1_host_uuid) @@ -2310,27 +2303,17 @@ sub get_peers "sys::anvil::peer_is" => $anvil->data->{sys}{anvil}{peer_is}, }}); } - elsif ($host_uuid eq $anvil_dr1_host_uuid) - { - # Found our Anvil!, and we're node 1. - $found = 1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { found => $found }}); - } if ($found) { $anvil->data->{sys}{anvil}{node1}{host_uuid} = $anvil_node1_host_uuid; $anvil->data->{sys}{anvil}{node1}{host_name} = $anvil->data->{hosts}{host_uuid}{$anvil_node1_host_uuid}{host_name}; $anvil->data->{sys}{anvil}{node2}{host_uuid} = $anvil_node2_host_uuid; $anvil->data->{sys}{anvil}{node2}{host_name} = $anvil->data->{hosts}{host_uuid}{$anvil_node2_host_uuid}{host_name}; - $anvil->data->{sys}{anvil}{dr1}{host_uuid} = $anvil_dr1_host_uuid ? $anvil_dr1_host_uuid : ""; - $anvil->data->{sys}{anvil}{dr1}{host_name} = $anvil_dr1_host_uuid ? $anvil->data->{hosts}{host_uuid}{$anvil_dr1_host_uuid}{host_name} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::anvil::node1::host_uuid" => $anvil->data->{sys}{anvil}{node1}{host_uuid}, "sys::anvil::node1::host_name" => $anvil->data->{sys}{anvil}{node1}{host_name}, "sys::anvil::node2::host_uuid" => $anvil->data->{sys}{anvil}{node2}{host_uuid}, "sys::anvil::node2::host_name" => $anvil->data->{sys}{anvil}{node2}{host_name}, - "sys::anvil::dr1::host_uuid" => $anvil->data->{sys}{anvil}{dr1}{host_uuid}, - "sys::anvil::dr1::host_name" => $anvil->data->{sys}{anvil}{dr1}{host_name}, }}); # If this is a node, return the peer's short host name. @@ -4286,6 +4269,49 @@ sub parse_quorum } +=head2 recover_server + +This tries to recover a C<< FAILED >> resource (server). + +Parameters; + +=head3 server_ (required) + +This is the server (resource) name to try to recover. + +=cut +sub recover_server +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Cluster->recover_server()" }}); + + my $server = defined $parameter->{server} ? $parameter->{server} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + server => $server, + }}); + + if (not $server) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Cluster->recover_server()", parameter => "server" }}); + return("!!error!!"); + } + + my $shell_call = $anvil->data->{path}{exe}{crm_resource}." --resource ".$server." --refresh"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }}); + + my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + output => $output, + return_code => $return_code, + }}); + + return(0); +} + + =head2 shutdown_server This shuts down a server that is running on the Anvil! system. If there is a problem, C<< !!error!! >> is returned. On success, C<< 0 >> is returned. diff --git a/Anvil/Tools/DRBD.pm b/Anvil/Tools/DRBD.pm index 40c7264f..93cfc116 100644 --- a/Anvil/Tools/DRBD.pm +++ b/Anvil/Tools/DRBD.pm @@ -663,6 +663,19 @@ sub delete_resource } } + # If we're DR, delete the definition file also. + my $definition_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$resource.".xml"; + my $host_type = $anvil->Get->host_type({debug => $debug}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + definition_file => $definition_file, + host_type => $host_type, + }}); + if (($host_type eq "dr") && (-f $definition_file)) + { + unlink $definition_file; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0134", variables => { file_path => $definition_file }}); + } + return(0); } @@ -1764,11 +1777,9 @@ sub get_next_resource # certain. my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - my $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { node1_host_uuid => $node1_host_uuid, node2_host_uuid => $node2_host_uuid, - dr1_host_uuid => $dr1_host_uuid, }}); my $query = " @@ -1805,14 +1816,7 @@ AND ( scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node1_host_uuid)." OR - scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node2_host_uuid)." "; - if ($dr1_host_uuid) - { - $query .= " - OR - scan_drbd_resource_host_uuid = ".$anvil->Database->quote($dr1_host_uuid)." "; - } - $query .= " + scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node2_host_uuid)." ) ORDER BY b.scan_drbd_resource_name ASC, diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 0f3d05e6..a05d6a43 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -2661,7 +2661,6 @@ Data is stored in two hashes, one sorted by C<< anvil_uuid >> and one by C<< anv anvils::anvil_uuid::::anvil_password anvils::anvil_uuid::::anvil_node1_host_uuid anvils::anvil_uuid::::anvil_node2_host_uuid - anvils::anvil_uuid::::anvil_dr1_host_uuid anvils::anvil_uuid::::modified_date anvils::anvil_name::::anvil_uuid @@ -2669,7 +2668,6 @@ Data is stored in two hashes, one sorted by C<< anvil_uuid >> and one by C<< anv anvils::anvil_name::::anvil_password anvils::anvil_name::::anvil_node1_host_uuid anvils::anvil_name::::anvil_node2_host_uuid - anvils::anvil_name::::anvil_dr1_host_uuid anvils::anvil_name::::modified_date When a host UUID is stored for either node or the DR host, it will be stored at: @@ -3592,21 +3590,23 @@ FROM if ($file_name) { - $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid} = $file_location_file_uuid; - $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_name} = $file_name; - $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_directory} = $file_directory; - $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size} = $file_size; - $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_md5sum} = $file_md5sum; - $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_type} = $file_type; - $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_mtime} = $file_mtime; + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid} = $file_location_file_uuid; + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_name} = $file_name; + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_directory} = $file_directory; + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size} = $file_size; + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_md5sum} = $file_md5sum; + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_type} = $file_type; + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_mtime} = $file_mtime; + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_location_uuid} = $file_location_uuid; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "anvils::anvil_uuid::${anvil_uuid}::file_name::${file_name}::file_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}, - "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_name" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_name}, - "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_directory" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_directory}, - "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_size" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size}}).")", - "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_md5sum" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_md5sum}, - "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_type" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_type}, - "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_mtime" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_mtime}, + "anvils::anvil_uuid::${anvil_uuid}::file_name::${file_name}::file_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}, + "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_name" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_name}, + "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_directory" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_directory}, + "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_size" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size}}).")", + "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_md5sum" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_md5sum}, + "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_type" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_type}, + "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_mtime" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_mtime}, + "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_location_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_location_uuid}, }}); } } @@ -6999,12 +6999,6 @@ If set, this is the file line number logged as the source of any INSERTs or UPDA This is a free-form description for this Anvil! system. If this is set to C<< DELETED >>, the Anvil! will be considered to be deleted and no longer used. -=head3 anvil_dr1_host_uuid (optional) - -This is the C<< hosts >> -> C<< host_uuid >> of the machine that is used as the DR host. - -B<< Note >>: If set, there must be a matching C<< hosts >> -> C<< host_uuid >> in the database. - =head3 anvil_name (required) This is the anvil's name. It is usually in the format C<< -anvil- >>. @@ -7052,7 +7046,6 @@ sub insert_or_update_anvils my $anvil_password = defined $parameter->{anvil_password} ? $parameter->{anvil_password} : ""; my $anvil_node1_host_uuid = defined $parameter->{anvil_node1_host_uuid} ? $parameter->{anvil_node1_host_uuid} : "NULL"; my $anvil_node2_host_uuid = defined $parameter->{anvil_node2_host_uuid} ? $parameter->{anvil_node2_host_uuid} : "NULL"; - my $anvil_dr1_host_uuid = defined $parameter->{anvil_dr1_host_uuid} ? $parameter->{anvil_dr1_host_uuid} : "NULL"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid, file => $file, @@ -7064,7 +7057,6 @@ sub insert_or_update_anvils anvil_password => $anvil->Log->is_secure($anvil_password), anvil_node1_host_uuid => $anvil_node1_host_uuid, anvil_node2_host_uuid => $anvil_node2_host_uuid, - anvil_dr1_host_uuid => $anvil_dr1_host_uuid, }}); if (not $delete) @@ -7133,11 +7125,6 @@ WHERE $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0128", variables => { uuid => $anvil_node2_host_uuid, column => "anvil_node2_host_uuid" }}); return(""); } - if (($anvil_dr1_host_uuid) && (not $anvil->data->{hosts}{host_uuid}{$anvil_dr1_host_uuid}{host_name})) - { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0128", variables => { uuid => $anvil_dr1_host_uuid, column => "anvil_dr1_host_uuid" }}); - return(""); - } if ($delete) { @@ -7190,8 +7177,6 @@ WHERE # NULL values can't be quoted my $say_anvil_node1_host_uuid = $anvil_node1_host_uuid eq "" ? "NULL" : $anvil->Database->quote($anvil_node1_host_uuid); my $say_anvil_node2_host_uuid = $anvil_node2_host_uuid eq "" ? "NULL" : $anvil->Database->quote($anvil_node2_host_uuid); - my $say_anvil_dr1_host_uuid = $anvil_dr1_host_uuid eq "" ? "NULL" : $anvil->Database->quote($anvil_dr1_host_uuid); - # If I still don't have an anvil_uuid, we're INSERT'ing . $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_uuid => $anvil_uuid }}); @@ -7211,7 +7196,6 @@ INSERT INTO anvil_password, anvil_node1_host_uuid, anvil_node2_host_uuid, - anvil_dr1_host_uuid, modified_date ) VALUES ( ".$anvil->Database->quote($anvil_uuid).", @@ -7220,7 +7204,6 @@ INSERT INTO ".$anvil->Database->quote($anvil_password).", ".$say_anvil_node1_host_uuid.", ".$say_anvil_node2_host_uuid.", - ".$say_anvil_dr1_host_uuid.", ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." ); "; @@ -7236,8 +7219,7 @@ SELECT anvil_description, anvil_password, anvil_node1_host_uuid, - anvil_node2_host_uuid, - anvil_dr1_host_uuid + anvil_node2_host_uuid FROM anvils WHERE @@ -7264,14 +7246,12 @@ WHERE my $old_anvil_password = $row->[2]; my $old_anvil_node1_host_uuid = defined $row->[3] ? $row->[3] : "NULL"; my $old_anvil_node2_host_uuid = defined $row->[4] ? $row->[4] : "NULL"; - my $old_anvil_dr1_host_uuid = defined $row->[5] ? $row->[5] : "NULL"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { old_anvil_name => $old_anvil_name, old_anvil_description => $old_anvil_description, old_anvil_password => $anvil->Log->is_secure($old_anvil_password), old_anvil_node1_host_uuid => $old_anvil_node1_host_uuid, old_anvil_node2_host_uuid => $old_anvil_node2_host_uuid, - old_anvil_dr1_host_uuid => $old_anvil_dr1_host_uuid, }}); # Anything change? @@ -7279,8 +7259,7 @@ WHERE ($old_anvil_description ne $anvil_description) or ($old_anvil_password ne $anvil_password) or ($old_anvil_node1_host_uuid ne $anvil_node1_host_uuid) or - ($old_anvil_node2_host_uuid ne $anvil_node2_host_uuid) or - ($old_anvil_dr1_host_uuid ne $anvil_dr1_host_uuid)) + ($old_anvil_node2_host_uuid ne $anvil_node2_host_uuid)) { # Something changed, save. my $query = " @@ -7292,7 +7271,6 @@ SET anvil_password = ".$anvil->Database->quote($anvil_password).", anvil_node1_host_uuid = ".$say_anvil_node1_host_uuid.", anvil_node2_host_uuid = ".$say_anvil_node2_host_uuid.", - anvil_dr1_host_uuid = ".$say_anvil_dr1_host_uuid.", modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." WHERE anvil_uuid = ".$anvil->Database->quote($anvil_uuid)." diff --git a/Anvil/Tools/ScanCore.pm b/Anvil/Tools/ScanCore.pm index d1b670a5..8af2b18f 100644 --- a/Anvil/Tools/ScanCore.pm +++ b/Anvil/Tools/ScanCore.pm @@ -1459,7 +1459,11 @@ sub post_scan_analysis_dr my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "ScanCore->post_scan_analysis_dr()" }}); - + # Now that DR hosts are outside of specific Anvil! systems, auto-configuring their IPMI is more + # tricky. If there's no 'host_ipmi' set, and if there is a BCN1 IP, we'll configure using the BCN's + # third octet, plus 1. We'll use the password used for the database password. Later, if the IPMI + # is changed in the DB, we'll reconfigure to match. + $anvil->System->configure_ipmi({debug => $debug, dr => 1}); return(0); } diff --git a/Anvil/Tools/Storage.pm b/Anvil/Tools/Storage.pm index f325390e..92595c26 100644 --- a/Anvil/Tools/Storage.pm +++ b/Anvil/Tools/Storage.pm @@ -1767,16 +1767,13 @@ sub get_size_of_block_device $anvil->Database->get_anvils({debug => $debug}); my $node1_host_uuid = ""; my $node2_host_uuid = ""; - my $dr1_host_uuid = ""; if ($anvil_uuid) { $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { node1_host_uuid => $node1_host_uuid, node1_host_uuid => $node2_host_uuid, - dr1_host_uuid => $dr1_host_uuid, }}); } @@ -1807,19 +1804,11 @@ AND ( scan_lvm_lv_host_uuid = ".$anvil->Database->quote($node1_host_uuid)." OR - scan_lvm_lv_host_uuid = ".$anvil->Database->quote($node2_host_uuid); - if ($dr1_host_uuid) - { - $query .= " - OR - scan_lvm_lv_host_uuid = ".$anvil->Database->quote($dr1_host_uuid); - } - $query .= " - )"; - } - $query .= " + scan_lvm_lv_host_uuid = ".$anvil->Database->quote($node2_host_uuid)." + ) LIMIT 1 ;"; + } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; @@ -1911,11 +1900,7 @@ LIMIT 1 ($anvil_uuid) && ( ($this_host_uuid eq $node1_host_uuid) or - ($this_host_uuid eq $node2_host_uuid) or - ( - ($dr1_host_uuid) && - ($this_host_uuid eq $dr1_host_uuid) - ) + ($this_host_uuid eq $node2_host_uuid) ) ) ) @@ -1976,14 +1961,7 @@ AND ( a.scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node1_host_uuid)." OR - a.scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node2_host_uuid); - if ($dr1_host_uuid) - { - $query .= " - OR - a.scan_drbd_resource_host_uuid = ".$anvil->Database->quote($dr1_host_uuid); - } - $query .= " + a.scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node2_host_uuid)." )"; } $query .= " @@ -2222,16 +2200,13 @@ sub get_storage_group_from_path $anvil->Database->get_anvils({debug => $debug}); my $node1_host_uuid = ""; my $node2_host_uuid = ""; - my $dr1_host_uuid = ""; if ($anvil_uuid) { $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { node1_host_uuid => $node1_host_uuid, node1_host_uuid => $node2_host_uuid, - dr1_host_uuid => $dr1_host_uuid, }}); } @@ -2272,11 +2247,7 @@ sub get_storage_group_from_path ($anvil_uuid) && ( ($this_host_uuid eq $node1_host_uuid) or - ($this_host_uuid eq $node2_host_uuid) or - ( - ($dr1_host_uuid) && - ($this_host_uuid eq $dr1_host_uuid) - ) + ($this_host_uuid eq $node2_host_uuid) ) ) ) diff --git a/Anvil/Tools/System.pm b/Anvil/Tools/System.pm index b395a793..601ca60f 100644 --- a/Anvil/Tools/System.pm +++ b/Anvil/Tools/System.pm @@ -1897,6 +1897,10 @@ B<< NOTE >>: The password used to set the IPMI BMC access is included both in th Parameters; +=head3 dr (Optional, default 0) + +This indicates that a DR host is being configured. If used, C<< manifest_uuid >> is ignored. + =head3 manifest_uuid (Optional, default sys::manifest_uuid) The C<< manifests >> -> C<< manifest_uuid >> used to pull out configuration data. This is required, but in most cases, it can be determined if not passed. @@ -1912,65 +1916,138 @@ sub configure_ipmi my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "System->configure_ipmi()" }}); + my $dr = defined $parameter->{dr} ? $parameter->{dr} : ""; my $manifest_uuid = defined $parameter->{manifest_uuid} ? $parameter->{manifest_uuid} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + dr => $dr, manifest_uuid => $manifest_uuid, }}); - if ((not $manifest_uuid) && (exists $anvil->data->{sys}{manifest_uuid}) && ($anvil->data->{sys}{manifest_uuid})) - { - $manifest_uuid = $anvil->data->{sys}{manifest_uuid}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { manifest_uuid => $manifest_uuid }}); - } - if (not $manifest_uuid) - { - # Nothing more we can do. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Systeme->configure_ipmi()", parameter => "manifest_uuid" }}); - return(0); - } - - # Is this host in an Anvil!? $anvil->Database->get_hosts(); $anvil->Database->get_anvils(); - - my $anvil_uuid = ""; - my $host_uuid = $anvil->Get->host_uuid; - if ((exists $anvil->data->{hosts}{host_uuid}{$host_uuid}) && ($anvil->data->{hosts}{host_uuid}{$host_uuid}{anvil_uuid})) - { - # We're in an Anvil! - $anvil_uuid = $anvil->data->{hosts}{host_uuid}{$host_uuid}{anvil_uuid}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_uuid => $anvil_uuid }}); + my $host_uuid = $anvil->Get->host_uuid; + my $ipmi_ip_address = ""; + my $ipmi_password = ""; + my $password_length = 0; + my $subnet_mask = ""; + my $gateway = ""; + my $in_network = ""; + if ($dr) + { + my $host_ipmi = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_ipmi => $host_ipmi }}); + if (not $host_ipmi) + { + # Do initial config. + $anvil->Network->load_ips({ + debug => $debug, + host_uuid => $host_uuid, + }); + foreach my $interface_name (sort {$a cmp $b} keys %{$anvil->data->{network}{$host_uuid}{interface}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { interface_name => $interface_name }}); + next if $interface_name !~ /^bcn1_/; + + my $ip_address_address = $anvil->data->{network}{$host_uuid}{interface}{$interface_name}{ip}; + my $ip_address_subnet_mask = $anvil->data->{network}{$host_uuid}{interface}{$interface_name}{subnet_mask}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + ip_address_address => $ip_address_address, + ip_address_subnet_mask => $ip_address_subnet_mask, + }}); + + # We need this to be a /16 + if ($ip_address_subnet_mask eq "255.255.0.0") + { + # Get the 3rd octet. + my ($first_octet, $second_octet, $third_octet, $fourth_octet) = ($ip_address_address =~ /^(\d+)\.(\d+)\.(\d+)\.(\d+)$/); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + first_octet => $first_octet, + second_octet => $second_octet, + third_octet => $third_octet, + fourth_octet => $fourth_octet, + }}); + + $ipmi_ip_address = $first_octet.".".$second_octet.".".($third_octet+1).".".$fourth_octet; + $subnet_mask = $ip_address_subnet_mask; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + ipmi_ip_address => $ipmi_ip_address, + subnet_mask => $subnet_mask, + }}); + last; + } + } + + # Get the password using the Striker password. + my $db_uuid = $anvil->data->{sys}{database}{read_uuid}; + $ipmi_password = $anvil->data->{database}{$db_uuid}{password}; + $password_length = length(Encode::encode('UTF-8', $ipmi_password)); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + db_uuid => $db_uuid, + ipmi_password => $anvil->Log->is_secure($ipmi_password), + password_length => $password_length, + }}); + + if ((not $anvil->Validate->ipv4({debug => $debug, ip => $ipmi_ip_address})) or (not $ipmi_password)) + { + # Can't proceed. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "warning_0149"}); + return(0); + } + } + else + { + # TODO: Check if the local IPMI config matches the value in host_ipmi and, if not, + # reconfigure to match. + return(0); + } } else { - # Not in an Anvil!, return 0. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0498"}); - return(0); - } - - # Look for a match in the anvils table for this host uuid. - my $machine = ""; - if ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid} eq $host_uuid) - { - $machine = "node1"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { machine => $machine }}); - } - elsif ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid} eq $host_uuid) - { - $machine = "node2"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { machine => $machine }}); - } - elsif ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} eq $host_uuid) - { - $machine = "dr1"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { machine => $machine }}); - } - - if (not $machine) - { - # Look for a job for 'anvil-join-anvil' for this host. With it, we'll figure out the password - # and which machine we are. - my $query = " + if ((not $manifest_uuid) && (exists $anvil->data->{sys}{manifest_uuid}) && ($anvil->data->{sys}{manifest_uuid})) + { + $manifest_uuid = $anvil->data->{sys}{manifest_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { manifest_uuid => $manifest_uuid }}); + } + if (not $manifest_uuid) + { + # Nothing more we can do. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Systeme->configure_ipmi()", parameter => "manifest_uuid" }}); + return(0); + } + + # Is this host in an Anvil!? + my $anvil_uuid = ""; + if ((exists $anvil->data->{hosts}{host_uuid}{$host_uuid}) && ($anvil->data->{hosts}{host_uuid}{$host_uuid}{anvil_uuid})) + { + # We're in an Anvil! + $anvil_uuid = $anvil->data->{hosts}{host_uuid}{$host_uuid}{anvil_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_uuid => $anvil_uuid }}); + } + else + { + # Not in an Anvil!, return 0. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0498"}); + return(0); + } + + # Look for a match in the anvils table for this host uuid. + my $machine = ""; + if ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid} eq $host_uuid) + { + $machine = "node1"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { machine => $machine }}); + } + elsif ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid} eq $host_uuid) + { + $machine = "node2"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { machine => $machine }}); + } + + if (not $machine) + { + # Look for a job for 'anvil-join-anvil' for this host. With it, we'll figure out the password + # and which machine we are. + my $query = " SELECT job_uuid, job_data @@ -1984,56 +2061,109 @@ ORDER BY modified_date DESC LIMIT 1 ;"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); - my $count = @{$results}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - results => $results, - count => $count, - }}); - my $job_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : ""; - my $job_data = defined $results->[0]->[1] ? $results->[0]->[1] : ""; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - job_uuid => $job_uuid, - job_data => $anvil->Log->is_secure($job_data), - }}); - if (not $job_uuid) + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + results => $results, + count => $count, + }}); + my $job_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : ""; + my $job_data = defined $results->[0]->[1] ? $results->[0]->[1] : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + job_uuid => $job_uuid, + job_data => $anvil->Log->is_secure($job_data), + }}); + if (not $job_uuid) + { + # Unable to proceed. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0501"}); + return(0); + } + + ($machine, $manifest_uuid, $anvil_uuid) = ($job_data =~ /as_machine=(.*?),manifest_uuid=(.*?),anvil_uuid=(.*?)$/); + $machine = "" if not defined $machine; + $manifest_uuid = "" if not defined $manifest_uuid; + $anvil_uuid = "" if not defined $anvil_uuid; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + machine => $machine, + manifest_uuid => $manifest_uuid, + anvil_uuid => $anvil_uuid, + }}); + } + + # Load the manifest. + my $problem = $anvil->Striker->load_manifest({debug => $debug, manifest_uuid => $manifest_uuid}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }}); + if ($problem) { - # Unable to proceed. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0501"}); + # The load_manifest method would log the details. return(0); } - ($machine, $manifest_uuid, $anvil_uuid) = ($job_data =~ /as_machine=(.*?),manifest_uuid=(.*?),anvil_uuid=(.*?)$/); - $machine = "" if not defined $machine; - $manifest_uuid = "" if not defined $manifest_uuid; - $anvil_uuid = "" if not defined $anvil_uuid; + # Make sure the IPMI IP, subnet mask and password are available. + $ipmi_ip_address = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{ipmi_ip}; + $ipmi_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password}; + $password_length = length(Encode::encode('UTF-8', $ipmi_password)); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - machine => $machine, - manifest_uuid => $manifest_uuid, - anvil_uuid => $anvil_uuid, + ipmi_ip_address => $ipmi_ip_address, + ipmi_password => $anvil->Log->is_secure($ipmi_password), + password_length => $password_length, }}); + + # Find the subnet the IPMI IP is in. + foreach my $network_type ("bcn", "ifn", "sn") + { + my $count = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{count}{$network_type}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + network_type => $network_type, + count => $count, + }}); + foreach my $i (1..$count) + { + my $network_name = $network_type.$i; + my $network = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network_name}{network}; + my $this_subnet_mask = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network_name}{subnet}; + my $this_gateway = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network_name}{gateway}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + network_name => $network_name, + network => $network, + this_subnet_mask => $this_subnet_mask, + this_gateway => $this_gateway, + }}); + + my $match = $anvil->Network->is_ip_in_network({ + network => $network, + subnet_mask => $this_subnet_mask, + ip => $ipmi_ip_address, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { network_name => $match}}); + if ($match) + { + $subnet_mask = $this_subnet_mask; + $gateway = $this_gateway; + $in_network = $network_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + subnet_mask => $subnet_mask, + gateway => $gateway, + in_network => $in_network, + }}); + last; + } + } + } + + # If we didn't find a network, we're done. + if (not $subnet_mask) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0502", variables => { + ip_address => $ipmi_ip_address, + manifest_uuid => $manifest_uuid, + }}); + return(0); + } } - # Load the manifest. - my $problem = $anvil->Striker->load_manifest({debug => $debug, manifest_uuid => $manifest_uuid}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }}); - if ($problem) - { - # The load_manifest method would log the details. - return(0); - } - - # Make sure the IPMI IP, subnet mask and password are available. - my $ipmi_ip_address = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{ipmi_ip}; - my $ipmi_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password}; - my $password_length = length(Encode::encode('UTF-8', $ipmi_password)); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - ipmi_ip_address => $ipmi_ip_address, - ipmi_password => $anvil->Log->is_secure($ipmi_password), - password_length => $password_length, - }}); - # If the password has spaces, some IPMI BMCs won't allow them. If we need to use it, we'll take out # the spaces and shrink the length. my $ipmi_no_space_password = ""; @@ -2044,61 +2174,7 @@ LIMIT 1 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => 1, list => { ipmi_no_space_password => $ipmi_no_space_password }}); } - my $subnet_mask = ""; - my $gateway = ""; - my $in_network = ""; - # Find the subnet the IPMI IP is in. - foreach my $network_type ("bcn", "ifn", "sn") - { - my $count = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{count}{$network_type}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - network_type => $network_type, - count => $count, - }}); - foreach my $i (1..$count) - { - my $network_name = $network_type.$i; - my $network = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network_name}{network}; - my $this_subnet_mask = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network_name}{subnet}; - my $this_gateway = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network_name}{gateway}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - network_name => $network_name, - network => $network, - this_subnet_mask => $this_subnet_mask, - this_gateway => $this_gateway, - }}); - - my $match = $anvil->Network->is_ip_in_network({ - network => $network, - subnet_mask => $this_subnet_mask, - ip => $ipmi_ip_address, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { network_name => $match}}); - if ($match) - { - $subnet_mask = $this_subnet_mask; - $gateway = $this_gateway; - $in_network = $network_name; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - subnet_mask => $subnet_mask, - gateway => $gateway, - in_network => $in_network, - }}); - last; - } - } - } - - # If we didn't find a network, we're done. - if (not $subnet_mask) - { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0502", variables => { - ip_address => $ipmi_ip_address, - manifest_uuid => $manifest_uuid, - }}); - return(0); - } # Call dmidecode to see if there even is an IPMI BMC on this host. my $host_ipmi = ""; diff --git a/Anvil/Tools/Words.pm b/Anvil/Tools/Words.pm index 4746e8e5..a6fb6e77 100644 --- a/Anvil/Tools/Words.pm +++ b/Anvil/Tools/Words.pm @@ -206,7 +206,7 @@ sub clean_spaces $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Words->clean_spaces()" }}); # Setup default values - my $string = defined $parameter->{string} ? $parameter->{string} : ""; + my $string = defined $parameter->{string} ? $parameter->{string} : ""; my $merge_spaces = defined $parameter->{merge_spaces} ? $parameter->{merge_spaces} : 1; $string =~ s/^\s+//; diff --git a/notes b/notes index 27fcdbb5..e302d858 100644 --- a/notes +++ b/notes @@ -1,8 +1,28 @@ + +Tasks; +1. When provisioning a server; + - First, check if either node is SyncSource, if so use that node. + - Second, check which node has the most servers by RAM count, use that node. +2. Provision; + - Create DRBD resource, force primary on install target + - Create pacemaker resource in stopped state + - Set location constraint to prefer target node + - Boot server + +Create "Node status" which returns "degraded" if the peer is gone + + Common queries; * SELECT a.job_uuid, b.host_name, a.job_command, a.job_data, a.job_progress, a.job_status FROM jobs a, hosts b WHERE a.job_host_uuid = b.host_uuid AND a.job_progress != 100; * SELECT a.host_name, b.file_name, c.file_location_active FROM hosts a, files b, file_locations c WHERE a.host_uuid = c.file_location_host_uuid AND b.file_uuid = c.file_location_file_uuid ORDER BY b.file_name ASC, a.host_name ASC; * SELECT a.dr_link_uuid, b.host_name, c.anvil_name, a.dr_link_note FROM dr_links a, hosts b, anvils c WHERE a.dr_link_host_uuid = b.host_uuid AND a.dr_link_anvil_uuid = c.anvil_uuid ORDER BY c.anvil_name ASC, b.host_name ASC; +# Fail a resource for testing purposes. +crm_resource --fail --resource srv02-b -N vm-a01n01 + +# Recover without reboot +crm_resource --resource srv01-a --refresh + uname -r; grubby --default-kernel; lsinitrd -m /boot/initramfs-4.18.0-448.el8.x86_64.img | grep lvm; systemctl is-enabled scancore.service; dnf -y update; systemctl disable --now anvil-daemon; systemctl disable --now scancore diff --git a/ocf/alteeve/server b/ocf/alteeve/server index 137a3e26..8e9b9842 100755 --- a/ocf/alteeve/server +++ b/ocf/alteeve/server @@ -256,6 +256,18 @@ if (not $anvil->data->{switches}{monitor}) show_environment($anvil, 3); } +# If there's a test fail file, return '1' to cause pacemaker to fail this resource. +if ($anvil->data->{environment}{OCF_RESKEY_name}) +{ + my $test_fail_file = "/tmp/".$anvil->data->{environment}{OCF_RESKEY_name}.".fail"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_fail_file => $test_fail_file }}); + if (-e $test_fail_file) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0150", variables => { fail_file => $test_fail_file }}); + $anvil->nice_exit({exit_code => 1}); + } +} + ### What are we being asked to do? # start  - Starts the resource. # stop  - Shuts down the resource. diff --git a/scancore-agents/scan-cluster/scan-cluster b/scancore-agents/scan-cluster/scan-cluster index 9e13d4d6..485dd070 100755 --- a/scancore-agents/scan-cluster/scan-cluster +++ b/scancore-agents/scan-cluster/scan-cluster @@ -99,6 +99,9 @@ check_config($anvil); # Check the fence delay check_fence_delay($anvil); +# Check for failed resources +check_resources($anvil); + # Shut down. $anvil->ScanCore->agent_shutdown({agent => $THIS_FILE}); @@ -107,6 +110,191 @@ $anvil->ScanCore->agent_shutdown({agent => $THIS_FILE}); # Functions # ############################################################################################################# +# This looks for failed resource and, if found, tries to recover them. +sub check_resources +{ + my ($anvil) = @_; + + foreach my $server (sort {$a cmp $b} keys %{$anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}}) + { + # This is used for alerts, if needed below. + my $variables = { server => $server }; + + my $failed = check_if_server_failed($anvil, $server); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + server => $server, + failed => $failed, + }}); + if ($failed) + { + # Who am I and who is my peer? See if the server is running on either host. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0003", variables => { server => $server }}); + my $attempt_recovery = 0; + my $server_found = 0; + my $both_nodes_ready = 1; + foreach my $target ("local", "peer") + { + my $node_ready = $anvil->data->{cib}{parsed}{$target}{ready}; + my $node_name = $anvil->data->{cib}{parsed}{$target}{name}; + my $host_uuid = $anvil->Get->host_uuid_from_name({host_name => $node_name}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0004", variables => { + node_name => $node_name, + host_uuid => $host_uuid, + node_ready => $node_ready, + }}); + if (not $node_ready) + { + $both_nodes_ready = 1; + } + + if ($host_uuid eq $anvil->Get->host_uuid) + { + # Search for the server here + $anvil->Server->find({debug => 2}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0005"}); + } + else + { + # Search for the server on the peer. + my $target_ip = $anvil->Network->find_target_ip({host_uuid => $host_uuid}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0006", variables => { target_ip => $target_ip }}); + $anvil->Server->find({ + debug => 2, + target => $target_ip, + }); + } + my $server_host = defined $anvil->data->{server}{location}{$server}{host_name} ? $anvil->data->{server}{location}{$server}{host_name} : ""; + my $server_status = defined $anvil->data->{server}{location}{$server}{status} ? $anvil->data->{server}{location}{$server}{status} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { + server_host => $server_host, + server_status => $server_status, + }}); + if ($server_host) + { + $server_found = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_found => $server_found }}); + if (($node_ready) && ($host_uuid eq $anvil->Get->host_uuid)) + { + # Go ahead with recovery + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0007"}); + $attempt_recovery = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { attempt_recovery => $attempt_recovery }}); + } + } + } + + if ((not $server_found) && ($both_nodes_ready)) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0008"}); + $attempt_recovery = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { attempt_recovery => $attempt_recovery }}); + } + elsif (($server_found) && (not $attempt_recovery)) + { + # The server was found to be running, but not here (or this node is not fully in the cluster). NOT attempting recovery yet. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0009"}); + } + elsif ($attempt_recovery) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0010"}); + $anvil->Cluster->recover_server({ + debug => 2, + server => $server, + }); + + # It'll leave 'failed state' for a bit, so we need to wait. + sleep 3; + my $wait_until = time + 10; + my $waiting = 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0011"}); + while($waiting) + { + my $failed = check_if_server_failed($anvil, $server); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }}); + if ($failed) + { + # No luck... + $waiting = 0; + my $changed = $anvil->Alert->check_alert_sent({ + record_locator => "scan_cluster::failed_server::".$server, + set_by => $THIS_FILE, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + waiting => $waiting, + changed => $changed, + }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_alert_0014", variables => $variables}); + if ($changed) + { + # Send an alert. + $anvil->Alert->register({debug => 2, alert_level => "notice", message => "scan_cluster_alert_0014", variables => $variables, set_by => $THIS_FILE}); + } + } + elsif (time > $wait_until) + { + # Success! + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_alert_0013", variables => $variables}); + $anvil->Alert->register({debug => 2, alert_level => "notice", message => "scan_cluster_alert_0013", variables => $variables, set_by => $THIS_FILE}); + + # Clear the alert, if it existed before + $waiting = 0; + my $changed = $anvil->Alert->check_alert_sent({ + record_locator => "scan_cluster::failed_server::".$server, + set_by => $THIS_FILE, + clear => 1, + }); + } + else + { + # Wait a sec + sleep 2; + } + } + } + } + else + { + # Make sure that this server wasn't previously failed. + my $changed = $anvil->Alert->check_alert_sent({ + record_locator => "scan_cluster::failed_server::".$server, + set_by => $THIS_FILE, + clear => 1, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { changed => $changed }}); + if ($changed) + { + # Send the All-good alert. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_alert_0015", variables => $variables}); + $anvil->Alert->register({debug => 2, alert_level => "notice", message => "scan_cluster_alert_0015", variables => $variables, set_by => $THIS_FILE}); + } + } + } + + return(0); +} + +sub check_if_server_failed +{ + my ($anvil, $server) = @_; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { server => $server }}); + + $anvil->Cluster->parse_crm_mon({debug => 3}); + my $failed = exists $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$server}{variables}{failed} ? $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$server}{variables}{failed} : 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }}); + if ($failed eq "true") + { + $failed = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }}); + } + elsif ($failed eq "false") + { + $failed = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }}); + } + + return($failed); +} + # Check to see if we need to move the fence delay. sub check_fence_delay { diff --git a/scancore-agents/scan-cluster/scan-cluster.xml b/scancore-agents/scan-cluster/scan-cluster.xml index 49c2b3e6..15063008 100644 --- a/scancore-agents/scan-cluster/scan-cluster.xml +++ b/scancore-agents/scan-cluster/scan-cluster.xml @@ -37,10 +37,22 @@ In Maintenance Mode: ..... [#!variable!maintenance_mode!#] #!variable!difference!# ==== + The server: [#!variable!server!#] was found to be failed in pacemaker, but it was successfully recovered. This does NOT mean the server rebooted, but it may have. Checking the server is advised. + The server: [#!variable!server!#] was found to be failed in pacemaker. The attempt to recover it appears to have failed. The server might well still be running ok, checking the server is advised. + The server: [#!variable!server!#] had been found to be failed in pacemaker. It's now recovered. This does NOT mean the server rebooted, but it may have. Checking the server is advised. Starting: [#!variable!program!#]. This host is a: [#!variable!host_type!#], this agent is only useful on nodes. Exiting. + [ Warning ] - The server: [#!variable!server!#] is in a FAILED state! Checking to see if it's safe to attempt recovery. + Searching node: [#!variable!node_name!# (#!variable!host_uuid!#] which is in ready state: [#!variable!node_ready!#]. + Searching for the server on the local system. + Searching for the server on the peer using IP: [#!variable!target_ip!#]. + The server is running locally and we're a full cluster member. Will attempt recover. + Both nodes are up and the server wasn't found anywhere. Attempting recovery. + The server was found to be running, but not here (or this node is not fully in the cluster). NOT attempting recovery yet. + Attempting recovery now... + Checking to see if the server has recovered yet... diff --git a/scancore-agents/scan-ipmitool/scan-ipmitool b/scancore-agents/scan-ipmitool/scan-ipmitool index 6f45d591..422d0336 100755 --- a/scancore-agents/scan-ipmitool/scan-ipmitool +++ b/scancore-agents/scan-ipmitool/scan-ipmitool @@ -2086,8 +2086,7 @@ sub find_ipmi_targets SELECT anvil_name, anvil_node1_host_uuid, - anvil_node2_host_uuid, - anvil_dr1_host_uuid + anvil_node2_host_uuid FROM anvils WHERE @@ -2107,12 +2106,10 @@ WHERE my $anvil_name = $row->[0]; my $anvil_node1_host_uuid = $row->[1]; my $anvil_node2_host_uuid = $row->[2]; - my $anvil_dr1_host_uuid = defined $row->[3] ? $row->[3] : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:anvil_name' => $anvil_name, 's2:anvil_node1_host_uuid' => $anvil_node1_host_uuid, 's3:anvil_node2_host_uuid' => $anvil_node2_host_uuid, - 's4:anvil_dr1_host_uuid' => $anvil_dr1_host_uuid, }}); my $query = " @@ -2128,14 +2125,7 @@ AND ( host_uuid = ".$anvil->Database->quote($anvil_node1_host_uuid)." OR - host_uuid = ".$anvil->Database->quote($anvil_node2_host_uuid); - if ($anvil_dr1_host_uuid) - { - $query .= " - OR - host_uuid = ".$anvil->Database->quote($anvil_dr1_host_uuid); - } - $query .= " + host_uuid = ".$anvil->Database->quote($anvil_node2_host_uuid)." ) ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); diff --git a/scancore-agents/scan-server/scan-server b/scancore-agents/scan-server/scan-server index 3e28c369..090bbd33 100755 --- a/scancore-agents/scan-server/scan-server +++ b/scancore-agents/scan-server/scan-server @@ -1081,7 +1081,7 @@ DELETED - Marks a server as no longer existing next if $server_host_uuid ne $anvil->Get->host_uuid; # If we're here, the server used to be on us, and isn't on the peer, so mark it as off. - if ($server_state ne "shut off") + if (($server_state ne "shut off") && ($server_state ne "DELETED")) { # Mark it as being off now. my $query = " diff --git a/scancore-agents/scan-storcli/scan-storcli b/scancore-agents/scan-storcli/scan-storcli index a499ff9e..de8eeea4 100755 --- a/scancore-agents/scan-storcli/scan-storcli +++ b/scancore-agents/scan-storcli/scan-storcli @@ -4381,6 +4381,7 @@ INSERT INTO foreach my $variable (sort {$a cmp $b} keys %{$anvil->data->{sql}{scan_storcli_variables}{scan_storcli_variable_uuid}{source_table}{'scan_storcli_cachevaults'}{source_uuid}{$cachevault_uuid}{$type}}) { # This variable has vanished + next if not defined $anvil->data->{sql}{scan_storcli_variables}{scan_storcli_variable_uuid}{source_table}{'scan_storcli_cachevaults'}{source_uuid}{$cachevault_uuid}{$type}{$variable}{scan_storcli_variable_value}; my $old_variable_value = $anvil->data->{sql}{scan_storcli_variables}{scan_storcli_variable_uuid}{source_table}{'scan_storcli_cachevaults'}{source_uuid}{$cachevault_uuid}{$type}{$variable}{scan_storcli_variable_value}; my $variable_uuid = $anvil->data->{sql}{scan_storcli_variables}{scan_storcli_variable_uuid}{source_table}{'scan_storcli_cachevaults'}{source_uuid}{$cachevault_uuid}{$type}{$variable}{scan_storcli_variable_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { @@ -9070,8 +9071,8 @@ sub get_cachevault_data ($variable, $value) = ($line =~ /^(.{$split_point})(.*)$/); $variable =~ s/\s+$//; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - variable => $variable, - value => $value, + 's1:variable' => $variable, + 's2:value' => $value, }}); } diff --git a/share/anvil.sql b/share/anvil.sql index 70074137..055c431b 100644 --- a/share/anvil.sql +++ b/share/anvil.sql @@ -1979,52 +1979,52 @@ CREATE TRIGGER trigger_temperature AFTER INSERT OR UPDATE ON temperature FOR EACH ROW EXECUTE PROCEDURE history_temperature(); --- -- Enable after Yan's review + -- This is used to audit major events by user. ---CREATE TABLE audits ( - --audit_uuid uuid primary key, - --audit_user_uuid uuid not null, -- This is the users -> user_uuid the audit is tracking - --audit_details text not null, -- This is the information explaining the action being audited. - --modified_date timestamp with time zone not null, +CREATE TABLE audits ( + audit_uuid uuid primary key, + audit_user_uuid uuid not null, -- This is the users -> user_uuid the audit is tracking + audit_details text not null, -- This is the information explaining the action being audited. + modified_date timestamp with time zone not null, - --FOREIGN KEY(audit_user_uuid) REFERENCES users(user_uuid) ---); ---ALTER TABLE audits OWNER TO admin; - ---CREATE TABLE history.audits ( - --history_id bigserial, - --audit_uuid uuid, - --audit_user_uuid uuid, - --audit_details text, - --modified_date timestamp with time zone not null ---); ---ALTER TABLE history.audits OWNER TO admin; - ---CREATE FUNCTION history_audits() RETURNS trigger ---AS $$ ---DECLARE - --history_audits RECORD; ---BEGIN - --SELECT INTO history_audits * FROM audits WHERE audit_uuid = new.audit_uuid; - --INSERT INTO history.audits - --(audit_uuid, - --audit_user_uuid, - --audit_details, - --modified_date) - --VALUES - --(history_audit.audit_uuid, - --history_audit.audit_user_uuid, - --history_audit.audit_details, - --history_audit.modified_date); - --RETURN NULL; ---END; ---$$ ---LANGUAGE plpgsql; ---ALTER FUNCTION history_audits() OWNER TO admin; - ---CREATE TRIGGER trigger_audits - --AFTER INSERT OR UPDATE ON audits - --FOR EACH ROW EXECUTE PROCEDURE history_audits(); + FOREIGN KEY(audit_user_uuid) REFERENCES users(user_uuid) +); +ALTER TABLE audits OWNER TO admin; + +CREATE TABLE history.audits ( + history_id bigserial, + audit_uuid uuid, + audit_user_uuid uuid, + audit_details text, + modified_date timestamp with time zone not null +); +ALTER TABLE history.audits OWNER TO admin; + +CREATE FUNCTION history_audits() RETURNS trigger +AS $$ +DECLARE + history_audits RECORD; +BEGIN + SELECT INTO history_audits * FROM audits WHERE audit_uuid = new.audit_uuid; + INSERT INTO history.audits + (audit_uuid, + audit_user_uuid, + audit_details, + modified_date) + VALUES + (history_audit.audit_uuid, + history_audit.audit_user_uuid, + history_audit.audit_details, + history_audit.modified_date); + RETURN NULL; +END; +$$ +LANGUAGE plpgsql; +ALTER FUNCTION history_audits() OWNER TO admin; + +CREATE TRIGGER trigger_audits + AFTER INSERT OR UPDATE ON audits + FOR EACH ROW EXECUTE PROCEDURE history_audits(); -- ------------------------------------------------------------------------------------------------------- -- diff --git a/share/words.xml b/share/words.xml index 86088c2a..3b9d0fe5 100644 --- a/share/words.xml +++ b/share/words.xml @@ -3582,6 +3582,8 @@ The error was: [ Warning ] - The interface: [#!variable!interface!#] appears to be down (state: [#!variable!state!#]). The system uptime is: [#!variable!uptime!#], so it might be a problem where the interface didn't start on boot as it should have. So we're going to bring the interface up. [ Warning ] - The IPMI stonith resource: [#!variable!resource!#] is in the role: [#!variable!role!#] (should be 'Started'). Will check the IPMI config now. + [ Warning ] - Failed to find a valid IP address or password to be used to setup the DR host's IPMI. + [ Warning ] - The test "fail file": [#!variable!fail_file!#] was found. So long as this file exists, the ocf:alteeve:server RA will return 'OCF_ERR_GENERIC' (exit code 1). Delete the file to resume normal operation. diff --git a/tools/anvil-delete-server b/tools/anvil-delete-server index 0d031eeb..3dac3791 100755 --- a/tools/anvil-delete-server +++ b/tools/anvil-delete-server @@ -75,8 +75,10 @@ if ($anvil->data->{switches}{'job-uuid'}) message => "message_0217", }); - # Are we in an Anvil! system? - if (not $anvil->data->{sys}{anvil_uuid}) + # Are we in an Anvil! system or are we a DR host? + my $host_type = $anvil->Get->host_type(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }}); + if ((not $anvil->data->{sys}{anvil_uuid}) && ($host_type ne "dr")) { # We're not in an Anvil!. if ($anvil->data->{switches}{'job-uuid'}) @@ -135,6 +137,32 @@ sub run_jobs server_name => $server_name, }}); + # Before we start, we need to know if this server is on DR hosts. To do this, we'll parse the DRBD + # config file and look for DR hosts. + $anvil->Database->get_hosts({debug => 2}); + $anvil->DRBD->gather_data({debug => 2}); + foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{peer}}) + { + my $peer_host_uuid = $anvil->Database->get_host_uuid_from_string({debug => 2, string => $peer}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + peer => $peer, + peer_host_uuid => $peer_host_uuid, + }}); + if (($peer_host_uuid) && (exists $anvil->data->{hosts}{host_uuid}{$peer_host_uuid})) + { + my $host_type = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{host_type}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }}); + + if ($host_type eq "dr") + { + $anvil->data->{dr_hosts}{$peer_host_uuid} = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "dr_hosts::${peer_host_uuid}" => $anvil->data->{dr_hosts}{$peer_host_uuid}, + }}); + } + } + } + if (not $anvil->data->{job}{peer_mode}) { # Remove the server from pacemaker (stopping it, if necessary). @@ -166,7 +194,6 @@ sub run_jobs job_status => "failed", }); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0228", variables => { resource => $server_name }}); - $anvil->nice_exit({exit_code => 1}); } $anvil->Job->update_progress({ @@ -360,11 +387,15 @@ sub remove_from_pacemaker # We're node 2 push @{$peers}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid} } - if ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}) + + if (exists $anvil->data->{dr_hosts}) { - # There's a DR host. - push @{$peers}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; + foreach my $peer_host_uuid (keys %{$anvil->data->{dr_hosts}}) + { + push @{$peers}, $peer_host_uuid; + } } + my $progress = 30; foreach my $host_uuid (@{$peers}) { diff --git a/tools/anvil-join-anvil b/tools/anvil-join-anvil index 6462bec1..6f968345 100755 --- a/tools/anvil-join-anvil +++ b/tools/anvil-join-anvil @@ -2339,11 +2339,9 @@ sub load_job # Load the anvil $anvil->data->{sys}{node1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; $anvil->data->{sys}{node2_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - $anvil->data->{sys}{dr1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "sys::node1_host_uuid" => $anvil->data->{sys}{node1_host_uuid}, "sys::node2_host_uuid" => $anvil->data->{sys}{node2_host_uuid}, - "sys::dr1_host_uuid" => $anvil->data->{sys}{dr1_host_uuid}, }}); update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0075,!!machine!".$anvil->data->{sys}{machine}."!!,!!manifest_uuid!".$anvil->data->{sys}{manifest_uuid}."!!"); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0075", variables => { diff --git a/tools/anvil-manage-files b/tools/anvil-manage-files index 1145ebed..f98fd8ea 100755 --- a/tools/anvil-manage-files +++ b/tools/anvil-manage-files @@ -242,7 +242,7 @@ ORDER BY # Do we have a file_location_uuid? If not, there will be soon but nothing to do until # then. - my $file_location_uuid = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; + my $file_location_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }}); next if not $file_location_uuid; diff --git a/tools/anvil-manage-server b/tools/anvil-manage-server index 5a3b4863..7dfe2c02 100755 --- a/tools/anvil-manage-server +++ b/tools/anvil-manage-server @@ -112,8 +112,6 @@ sub process_interactive $anvil->data->{target_server}{anvil_node1_host_name} = "" if not exists $anvil->data->{target_server}{anvil_node1_host_name}; $anvil->data->{target_server}{anvil_node2_host_uuid} = "" if not exists $anvil->data->{target_server}{anvil_node2_host_uuid}; $anvil->data->{target_server}{anvil_node2_host_name} = "" if not exists $anvil->data->{target_server}{anvil_node2_host_name}; - $anvil->data->{target_server}{anvil_dr1_host_uuid} = "" if not exists $anvil->data->{target_server}{anvil_dr1_host_uuid}; - $anvil->data->{target_server}{anvil_dr1_host_name} = "" if not exists $anvil->data->{target_server}{anvil_dr1_host_name}; $anvil->data->{new_config}{cpu}{sockets} = ""; @@ -216,8 +214,6 @@ sub interactive_question $anvil->data->{target_server}{anvil_node1_host_name} = "" if not defined $anvil->data->{target_server}{anvil_node1_host_name}; $anvil->data->{target_server}{anvil_node2_host_uuid} = "" if not defined $anvil->data->{target_server}{anvil_node2_host_uuid}; $anvil->data->{target_server}{anvil_node2_host_name} = "" if not defined $anvil->data->{target_server}{anvil_node2_host_name}; - $anvil->data->{target_server}{anvil_dr1_host_uuid} = "" if not defined $anvil->data->{target_server}{anvil_dr1_host_uuid}; - $anvil->data->{target_server}{anvil_dr1_host_name} = "" if not defined $anvil->data->{target_server}{anvil_dr1_host_name}; ### Anvil # If 'switches::anvil' is set, see if it's a UUID and then set either 'anvil-uuid' or 'anvil-name'. @@ -405,14 +401,12 @@ sub interactive_question my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid}; my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; - my $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid, server_name => $server_name, server_host_uuid => $server_host_uuid, node1_host_uuid => $node1_host_uuid, node2_host_uuid => $node2_host_uuid, - dr1_host_uuid => $dr1_host_uuid, }}); $anvil->data->{target_server}{anvil_uuid} = $anvil_uuid; @@ -424,8 +418,6 @@ sub interactive_question $anvil->data->{target_server}{anvil_node1_host_name} = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name}; $anvil->data->{target_server}{anvil_node2_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; $anvil->data->{target_server}{anvil_node2_host_name} = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name}; - $anvil->data->{target_server}{anvil_dr1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; - $anvil->data->{target_server}{anvil_dr1_host_name} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "target_server::anvil_uuid" => $anvil->data->{target_server}{anvil_uuid}, "target_server::server_name" => $anvil->data->{target_server}{server_name}, @@ -435,18 +427,7 @@ sub interactive_question "target_server::anvil_node1_host_name" => $anvil->data->{target_server}{anvil_node1_host_name}, "target_server::anvil_node2_host_uuid" => $anvil->data->{target_server}{anvil_node2_host_uuid}, "target_server::anvil_node2_host_name" => $anvil->data->{target_server}{anvil_node2_host_name}, - "target_server::anvil_dr1_host_uuid" => $anvil->data->{target_server}{anvil_dr1_host_uuid}, - "target_server::anvil_dr1_host_name" => $anvil->data->{target_server}{anvil_dr1_host_name}, }}); - if ($dr1_host_uuid) - { - $anvil->data->{target_server}{anvil_dr1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; - $anvil->data->{target_server}{anvil_dr1_host_name} = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name};; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "target_server::anvil_dr1_host_uuid" => $anvil->data->{target_server}{anvil_dr1_host_uuid}, - "target_server::anvil_dr1_host_name" => $anvil->data->{target_server}{anvil_dr1_host_name}, - }}); - } ### Pull out details of the server. # How much RAM are we using and how much is configured? diff --git a/tools/anvil-provision-server b/tools/anvil-provision-server index 8a8435cb..fce6076f 100755 --- a/tools/anvil-provision-server +++ b/tools/anvil-provision-server @@ -394,6 +394,12 @@ sub write_definition $anvil->Database->get_servers(); $anvil->Database->get_server_definitions(); + if (not exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid}) + { + sleep 1; + next; + } + my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid}; my $server_definition = exists $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid} ? $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition => $server_definition }}); @@ -2231,7 +2237,6 @@ sub interactive_ask_server_ram } my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - my $dr1_host_uuid = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : ""; print $anvil->Words->string({key => "job_0167", variables => { ram_available => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}, unit => "M"}).")", @@ -2534,8 +2539,8 @@ sub interactive_ask_server_install_media my $retry = 0; while(1) { - $anvil->Database->get_files(); - $anvil->Database->get_file_locations(); + $anvil->Database->get_files({debug => 2}); + $anvil->Database->get_file_locations({debug => 2}); my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); @@ -2550,7 +2555,7 @@ sub interactive_ask_server_install_media my $file_size = $anvil->data->{files}{file_name}{$file_name}{file_size}; my $file_type = $anvil->data->{files}{file_name}{$file_name}{file_type}; my $file_path = $file_directory."/".$file_name; - my $file_location_uuid = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; + my $file_location_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid, @@ -2650,7 +2655,7 @@ sub interactive_ask_server_driver_disc my $file_size = $anvil->data->{files}{file_name}{$file_name}{file_size}; my $file_type = $anvil->data->{files}{file_name}{$file_name}{file_type}; my $file_path = $file_directory."/".$file_name; - my $file_location_uuid = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; + my $file_location_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid, @@ -2690,7 +2695,6 @@ sub interactive_ask_server_driver_disc } my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - my $dr1_host_uuid = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : ""; print $anvil->Words->string({key => "job_0182"})."\n"; print $iso_list."\n"; diff --git a/tools/anvil-sync-shared b/tools/anvil-sync-shared index 4fb02354..be8d839c 100755 --- a/tools/anvil-sync-shared +++ b/tools/anvil-sync-shared @@ -733,31 +733,19 @@ sub process_pull_file my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - my $anvil_dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_node1_host_uuid => $anvil_node1_host_uuid, anvil_node2_host_uuid => $anvil_node2_host_uuid, - anvil_dr1_host_uuid => $anvil_dr1_host_uuid, }}); # Where we pull from will depend on which machine we are and how many strikers we have. If we have - # one Anvil!, node 1 and 2 download at the same time, and DR waits. If there are two strikers, Each - # node will download from a different striker (if possible) and DR waits. If there are 3 or more - # Strikers, DR does not wait, and downloads from a different striker than the striker's nodes use. + # one Anvil!, sub-node 1 and 2 download at the same time. If there are two strikers, Each node will + # download from a different striker (if possible). my $i_am = "node1"; if ($anvil->Get->host_uuid eq $anvil_node2_host_uuid) { $i_am = "node2"; } - elsif ($anvil->Get->host_uuid eq $anvil_dr1_host_uuid) - { - $i_am = "dr1"; - - # As we're DR, we'll likely be pinging the nodes to seee if they're up when we wait for them - # to finish jobs. As such, load their IPs into memory. - $anvil->Network->load_ips({debug => 2, host_uuid => $anvil_node1_host_uuid}); - $anvil->Network->load_ips({debug => 2, host_uuid => $anvil_node2_host_uuid}); - } my $striker_count = @{$anvil->data->{target_strikers}}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { @@ -777,158 +765,26 @@ sub process_pull_file $anvil->nice_exit({exit_code => 2}); } - my $use = 0; - my $dr_wait = 1; + my $use = 0; if ($striker_count >= 3) { - $dr_wait = 0; if ($i_am eq "node1") { $use = 0; } elsif ($i_am eq "node2") { $use = 1; } - elsif ($i_am eq "dr1") { $use = 2; } - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - 'use' => $use, - dr_wait => $dr_wait, - }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'use' => $use }}); } elsif ($striker_count == 2) { - # Two strikers, each node will use the other Striker, DR waits for both to be done. + # Two strikers, each node will use the other Striker if ($i_am eq "node1") { $use = 0; } elsif ($i_am eq "node2") { $use = 1; } - elsif ($i_am eq "dr1") { $use = 1; } - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - 'use' => $use, - dr_wait => $dr_wait, - }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'use' => $use }}); } elsif ($striker_count == 1) { # Only 1 Striker if ($i_am eq "node1") { $use = 0; } elsif ($i_am eq "node2") { $use = 0; } - elsif ($i_am eq "dr1") { $use = 0; } - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - 'use' => $use, - dr_wait => $dr_wait, - }}); - } - - # If I'm DR and need to wait, look for jobs on node1 and node2 and wait until both are done (or is - # offline). - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - i_am => $i_am, - dr_wait => $dr_wait, - }}); - if (($i_am eq "dr1") && ($dr_wait)) - { - my $node1_job_uuid = ""; - my $node1_online = 1; - my $node2_job_uuid = ""; - my $node2_online = 1; - my $node1_waiting = 1; - my $node2_waiting = 1; - - my $query = " -SELECT - job_uuid, - job_host_uuid -FROM - jobs -WHERE - job_name = 'storage::pull_file' -AND - ( - job_host_uuid = ".$anvil->Database->quote($anvil_node1_host_uuid)." - OR - job_host_uuid = ".$anvil->Database->quote($anvil_node2_host_uuid)." - ) -;"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); - - my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); - my $count = @{$results}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - results => $results, - count => $count, - }}); - foreach my $row (@{$results}) - { - my $job_uuid = $row->[0]; - my $job_host_uuid = $row->[1]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - job_uuid => $job_uuid, - job_host_uuid => $job_host_uuid, - }}); - - if ($job_host_uuid eq $anvil_node1_host_uuid) - { - $node1_job_uuid = $job_uuid; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_job_uuid => $node1_job_uuid }}); - } - if ($job_host_uuid eq $anvil_node2_host_uuid) - { - $node2_job_uuid = $job_uuid; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_job_uuid => $node2_job_uuid }}); - } - } - - my $waiting = 1; - while($waiting) - { - if ($node1_waiting) - { - if (not $node1_job_uuid) - { - $node1_waiting = 0; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_waiting => $node1_waiting }}); - } - else - { - $node1_waiting = wait_on_host($anvil, $anvil_node1_host_uuid, $node1_job_uuid); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_waiting => $node1_waiting }}); - } - } - if ($node2_waiting) - { - if (not $node2_job_uuid) - { - $node2_waiting = 0; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_waiting => $node2_waiting }}); - } - else - { - $node2_waiting = wait_on_host($anvil, $anvil_node2_host_uuid, $node2_job_uuid); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_waiting => $node2_waiting }}); - } - } - - if ((not $node1_waiting) && (not $node2_waiting)) - { - # We can proceed. - $waiting = 0; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); - } - else - { - # Sleep for a bit. - my $sleep_time = 30; - my $wait_until = $anvil->Get->date_and_time({offset => $sleep_time}); - - $anvil->Job->update_progress({ - progress => 1, - message => "message_0199,!!strikers!".$striker_count."!!,!!node1_waiting!".$node1_waiting."!!,!!node2_waiting!".$node2_waiting."!!,!!wait_until!".$wait_until."!!", - job_status => "failed", - }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0199", variables => { - strikers => $striker_count, - node1_waiting => $node1_waiting, - node2_waiting => $node2_waiting, - wait_until => $wait_until, - }}); - - sleep $sleep_time; - } - } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'use' => $use }}); } # Now proceed with the download! diff --git a/tools/anvil-version-changes b/tools/anvil-version-changes index ea0d4e4d..b303a2ac 100755 --- a/tools/anvil-version-changes +++ b/tools/anvil-version-changes @@ -75,7 +75,7 @@ sub striker_checks ### NOTE: Disabled until review complete # This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release) - #update_audits($anvil); + update_audits($anvil); ### NOTE: Disabled until review complete # This checks to make sure that the new dr_links table exists, and that existing anvil_dr1_host_uuid diff --git a/tools/striker-open-ssh-tunnel b/tools/striker-open-ssh-tunnel index bd0323f8..3d1b6bec 100755 --- a/tools/striker-open-ssh-tunnel +++ b/tools/striker-open-ssh-tunnel @@ -49,8 +49,7 @@ SELECT anv.anvil_password FROM hosts AS hos JOIN anvils AS anv ON hos.host_uuid = anv.anvil_node1_host_uuid - OR hos.host_uuid = anv.anvil_node2_host_uuid - OR hos.host_uuid = anv.anvil_dr1_host_uuid +OR hos.host_uuid = anv.anvil_node2_host_uuid WHERE hos.host_name = ".$anvil->Database->quote($target)." ;"; diff --git a/tools/striker-purge-target b/tools/striker-purge-target index e0bc6cc7..82b14c74 100755 --- a/tools/striker-purge-target +++ b/tools/striker-purge-target @@ -103,10 +103,6 @@ if ($anvil->data->{switches}{'anvil'}) $anvil_uuid = $anvil->data->{purge}{anvil_uuid}; push @{$anvil->data->{purge}{hosts}}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; push @{$anvil->data->{purge}{hosts}}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - if ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}) - { - push @{$anvil->data->{purge}{hosts}}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; - } } elsif($anvil->data->{switches}{'host'}) { @@ -198,14 +194,6 @@ else }}); # DR, if set. - if ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}) - { - my $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0244", variables => { - host_name => $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name}, - host_uuid => $dr1_host_uuid, - }}); - } } else { @@ -261,10 +249,6 @@ foreach my $host_uuid (@{$anvil->data->{purge}{hosts}}) { $host_key = "anvil_node2_host_uuid"; } - elsif ($host_uuid eq $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}) - { - $host_key = "anvil_dr1_host_uuid"; - } if ($host_key) { my $query = "