Merge pull request #319 from ClusterLabs/anvil-tools-dev

Anvil tools dev
main
Digimer 2 years ago committed by GitHub
commit 6bce292969
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 74
      Anvil/Tools/Cluster.pm
  2. 24
      Anvil/Tools/DRBD.pm
  3. 58
      Anvil/Tools/Database.pm
  4. 6
      Anvil/Tools/ScanCore.pm
  5. 41
      Anvil/Tools/Storage.pm
  6. 366
      Anvil/Tools/System.pm
  7. 2
      Anvil/Tools/Words.pm
  8. 20
      notes
  9. 12
      ocf/alteeve/server
  10. 188
      scancore-agents/scan-cluster/scan-cluster
  11. 12
      scancore-agents/scan-cluster/scan-cluster.xml
  12. 14
      scancore-agents/scan-ipmitool/scan-ipmitool
  13. 2
      scancore-agents/scan-server/scan-server
  14. 5
      scancore-agents/scan-storcli/scan-storcli
  15. 88
      share/anvil.sql
  16. 2
      share/words.xml
  17. 43
      tools/anvil-delete-server
  18. 2
      tools/anvil-join-anvil
  19. 2
      tools/anvil-manage-files
  20. 19
      tools/anvil-manage-server
  21. 16
      tools/anvil-provision-server
  22. 158
      tools/anvil-sync-shared
  23. 2
      tools/anvil-version-changes
  24. 3
      tools/striker-open-ssh-tunnel
  25. 16
      tools/striker-purge-target

@ -35,6 +35,7 @@ my $THIS_FILE = "Cluster.pm";
# parse_cib # parse_cib
# parse_crm_mon # parse_crm_mon
# parse_quorum # parse_quorum
# recover_server
# shutdown_server # shutdown_server
# start_cluster # start_cluster
# which_node # which_node
@ -863,9 +864,10 @@ sub check_node_status
=head2 check_server_constraints =head2 check_server_constraints
This method checks to see if the peer node is offline, and the local node is only. If this is the case, the location constraints for servers are checked to ensure that they favour the current host. If not, the location constraint is updated. This checks to see if the constraints on a server are sane. Specifically;
This is meant to be used to prevent servers from automatically migrating back to a node after it was fenced. * If the server is on a sub-node and the peer is offline, ensure that the location constraints prefer the current host. This prevents migrations back to the old host.
* Check to see if a DRBD resource constriant was applied against a node, and the node's DRBD resource is UpToDate. If so, remove the constraint.
This method takes no parameters. This method takes no parameters.
@ -2204,17 +2206,14 @@ sub get_anvil_uuid
my $anvil_name = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name}; my $anvil_name = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name};
my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
my $anvil_dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
anvil_name => $anvil_name, anvil_name => $anvil_name,
anvil_node1_host_uuid => $anvil_node1_host_uuid, anvil_node1_host_uuid => $anvil_node1_host_uuid,
anvil_node2_host_uuid => $anvil_node2_host_uuid, anvil_node2_host_uuid => $anvil_node2_host_uuid,
anvil_dr1_host_uuid => $anvil_dr1_host_uuid,
}}); }});
if (($host_uuid eq $anvil_node1_host_uuid) or if (($host_uuid eq $anvil_node1_host_uuid) or
($host_uuid eq $anvil_node2_host_uuid) or ($host_uuid eq $anvil_node2_host_uuid))
($host_uuid eq $anvil_dr1_host_uuid))
{ {
# Found ot! # Found ot!
$member_anvil_uuid = $anvil_uuid; $member_anvil_uuid = $anvil_uuid;
@ -2237,13 +2236,11 @@ The data is stored as;
sys::anvil::node1::host_name sys::anvil::node1::host_name
sys::anvil::node2::host_uuid sys::anvil::node2::host_uuid
sys::anvil::node2::host_name sys::anvil::node2::host_name
sys::anvil::dr1::host_uuid
sys::anvil::dr1::host_name
To assist with lookup, the following are also set; To assist with lookup, the following are also set;
sys::anvil::i_am = {node1,node2,dr1} sys::anvil::i_am = {node1,node2}
sys::anvil::peer_is = {node1,node2} # Not set if this host is 'dr1' sys::anvil::peer_is = {node1,node2}
This method takes no parameters. This method takes no parameters.
@ -2260,8 +2257,6 @@ sub get_peers
$anvil->data->{sys}{anvil}{node1}{host_name} = ""; $anvil->data->{sys}{anvil}{node1}{host_name} = "";
$anvil->data->{sys}{anvil}{node2}{host_uuid} = ""; $anvil->data->{sys}{anvil}{node2}{host_uuid} = "";
$anvil->data->{sys}{anvil}{node2}{host_name} = ""; $anvil->data->{sys}{anvil}{node2}{host_name} = "";
$anvil->data->{sys}{anvil}{dr1}{host_uuid} = "";
$anvil->data->{sys}{anvil}{dr1}{host_name} = "";
$anvil->data->{sys}{anvil}{i_am} = ""; $anvil->data->{sys}{anvil}{i_am} = "";
$anvil->data->{sys}{anvil}{peer_is} = ""; $anvil->data->{sys}{anvil}{peer_is} = "";
@ -2279,11 +2274,9 @@ sub get_peers
{ {
my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
my $anvil_dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
anvil_node1_host_uuid => $anvil_node1_host_uuid, anvil_node1_host_uuid => $anvil_node1_host_uuid,
anvil_node2_host_uuid => $anvil_node2_host_uuid, anvil_node2_host_uuid => $anvil_node2_host_uuid,
anvil_dr1_host_uuid => $anvil_dr1_host_uuid,
}}); }});
if ($host_uuid eq $anvil_node1_host_uuid) if ($host_uuid eq $anvil_node1_host_uuid)
@ -2310,27 +2303,17 @@ sub get_peers
"sys::anvil::peer_is" => $anvil->data->{sys}{anvil}{peer_is}, "sys::anvil::peer_is" => $anvil->data->{sys}{anvil}{peer_is},
}}); }});
} }
elsif ($host_uuid eq $anvil_dr1_host_uuid)
{
# Found our Anvil!, and we're node 1.
$found = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { found => $found }});
}
if ($found) if ($found)
{ {
$anvil->data->{sys}{anvil}{node1}{host_uuid} = $anvil_node1_host_uuid; $anvil->data->{sys}{anvil}{node1}{host_uuid} = $anvil_node1_host_uuid;
$anvil->data->{sys}{anvil}{node1}{host_name} = $anvil->data->{hosts}{host_uuid}{$anvil_node1_host_uuid}{host_name}; $anvil->data->{sys}{anvil}{node1}{host_name} = $anvil->data->{hosts}{host_uuid}{$anvil_node1_host_uuid}{host_name};
$anvil->data->{sys}{anvil}{node2}{host_uuid} = $anvil_node2_host_uuid; $anvil->data->{sys}{anvil}{node2}{host_uuid} = $anvil_node2_host_uuid;
$anvil->data->{sys}{anvil}{node2}{host_name} = $anvil->data->{hosts}{host_uuid}{$anvil_node2_host_uuid}{host_name}; $anvil->data->{sys}{anvil}{node2}{host_name} = $anvil->data->{hosts}{host_uuid}{$anvil_node2_host_uuid}{host_name};
$anvil->data->{sys}{anvil}{dr1}{host_uuid} = $anvil_dr1_host_uuid ? $anvil_dr1_host_uuid : "";
$anvil->data->{sys}{anvil}{dr1}{host_name} = $anvil_dr1_host_uuid ? $anvil->data->{hosts}{host_uuid}{$anvil_dr1_host_uuid}{host_name} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::anvil::node1::host_uuid" => $anvil->data->{sys}{anvil}{node1}{host_uuid}, "sys::anvil::node1::host_uuid" => $anvil->data->{sys}{anvil}{node1}{host_uuid},
"sys::anvil::node1::host_name" => $anvil->data->{sys}{anvil}{node1}{host_name}, "sys::anvil::node1::host_name" => $anvil->data->{sys}{anvil}{node1}{host_name},
"sys::anvil::node2::host_uuid" => $anvil->data->{sys}{anvil}{node2}{host_uuid}, "sys::anvil::node2::host_uuid" => $anvil->data->{sys}{anvil}{node2}{host_uuid},
"sys::anvil::node2::host_name" => $anvil->data->{sys}{anvil}{node2}{host_name}, "sys::anvil::node2::host_name" => $anvil->data->{sys}{anvil}{node2}{host_name},
"sys::anvil::dr1::host_uuid" => $anvil->data->{sys}{anvil}{dr1}{host_uuid},
"sys::anvil::dr1::host_name" => $anvil->data->{sys}{anvil}{dr1}{host_name},
}}); }});
# If this is a node, return the peer's short host name. # If this is a node, return the peer's short host name.
@ -4286,6 +4269,49 @@ sub parse_quorum
} }
=head2 recover_server
This tries to recover a C<< FAILED >> resource (server).
Parameters;
=head3 server_ (required)
This is the server (resource) name to try to recover.
=cut
sub recover_server
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Cluster->recover_server()" }});
my $server = defined $parameter->{server} ? $parameter->{server} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
server => $server,
}});
if (not $server)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Cluster->recover_server()", parameter => "server" }});
return("!!error!!");
}
my $shell_call = $anvil->data->{path}{exe}{crm_resource}." --resource ".$server." --refresh";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
output => $output,
return_code => $return_code,
}});
return(0);
}
=head2 shutdown_server =head2 shutdown_server
This shuts down a server that is running on the Anvil! system. If there is a problem, C<< !!error!! >> is returned. On success, C<< 0 >> is returned. This shuts down a server that is running on the Anvil! system. If there is a problem, C<< !!error!! >> is returned. On success, C<< 0 >> is returned.

@ -663,6 +663,19 @@ sub delete_resource
} }
} }
# If we're DR, delete the definition file also.
my $definition_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$resource.".xml";
my $host_type = $anvil->Get->host_type({debug => $debug});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
definition_file => $definition_file,
host_type => $host_type,
}});
if (($host_type eq "dr") && (-f $definition_file))
{
unlink $definition_file;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0134", variables => { file_path => $definition_file }});
}
return(0); return(0);
} }
@ -1764,11 +1777,9 @@ sub get_next_resource
# certain. # certain.
my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
my $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
node1_host_uuid => $node1_host_uuid, node1_host_uuid => $node1_host_uuid,
node2_host_uuid => $node2_host_uuid, node2_host_uuid => $node2_host_uuid,
dr1_host_uuid => $dr1_host_uuid,
}}); }});
my $query = " my $query = "
@ -1805,14 +1816,7 @@ AND
( (
scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node1_host_uuid)." scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node1_host_uuid)."
OR OR
scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node2_host_uuid)." "; scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node2_host_uuid)."
if ($dr1_host_uuid)
{
$query .= "
OR
scan_drbd_resource_host_uuid = ".$anvil->Database->quote($dr1_host_uuid)." ";
}
$query .= "
) )
ORDER BY ORDER BY
b.scan_drbd_resource_name ASC, b.scan_drbd_resource_name ASC,

@ -2661,7 +2661,6 @@ Data is stored in two hashes, one sorted by C<< anvil_uuid >> and one by C<< anv
anvils::anvil_uuid::<anvil_uuid>::anvil_password anvils::anvil_uuid::<anvil_uuid>::anvil_password
anvils::anvil_uuid::<anvil_uuid>::anvil_node1_host_uuid anvils::anvil_uuid::<anvil_uuid>::anvil_node1_host_uuid
anvils::anvil_uuid::<anvil_uuid>::anvil_node2_host_uuid anvils::anvil_uuid::<anvil_uuid>::anvil_node2_host_uuid
anvils::anvil_uuid::<anvil_uuid>::anvil_dr1_host_uuid
anvils::anvil_uuid::<anvil_uuid>::modified_date anvils::anvil_uuid::<anvil_uuid>::modified_date
anvils::anvil_name::<anvil_name>::anvil_uuid anvils::anvil_name::<anvil_name>::anvil_uuid
@ -2669,7 +2668,6 @@ Data is stored in two hashes, one sorted by C<< anvil_uuid >> and one by C<< anv
anvils::anvil_name::<anvil_name>::anvil_password anvils::anvil_name::<anvil_name>::anvil_password
anvils::anvil_name::<anvil_name>::anvil_node1_host_uuid anvils::anvil_name::<anvil_name>::anvil_node1_host_uuid
anvils::anvil_name::<anvil_name>::anvil_node2_host_uuid anvils::anvil_name::<anvil_name>::anvil_node2_host_uuid
anvils::anvil_name::<anvil_name>::anvil_dr1_host_uuid
anvils::anvil_name::<anvil_name>::modified_date anvils::anvil_name::<anvil_name>::modified_date
When a host UUID is stored for either node or the DR host, it will be stored at: When a host UUID is stored for either node or the DR host, it will be stored at:
@ -3592,21 +3590,23 @@ FROM
if ($file_name) if ($file_name)
{ {
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid} = $file_location_file_uuid; $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid} = $file_location_file_uuid;
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_name} = $file_name; $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_name} = $file_name;
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_directory} = $file_directory; $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_directory} = $file_directory;
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size} = $file_size; $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size} = $file_size;
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_md5sum} = $file_md5sum; $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_md5sum} = $file_md5sum;
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_type} = $file_type; $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_type} = $file_type;
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_mtime} = $file_mtime; $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_mtime} = $file_mtime;
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_location_uuid} = $file_location_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"anvils::anvil_uuid::${anvil_uuid}::file_name::${file_name}::file_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}, "anvils::anvil_uuid::${anvil_uuid}::file_name::${file_name}::file_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_name" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_name}, "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_name" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_name},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_directory" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_directory}, "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_directory" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_directory},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_size" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size}}).")", "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_size" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size}}).")",
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_md5sum" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_md5sum}, "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_md5sum" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_md5sum},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_type" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_type}, "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_type" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_type},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_mtime" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_mtime}, "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_mtime" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_mtime},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_location_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_location_uuid},
}}); }});
} }
} }
@ -6999,12 +6999,6 @@ If set, this is the file line number logged as the source of any INSERTs or UPDA
This is a free-form description for this Anvil! system. If this is set to C<< DELETED >>, the Anvil! will be considered to be deleted and no longer used. This is a free-form description for this Anvil! system. If this is set to C<< DELETED >>, the Anvil! will be considered to be deleted and no longer used.
=head3 anvil_dr1_host_uuid (optional)
This is the C<< hosts >> -> C<< host_uuid >> of the machine that is used as the DR host.
B<< Note >>: If set, there must be a matching C<< hosts >> -> C<< host_uuid >> in the database.
=head3 anvil_name (required) =head3 anvil_name (required)
This is the anvil's name. It is usually in the format C<< <prefix>-anvil-<zero-padded-sequence> >>. This is the anvil's name. It is usually in the format C<< <prefix>-anvil-<zero-padded-sequence> >>.
@ -7052,7 +7046,6 @@ sub insert_or_update_anvils
my $anvil_password = defined $parameter->{anvil_password} ? $parameter->{anvil_password} : ""; my $anvil_password = defined $parameter->{anvil_password} ? $parameter->{anvil_password} : "";
my $anvil_node1_host_uuid = defined $parameter->{anvil_node1_host_uuid} ? $parameter->{anvil_node1_host_uuid} : "NULL"; my $anvil_node1_host_uuid = defined $parameter->{anvil_node1_host_uuid} ? $parameter->{anvil_node1_host_uuid} : "NULL";
my $anvil_node2_host_uuid = defined $parameter->{anvil_node2_host_uuid} ? $parameter->{anvil_node2_host_uuid} : "NULL"; my $anvil_node2_host_uuid = defined $parameter->{anvil_node2_host_uuid} ? $parameter->{anvil_node2_host_uuid} : "NULL";
my $anvil_dr1_host_uuid = defined $parameter->{anvil_dr1_host_uuid} ? $parameter->{anvil_dr1_host_uuid} : "NULL";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
uuid => $uuid, uuid => $uuid,
file => $file, file => $file,
@ -7064,7 +7057,6 @@ sub insert_or_update_anvils
anvil_password => $anvil->Log->is_secure($anvil_password), anvil_password => $anvil->Log->is_secure($anvil_password),
anvil_node1_host_uuid => $anvil_node1_host_uuid, anvil_node1_host_uuid => $anvil_node1_host_uuid,
anvil_node2_host_uuid => $anvil_node2_host_uuid, anvil_node2_host_uuid => $anvil_node2_host_uuid,
anvil_dr1_host_uuid => $anvil_dr1_host_uuid,
}}); }});
if (not $delete) if (not $delete)
@ -7133,11 +7125,6 @@ WHERE
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0128", variables => { uuid => $anvil_node2_host_uuid, column => "anvil_node2_host_uuid" }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0128", variables => { uuid => $anvil_node2_host_uuid, column => "anvil_node2_host_uuid" }});
return(""); return("");
} }
if (($anvil_dr1_host_uuid) && (not $anvil->data->{hosts}{host_uuid}{$anvil_dr1_host_uuid}{host_name}))
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0128", variables => { uuid => $anvil_dr1_host_uuid, column => "anvil_dr1_host_uuid" }});
return("");
}
if ($delete) if ($delete)
{ {
@ -7190,8 +7177,6 @@ WHERE
# NULL values can't be quoted # NULL values can't be quoted
my $say_anvil_node1_host_uuid = $anvil_node1_host_uuid eq "" ? "NULL" : $anvil->Database->quote($anvil_node1_host_uuid); my $say_anvil_node1_host_uuid = $anvil_node1_host_uuid eq "" ? "NULL" : $anvil->Database->quote($anvil_node1_host_uuid);
my $say_anvil_node2_host_uuid = $anvil_node2_host_uuid eq "" ? "NULL" : $anvil->Database->quote($anvil_node2_host_uuid); my $say_anvil_node2_host_uuid = $anvil_node2_host_uuid eq "" ? "NULL" : $anvil->Database->quote($anvil_node2_host_uuid);
my $say_anvil_dr1_host_uuid = $anvil_dr1_host_uuid eq "" ? "NULL" : $anvil->Database->quote($anvil_dr1_host_uuid);
# If I still don't have an anvil_uuid, we're INSERT'ing . # If I still don't have an anvil_uuid, we're INSERT'ing .
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_uuid => $anvil_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_uuid => $anvil_uuid }});
@ -7211,7 +7196,6 @@ INSERT INTO
anvil_password, anvil_password,
anvil_node1_host_uuid, anvil_node1_host_uuid,
anvil_node2_host_uuid, anvil_node2_host_uuid,
anvil_dr1_host_uuid,
modified_date modified_date
) VALUES ( ) VALUES (
".$anvil->Database->quote($anvil_uuid).", ".$anvil->Database->quote($anvil_uuid).",
@ -7220,7 +7204,6 @@ INSERT INTO
".$anvil->Database->quote($anvil_password).", ".$anvil->Database->quote($anvil_password).",
".$say_anvil_node1_host_uuid.", ".$say_anvil_node1_host_uuid.",
".$say_anvil_node2_host_uuid.", ".$say_anvil_node2_host_uuid.",
".$say_anvil_dr1_host_uuid.",
".$anvil->Database->quote($anvil->Database->refresh_timestamp)." ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
); );
"; ";
@ -7236,8 +7219,7 @@ SELECT
anvil_description, anvil_description,
anvil_password, anvil_password,
anvil_node1_host_uuid, anvil_node1_host_uuid,
anvil_node2_host_uuid, anvil_node2_host_uuid
anvil_dr1_host_uuid
FROM FROM
anvils anvils
WHERE WHERE
@ -7264,14 +7246,12 @@ WHERE
my $old_anvil_password = $row->[2]; my $old_anvil_password = $row->[2];
my $old_anvil_node1_host_uuid = defined $row->[3] ? $row->[3] : "NULL"; my $old_anvil_node1_host_uuid = defined $row->[3] ? $row->[3] : "NULL";
my $old_anvil_node2_host_uuid = defined $row->[4] ? $row->[4] : "NULL"; my $old_anvil_node2_host_uuid = defined $row->[4] ? $row->[4] : "NULL";
my $old_anvil_dr1_host_uuid = defined $row->[5] ? $row->[5] : "NULL";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
old_anvil_name => $old_anvil_name, old_anvil_name => $old_anvil_name,
old_anvil_description => $old_anvil_description, old_anvil_description => $old_anvil_description,
old_anvil_password => $anvil->Log->is_secure($old_anvil_password), old_anvil_password => $anvil->Log->is_secure($old_anvil_password),
old_anvil_node1_host_uuid => $old_anvil_node1_host_uuid, old_anvil_node1_host_uuid => $old_anvil_node1_host_uuid,
old_anvil_node2_host_uuid => $old_anvil_node2_host_uuid, old_anvil_node2_host_uuid => $old_anvil_node2_host_uuid,
old_anvil_dr1_host_uuid => $old_anvil_dr1_host_uuid,
}}); }});
# Anything change? # Anything change?
@ -7279,8 +7259,7 @@ WHERE
($old_anvil_description ne $anvil_description) or ($old_anvil_description ne $anvil_description) or
($old_anvil_password ne $anvil_password) or ($old_anvil_password ne $anvil_password) or
($old_anvil_node1_host_uuid ne $anvil_node1_host_uuid) or ($old_anvil_node1_host_uuid ne $anvil_node1_host_uuid) or
($old_anvil_node2_host_uuid ne $anvil_node2_host_uuid) or ($old_anvil_node2_host_uuid ne $anvil_node2_host_uuid))
($old_anvil_dr1_host_uuid ne $anvil_dr1_host_uuid))
{ {
# Something changed, save. # Something changed, save.
my $query = " my $query = "
@ -7292,7 +7271,6 @@ SET
anvil_password = ".$anvil->Database->quote($anvil_password).", anvil_password = ".$anvil->Database->quote($anvil_password).",
anvil_node1_host_uuid = ".$say_anvil_node1_host_uuid.", anvil_node1_host_uuid = ".$say_anvil_node1_host_uuid.",
anvil_node2_host_uuid = ".$say_anvil_node2_host_uuid.", anvil_node2_host_uuid = ".$say_anvil_node2_host_uuid.",
anvil_dr1_host_uuid = ".$say_anvil_dr1_host_uuid.",
modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
WHERE WHERE
anvil_uuid = ".$anvil->Database->quote($anvil_uuid)." anvil_uuid = ".$anvil->Database->quote($anvil_uuid)."

@ -1459,7 +1459,11 @@ sub post_scan_analysis_dr
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "ScanCore->post_scan_analysis_dr()" }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "ScanCore->post_scan_analysis_dr()" }});
# Now that DR hosts are outside of specific Anvil! systems, auto-configuring their IPMI is more
# tricky. If there's no 'host_ipmi' set, and if there is a BCN1 IP, we'll configure using the BCN's
# third octet, plus 1. We'll use the password used for the database password. Later, if the IPMI
# is changed in the DB, we'll reconfigure to match.
$anvil->System->configure_ipmi({debug => $debug, dr => 1});
return(0); return(0);
} }

@ -1767,16 +1767,13 @@ sub get_size_of_block_device
$anvil->Database->get_anvils({debug => $debug}); $anvil->Database->get_anvils({debug => $debug});
my $node1_host_uuid = ""; my $node1_host_uuid = "";
my $node2_host_uuid = ""; my $node2_host_uuid = "";
my $dr1_host_uuid = "";
if ($anvil_uuid) if ($anvil_uuid)
{ {
$node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
$node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
$dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
node1_host_uuid => $node1_host_uuid, node1_host_uuid => $node1_host_uuid,
node1_host_uuid => $node2_host_uuid, node1_host_uuid => $node2_host_uuid,
dr1_host_uuid => $dr1_host_uuid,
}}); }});
} }
@ -1807,19 +1804,11 @@ AND
( (
scan_lvm_lv_host_uuid = ".$anvil->Database->quote($node1_host_uuid)." scan_lvm_lv_host_uuid = ".$anvil->Database->quote($node1_host_uuid)."
OR OR
scan_lvm_lv_host_uuid = ".$anvil->Database->quote($node2_host_uuid); scan_lvm_lv_host_uuid = ".$anvil->Database->quote($node2_host_uuid)."
if ($dr1_host_uuid) )
{
$query .= "
OR
scan_lvm_lv_host_uuid = ".$anvil->Database->quote($dr1_host_uuid);
}
$query .= "
)";
}
$query .= "
LIMIT 1 LIMIT 1
;"; ;";
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results}; my $count = @{$results};
@ -1911,11 +1900,7 @@ LIMIT 1
($anvil_uuid) && ($anvil_uuid) &&
( (
($this_host_uuid eq $node1_host_uuid) or ($this_host_uuid eq $node1_host_uuid) or
($this_host_uuid eq $node2_host_uuid) or ($this_host_uuid eq $node2_host_uuid)
(
($dr1_host_uuid) &&
($this_host_uuid eq $dr1_host_uuid)
)
) )
) )
) )
@ -1976,14 +1961,7 @@ AND
( (
a.scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node1_host_uuid)." a.scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node1_host_uuid)."
OR OR
a.scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node2_host_uuid); a.scan_drbd_resource_host_uuid = ".$anvil->Database->quote($node2_host_uuid)."
if ($dr1_host_uuid)
{
$query .= "
OR
a.scan_drbd_resource_host_uuid = ".$anvil->Database->quote($dr1_host_uuid);
}
$query .= "
)"; )";
} }
$query .= " $query .= "
@ -2222,16 +2200,13 @@ sub get_storage_group_from_path
$anvil->Database->get_anvils({debug => $debug}); $anvil->Database->get_anvils({debug => $debug});
my $node1_host_uuid = ""; my $node1_host_uuid = "";
my $node2_host_uuid = ""; my $node2_host_uuid = "";
my $dr1_host_uuid = "";
if ($anvil_uuid) if ($anvil_uuid)
{ {
$node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
$node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
$dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
node1_host_uuid => $node1_host_uuid, node1_host_uuid => $node1_host_uuid,
node1_host_uuid => $node2_host_uuid, node1_host_uuid => $node2_host_uuid,
dr1_host_uuid => $dr1_host_uuid,
}}); }});
} }
@ -2272,11 +2247,7 @@ sub get_storage_group_from_path
($anvil_uuid) && ($anvil_uuid) &&
( (
($this_host_uuid eq $node1_host_uuid) or ($this_host_uuid eq $node1_host_uuid) or
($this_host_uuid eq $node2_host_uuid) or ($this_host_uuid eq $node2_host_uuid)
(
($dr1_host_uuid) &&
($this_host_uuid eq $dr1_host_uuid)
)
) )
) )
) )

@ -1897,6 +1897,10 @@ B<< NOTE >>: The password used to set the IPMI BMC access is included both in th
Parameters; Parameters;
=head3 dr (Optional, default 0)
This indicates that a DR host is being configured. If used, C<< manifest_uuid >> is ignored.
=head3 manifest_uuid (Optional, default sys::manifest_uuid) =head3 manifest_uuid (Optional, default sys::manifest_uuid)
The C<< manifests >> -> C<< manifest_uuid >> used to pull out configuration data. This is required, but in most cases, it can be determined if not passed. The C<< manifests >> -> C<< manifest_uuid >> used to pull out configuration data. This is required, but in most cases, it can be determined if not passed.
@ -1912,65 +1916,138 @@ sub configure_ipmi
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "System->configure_ipmi()" }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "System->configure_ipmi()" }});
my $dr = defined $parameter->{dr} ? $parameter->{dr} : "";
my $manifest_uuid = defined $parameter->{manifest_uuid} ? $parameter->{manifest_uuid} : ""; my $manifest_uuid = defined $parameter->{manifest_uuid} ? $parameter->{manifest_uuid} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
dr => $dr,
manifest_uuid => $manifest_uuid, manifest_uuid => $manifest_uuid,
}}); }});
if ((not $manifest_uuid) && (exists $anvil->data->{sys}{manifest_uuid}) && ($anvil->data->{sys}{manifest_uuid}))
{
$manifest_uuid = $anvil->data->{sys}{manifest_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { manifest_uuid => $manifest_uuid }});
}
if (not $manifest_uuid)
{
# Nothing more we can do.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Systeme->configure_ipmi()", parameter => "manifest_uuid" }});
return(0);
}
# Is this host in an Anvil!?
$anvil->Database->get_hosts(); $anvil->Database->get_hosts();
$anvil->Database->get_anvils(); $anvil->Database->get_anvils();
my $host_uuid = $anvil->Get->host_uuid;
my $anvil_uuid = ""; my $ipmi_ip_address = "";
my $host_uuid = $anvil->Get->host_uuid; my $ipmi_password = "";
if ((exists $anvil->data->{hosts}{host_uuid}{$host_uuid}) && ($anvil->data->{hosts}{host_uuid}{$host_uuid}{anvil_uuid})) my $password_length = 0;
{ my $subnet_mask = "";
# We're in an Anvil! my $gateway = "";
$anvil_uuid = $anvil->data->{hosts}{host_uuid}{$host_uuid}{anvil_uuid}; my $in_network = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_uuid => $anvil_uuid }}); if ($dr)
{
my $host_ipmi = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_ipmi => $host_ipmi }});
if (not $host_ipmi)
{
# Do initial config.
$anvil->Network->load_ips({
debug => $debug,
host_uuid => $host_uuid,
});
foreach my $interface_name (sort {$a cmp $b} keys %{$anvil->data->{network}{$host_uuid}{interface}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { interface_name => $interface_name }});
next if $interface_name !~ /^bcn1_/;
my $ip_address_address = $anvil->data->{network}{$host_uuid}{interface}{$interface_name}{ip};
my $ip_address_subnet_mask = $anvil->data->{network}{$host_uuid}{interface}{$interface_name}{subnet_mask};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
ip_address_address => $ip_address_address,
ip_address_subnet_mask => $ip_address_subnet_mask,
}});
# We need this to be a /16
if ($ip_address_subnet_mask eq "255.255.0.0")
{
# Get the 3rd octet.
my ($first_octet, $second_octet, $third_octet, $fourth_octet) = ($ip_address_address =~ /^(\d+)\.(\d+)\.(\d+)\.(\d+)$/);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
first_octet => $first_octet,
second_octet => $second_octet,
third_octet => $third_octet,
fourth_octet => $fourth_octet,
}});
$ipmi_ip_address = $first_octet.".".$second_octet.".".($third_octet+1).".".$fourth_octet;
$subnet_mask = $ip_address_subnet_mask;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
ipmi_ip_address => $ipmi_ip_address,
subnet_mask => $subnet_mask,
}});
last;
}
}
# Get the password using the Striker password.
my $db_uuid = $anvil->data->{sys}{database}{read_uuid};
$ipmi_password = $anvil->data->{database}{$db_uuid}{password};
$password_length = length(Encode::encode('UTF-8', $ipmi_password));
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
db_uuid => $db_uuid,
ipmi_password => $anvil->Log->is_secure($ipmi_password),
password_length => $password_length,
}});
if ((not $anvil->Validate->ipv4({debug => $debug, ip => $ipmi_ip_address})) or (not $ipmi_password))
{
# Can't proceed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "warning_0149"});
return(0);
}
}
else
{
# TODO: Check if the local IPMI config matches the value in host_ipmi and, if not,
# reconfigure to match.
return(0);
}
} }
else else
{ {
# Not in an Anvil!, return 0. if ((not $manifest_uuid) && (exists $anvil->data->{sys}{manifest_uuid}) && ($anvil->data->{sys}{manifest_uuid}))
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0498"}); {
return(0); $manifest_uuid = $anvil->data->{sys}{manifest_uuid};
} $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { manifest_uuid => $manifest_uuid }});
}
# Look for a match in the anvils table for this host uuid. if (not $manifest_uuid)
my $machine = ""; {
if ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid} eq $host_uuid) # Nothing more we can do.
{ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Systeme->configure_ipmi()", parameter => "manifest_uuid" }});
$machine = "node1"; return(0);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { machine => $machine }}); }
}
elsif ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid} eq $host_uuid) # Is this host in an Anvil!?
{ my $anvil_uuid = "";
$machine = "node2"; if ((exists $anvil->data->{hosts}{host_uuid}{$host_uuid}) && ($anvil->data->{hosts}{host_uuid}{$host_uuid}{anvil_uuid}))
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { machine => $machine }}); {
} # We're in an Anvil!
elsif ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} eq $host_uuid) $anvil_uuid = $anvil->data->{hosts}{host_uuid}{$host_uuid}{anvil_uuid};
{ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_uuid => $anvil_uuid }});
$machine = "dr1"; }
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { machine => $machine }}); else
} {
# Not in an Anvil!, return 0.
if (not $machine) $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0498"});
{ return(0);
# Look for a job for 'anvil-join-anvil' for this host. With it, we'll figure out the password }
# and which machine we are.
my $query = " # Look for a match in the anvils table for this host uuid.
my $machine = "";
if ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid} eq $host_uuid)
{
$machine = "node1";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { machine => $machine }});
}
elsif ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid} eq $host_uuid)
{
$machine = "node2";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { machine => $machine }});
}
if (not $machine)
{
# Look for a job for 'anvil-join-anvil' for this host. With it, we'll figure out the password
# and which machine we are.
my $query = "
SELECT SELECT
job_uuid, job_uuid,
job_data job_data
@ -1984,56 +2061,109 @@ ORDER BY
modified_date DESC modified_date DESC
LIMIT 1 LIMIT 1
;"; ;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results}; my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results, results => $results,
count => $count, count => $count,
}}); }});
my $job_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : ""; my $job_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : "";
my $job_data = defined $results->[0]->[1] ? $results->[0]->[1] : ""; my $job_data = defined $results->[0]->[1] ? $results->[0]->[1] : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
job_uuid => $job_uuid, job_uuid => $job_uuid,
job_data => $anvil->Log->is_secure($job_data), job_data => $anvil->Log->is_secure($job_data),
}}); }});
if (not $job_uuid) if (not $job_uuid)
{
# Unable to proceed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0501"});
return(0);
}
($machine, $manifest_uuid, $anvil_uuid) = ($job_data =~ /as_machine=(.*?),manifest_uuid=(.*?),anvil_uuid=(.*?)$/);
$machine = "" if not defined $machine;
$manifest_uuid = "" if not defined $manifest_uuid;
$anvil_uuid = "" if not defined $anvil_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
machine => $machine,
manifest_uuid => $manifest_uuid,
anvil_uuid => $anvil_uuid,
}});
}
# Load the manifest.
my $problem = $anvil->Striker->load_manifest({debug => $debug, manifest_uuid => $manifest_uuid});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }});
if ($problem)
{ {
# Unable to proceed. # The load_manifest method would log the details.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0501"});
return(0); return(0);
} }
($machine, $manifest_uuid, $anvil_uuid) = ($job_data =~ /as_machine=(.*?),manifest_uuid=(.*?),anvil_uuid=(.*?)$/); # Make sure the IPMI IP, subnet mask and password are available.
$machine = "" if not defined $machine; $ipmi_ip_address = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{ipmi_ip};
$manifest_uuid = "" if not defined $manifest_uuid; $ipmi_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
$anvil_uuid = "" if not defined $anvil_uuid; $password_length = length(Encode::encode('UTF-8', $ipmi_password));
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
machine => $machine, ipmi_ip_address => $ipmi_ip_address,
manifest_uuid => $manifest_uuid, ipmi_password => $anvil->Log->is_secure($ipmi_password),
anvil_uuid => $anvil_uuid, password_length => $password_length,
}}); }});
# Find the subnet the IPMI IP is in.
foreach my $network_type ("bcn", "ifn", "sn")
{
my $count = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{count}{$network_type};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
network_type => $network_type,
count => $count,
}});
foreach my $i (1..$count)
{
my $network_name = $network_type.$i;
my $network = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network_name}{network};
my $this_subnet_mask = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network_name}{subnet};
my $this_gateway = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network_name}{gateway};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
network_name => $network_name,
network => $network,
this_subnet_mask => $this_subnet_mask,
this_gateway => $this_gateway,
}});
my $match = $anvil->Network->is_ip_in_network({
network => $network,
subnet_mask => $this_subnet_mask,
ip => $ipmi_ip_address,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { network_name => $match}});
if ($match)
{
$subnet_mask = $this_subnet_mask;
$gateway = $this_gateway;
$in_network = $network_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
subnet_mask => $subnet_mask,
gateway => $gateway,
in_network => $in_network,
}});
last;
}
}
}
# If we didn't find a network, we're done.
if (not $subnet_mask)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0502", variables => {
ip_address => $ipmi_ip_address,
manifest_uuid => $manifest_uuid,
}});
return(0);
}
} }
# Load the manifest.
my $problem = $anvil->Striker->load_manifest({debug => $debug, manifest_uuid => $manifest_uuid});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }});
if ($problem)
{
# The load_manifest method would log the details.
return(0);
}
# Make sure the IPMI IP, subnet mask and password are available.
my $ipmi_ip_address = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{ipmi_ip};
my $ipmi_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
my $password_length = length(Encode::encode('UTF-8', $ipmi_password));
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
ipmi_ip_address => $ipmi_ip_address,
ipmi_password => $anvil->Log->is_secure($ipmi_password),
password_length => $password_length,
}});
# If the password has spaces, some IPMI BMCs won't allow them. If we need to use it, we'll take out # If the password has spaces, some IPMI BMCs won't allow them. If we need to use it, we'll take out
# the spaces and shrink the length. # the spaces and shrink the length.
my $ipmi_no_space_password = ""; my $ipmi_no_space_password = "";
@ -2044,61 +2174,7 @@ LIMIT 1
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => 1, list => { ipmi_no_space_password => $ipmi_no_space_password }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => 1, list => { ipmi_no_space_password => $ipmi_no_space_password }});
} }
my $subnet_mask = "";
my $gateway = "";
my $in_network = "";
# Find the subnet the IPMI IP is in.
foreach my $network_type ("bcn", "ifn", "sn")
{
my $count = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{count}{$network_type};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
network_type => $network_type,
count => $count,
}});
foreach my $i (1..$count)
{
my $network_name = $network_type.$i;
my $network = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network_name}{network};
my $this_subnet_mask = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network_name}{subnet};
my $this_gateway = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network_name}{gateway};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
network_name => $network_name,
network => $network,
this_subnet_mask => $this_subnet_mask,
this_gateway => $this_gateway,
}});
my $match = $anvil->Network->is_ip_in_network({
network => $network,
subnet_mask => $this_subnet_mask,
ip => $ipmi_ip_address,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { network_name => $match}});
if ($match)
{
$subnet_mask = $this_subnet_mask;
$gateway = $this_gateway;
$in_network = $network_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
subnet_mask => $subnet_mask,
gateway => $gateway,
in_network => $in_network,
}});
last;
}
}
}
# If we didn't find a network, we're done.
if (not $subnet_mask)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "err", key => "log_0502", variables => {
ip_address => $ipmi_ip_address,
manifest_uuid => $manifest_uuid,
}});
return(0);
}
# Call dmidecode to see if there even is an IPMI BMC on this host. # Call dmidecode to see if there even is an IPMI BMC on this host.
my $host_ipmi = ""; my $host_ipmi = "";

@ -206,7 +206,7 @@ sub clean_spaces
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Words->clean_spaces()" }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Words->clean_spaces()" }});
# Setup default values # Setup default values
my $string = defined $parameter->{string} ? $parameter->{string} : ""; my $string = defined $parameter->{string} ? $parameter->{string} : "";
my $merge_spaces = defined $parameter->{merge_spaces} ? $parameter->{merge_spaces} : 1; my $merge_spaces = defined $parameter->{merge_spaces} ? $parameter->{merge_spaces} : 1;
$string =~ s/^\s+//; $string =~ s/^\s+//;

20
notes

@ -1,8 +1,28 @@
Tasks;
1. When provisioning a server;
- First, check if either node is SyncSource, if so use that node.
- Second, check which node has the most servers by RAM count, use that node.
2. Provision;
- Create DRBD resource, force primary on install target
- Create pacemaker resource in stopped state
- Set location constraint to prefer target node
- Boot server
Create "Node status" which returns "degraded" if the peer is gone
Common queries; Common queries;
* SELECT a.job_uuid, b.host_name, a.job_command, a.job_data, a.job_progress, a.job_status FROM jobs a, hosts b WHERE a.job_host_uuid = b.host_uuid AND a.job_progress != 100; * SELECT a.job_uuid, b.host_name, a.job_command, a.job_data, a.job_progress, a.job_status FROM jobs a, hosts b WHERE a.job_host_uuid = b.host_uuid AND a.job_progress != 100;
* SELECT a.host_name, b.file_name, c.file_location_active FROM hosts a, files b, file_locations c WHERE a.host_uuid = c.file_location_host_uuid AND b.file_uuid = c.file_location_file_uuid ORDER BY b.file_name ASC, a.host_name ASC; * SELECT a.host_name, b.file_name, c.file_location_active FROM hosts a, files b, file_locations c WHERE a.host_uuid = c.file_location_host_uuid AND b.file_uuid = c.file_location_file_uuid ORDER BY b.file_name ASC, a.host_name ASC;
* SELECT a.dr_link_uuid, b.host_name, c.anvil_name, a.dr_link_note FROM dr_links a, hosts b, anvils c WHERE a.dr_link_host_uuid = b.host_uuid AND a.dr_link_anvil_uuid = c.anvil_uuid ORDER BY c.anvil_name ASC, b.host_name ASC; * SELECT a.dr_link_uuid, b.host_name, c.anvil_name, a.dr_link_note FROM dr_links a, hosts b, anvils c WHERE a.dr_link_host_uuid = b.host_uuid AND a.dr_link_anvil_uuid = c.anvil_uuid ORDER BY c.anvil_name ASC, b.host_name ASC;
# Fail a resource for testing purposes.
crm_resource --fail --resource srv02-b -N vm-a01n01
# Recover without reboot
crm_resource --resource srv01-a --refresh
uname -r; grubby --default-kernel; lsinitrd -m /boot/initramfs-4.18.0-448.el8.x86_64.img | grep lvm; systemctl is-enabled scancore.service; uname -r; grubby --default-kernel; lsinitrd -m /boot/initramfs-4.18.0-448.el8.x86_64.img | grep lvm; systemctl is-enabled scancore.service;
dnf -y update; systemctl disable --now anvil-daemon; systemctl disable --now scancore dnf -y update; systemctl disable --now anvil-daemon; systemctl disable --now scancore

@ -256,6 +256,18 @@ if (not $anvil->data->{switches}{monitor})
show_environment($anvil, 3); show_environment($anvil, 3);
} }
# If there's a test fail file, return '1' to cause pacemaker to fail this resource.
if ($anvil->data->{environment}{OCF_RESKEY_name})
{
my $test_fail_file = "/tmp/".$anvil->data->{environment}{OCF_RESKEY_name}.".fail";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_fail_file => $test_fail_file }});
if (-e $test_fail_file)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0150", variables => { fail_file => $test_fail_file }});
$anvil->nice_exit({exit_code => 1});
}
}
### What are we being asked to do? ### What are we being asked to do?
# start -Starts the resource. # start -Starts the resource.
# stop -Shuts down the resource. # stop -Shuts down the resource.

@ -99,6 +99,9 @@ check_config($anvil);
# Check the fence delay # Check the fence delay
check_fence_delay($anvil); check_fence_delay($anvil);
# Check for failed resources
check_resources($anvil);
# Shut down. # Shut down.
$anvil->ScanCore->agent_shutdown({agent => $THIS_FILE}); $anvil->ScanCore->agent_shutdown({agent => $THIS_FILE});
@ -107,6 +110,191 @@ $anvil->ScanCore->agent_shutdown({agent => $THIS_FILE});
# Functions # # Functions #
############################################################################################################# #############################################################################################################
# This looks for failed resource and, if found, tries to recover them.
sub check_resources
{
my ($anvil) = @_;
foreach my $server (sort {$a cmp $b} keys %{$anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}})
{
# This is used for alerts, if needed below.
my $variables = { server => $server };
my $failed = check_if_server_failed($anvil, $server);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
server => $server,
failed => $failed,
}});
if ($failed)
{
# Who am I and who is my peer? See if the server is running on either host.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0003", variables => { server => $server }});
my $attempt_recovery = 0;
my $server_found = 0;
my $both_nodes_ready = 1;
foreach my $target ("local", "peer")
{
my $node_ready = $anvil->data->{cib}{parsed}{$target}{ready};
my $node_name = $anvil->data->{cib}{parsed}{$target}{name};
my $host_uuid = $anvil->Get->host_uuid_from_name({host_name => $node_name});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0004", variables => {
node_name => $node_name,
host_uuid => $host_uuid,
node_ready => $node_ready,
}});
if (not $node_ready)
{
$both_nodes_ready = 1;
}
if ($host_uuid eq $anvil->Get->host_uuid)
{
# Search for the server here
$anvil->Server->find({debug => 2});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0005"});
}
else
{
# Search for the server on the peer.
my $target_ip = $anvil->Network->find_target_ip({host_uuid => $host_uuid});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0006", variables => { target_ip => $target_ip }});
$anvil->Server->find({
debug => 2,
target => $target_ip,
});
}
my $server_host = defined $anvil->data->{server}{location}{$server}{host_name} ? $anvil->data->{server}{location}{$server}{host_name} : "";
my $server_status = defined $anvil->data->{server}{location}{$server}{status} ? $anvil->data->{server}{location}{$server}{status} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => {
server_host => $server_host,
server_status => $server_status,
}});
if ($server_host)
{
$server_found = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_found => $server_found }});
if (($node_ready) && ($host_uuid eq $anvil->Get->host_uuid))
{
# Go ahead with recovery
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0007"});
$attempt_recovery = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { attempt_recovery => $attempt_recovery }});
}
}
}
if ((not $server_found) && ($both_nodes_ready))
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0008"});
$attempt_recovery = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { attempt_recovery => $attempt_recovery }});
}
elsif (($server_found) && (not $attempt_recovery))
{
# The server was found to be running, but not here (or this node is not fully in the cluster). NOT attempting recovery yet.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0009"});
}
elsif ($attempt_recovery)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0010"});
$anvil->Cluster->recover_server({
debug => 2,
server => $server,
});
# It'll leave 'failed state' for a bit, so we need to wait.
sleep 3;
my $wait_until = time + 10;
my $waiting = 1;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_log_0011"});
while($waiting)
{
my $failed = check_if_server_failed($anvil, $server);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
if ($failed)
{
# No luck...
$waiting = 0;
my $changed = $anvil->Alert->check_alert_sent({
record_locator => "scan_cluster::failed_server::".$server,
set_by => $THIS_FILE,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
waiting => $waiting,
changed => $changed,
}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_alert_0014", variables => $variables});
if ($changed)
{
# Send an alert.
$anvil->Alert->register({debug => 2, alert_level => "notice", message => "scan_cluster_alert_0014", variables => $variables, set_by => $THIS_FILE});
}
}
elsif (time > $wait_until)
{
# Success!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_alert_0013", variables => $variables});
$anvil->Alert->register({debug => 2, alert_level => "notice", message => "scan_cluster_alert_0013", variables => $variables, set_by => $THIS_FILE});
# Clear the alert, if it existed before
$waiting = 0;
my $changed = $anvil->Alert->check_alert_sent({
record_locator => "scan_cluster::failed_server::".$server,
set_by => $THIS_FILE,
clear => 1,
});
}
else
{
# Wait a sec
sleep 2;
}
}
}
}
else
{
# Make sure that this server wasn't previously failed.
my $changed = $anvil->Alert->check_alert_sent({
record_locator => "scan_cluster::failed_server::".$server,
set_by => $THIS_FILE,
clear => 1,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { changed => $changed }});
if ($changed)
{
# Send the All-good alert.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "scan_cluster_alert_0015", variables => $variables});
$anvil->Alert->register({debug => 2, alert_level => "notice", message => "scan_cluster_alert_0015", variables => $variables, set_by => $THIS_FILE});
}
}
}
return(0);
}
sub check_if_server_failed
{
my ($anvil, $server) = @_;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { server => $server }});
$anvil->Cluster->parse_crm_mon({debug => 3});
my $failed = exists $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$server}{variables}{failed} ? $anvil->data->{crm_mon}{parsed}{'pacemaker-result'}{resources}{resource}{$server}{variables}{failed} : 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }});
if ($failed eq "true")
{
$failed = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }});
}
elsif ($failed eq "false")
{
$failed = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }});
}
return($failed);
}
# Check to see if we need to move the fence delay. # Check to see if we need to move the fence delay.
sub check_fence_delay sub check_fence_delay
{ {

@ -37,10 +37,22 @@ In Maintenance Mode: ..... [#!variable!maintenance_mode!#]
#!variable!difference!# #!variable!difference!#
==== ====
</key> </key>
<key name="scan_cluster_alert_0013">The server: [#!variable!server!#] was found to be failed in pacemaker, but it was successfully recovered. This does NOT mean the server rebooted, but it may have. Checking the server is advised.</key>
<key name="scan_cluster_alert_0014">The server: [#!variable!server!#] was found to be failed in pacemaker. The attempt to recover it appears to have failed. The server might well still be running ok, checking the server is advised.</key>
<key name="scan_cluster_alert_0015">The server: [#!variable!server!#] had been found to be failed in pacemaker. It's now recovered. This does NOT mean the server rebooted, but it may have. Checking the server is advised.</key>
<!-- Log entries --> <!-- Log entries -->
<key name="scan_cluster_log_0001">Starting: [#!variable!program!#].</key> <key name="scan_cluster_log_0001">Starting: [#!variable!program!#].</key>
<key name="scan_cluster_log_0002">This host is a: [#!variable!host_type!#], this agent is only useful on nodes. Exiting.</key> <key name="scan_cluster_log_0002">This host is a: [#!variable!host_type!#], this agent is only useful on nodes. Exiting.</key>
<key name="scan_cluster_log_0003">[ Warning ] - The server: [#!variable!server!#] is in a FAILED state! Checking to see if it's safe to attempt recovery.</key>
<key name="scan_cluster_log_0004">Searching node: [#!variable!node_name!# (#!variable!host_uuid!#] which is in ready state: [#!variable!node_ready!#].</key>
<key name="scan_cluster_log_0005">Searching for the server on the local system.</key>
<key name="scan_cluster_log_0006">Searching for the server on the peer using IP: [#!variable!target_ip!#].</key>
<key name="scan_cluster_log_0007">The server is running locally and we're a full cluster member. Will attempt recover.</key>
<key name="scan_cluster_log_0008">Both nodes are up and the server wasn't found anywhere. Attempting recovery.</key>
<key name="scan_cluster_log_0009">The server was found to be running, but not here (or this node is not fully in the cluster). NOT attempting recovery yet.</key>
<key name="scan_cluster_log_0010">Attempting recovery now...</key>
<key name="scan_cluster_log_0011">Checking to see if the server has recovered yet...</key>
<!-- Message entries (usually meant to be alerts) --> <!-- Message entries (usually meant to be alerts) -->
<key name="scan_cluster_message_0001"></key> <key name="scan_cluster_message_0001"></key>

@ -2086,8 +2086,7 @@ sub find_ipmi_targets
SELECT SELECT
anvil_name, anvil_name,
anvil_node1_host_uuid, anvil_node1_host_uuid,
anvil_node2_host_uuid, anvil_node2_host_uuid
anvil_dr1_host_uuid
FROM FROM
anvils anvils
WHERE WHERE
@ -2107,12 +2106,10 @@ WHERE
my $anvil_name = $row->[0]; my $anvil_name = $row->[0];
my $anvil_node1_host_uuid = $row->[1]; my $anvil_node1_host_uuid = $row->[1];
my $anvil_node2_host_uuid = $row->[2]; my $anvil_node2_host_uuid = $row->[2];
my $anvil_dr1_host_uuid = defined $row->[3] ? $row->[3] : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:anvil_name' => $anvil_name, 's1:anvil_name' => $anvil_name,
's2:anvil_node1_host_uuid' => $anvil_node1_host_uuid, 's2:anvil_node1_host_uuid' => $anvil_node1_host_uuid,
's3:anvil_node2_host_uuid' => $anvil_node2_host_uuid, 's3:anvil_node2_host_uuid' => $anvil_node2_host_uuid,
's4:anvil_dr1_host_uuid' => $anvil_dr1_host_uuid,
}}); }});
my $query = " my $query = "
@ -2128,14 +2125,7 @@ AND
( (
host_uuid = ".$anvil->Database->quote($anvil_node1_host_uuid)." host_uuid = ".$anvil->Database->quote($anvil_node1_host_uuid)."
OR OR
host_uuid = ".$anvil->Database->quote($anvil_node2_host_uuid); host_uuid = ".$anvil->Database->quote($anvil_node2_host_uuid)."
if ($anvil_dr1_host_uuid)
{
$query .= "
OR
host_uuid = ".$anvil->Database->quote($anvil_dr1_host_uuid);
}
$query .= "
) )
;"; ;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});

@ -1081,7 +1081,7 @@ DELETED - Marks a server as no longer existing
next if $server_host_uuid ne $anvil->Get->host_uuid; next if $server_host_uuid ne $anvil->Get->host_uuid;
# If we're here, the server used to be on us, and isn't on the peer, so mark it as off. # If we're here, the server used to be on us, and isn't on the peer, so mark it as off.
if ($server_state ne "shut off") if (($server_state ne "shut off") && ($server_state ne "DELETED"))
{ {
# Mark it as being off now. # Mark it as being off now.
my $query = " my $query = "

@ -4381,6 +4381,7 @@ INSERT INTO
foreach my $variable (sort {$a cmp $b} keys %{$anvil->data->{sql}{scan_storcli_variables}{scan_storcli_variable_uuid}{source_table}{'scan_storcli_cachevaults'}{source_uuid}{$cachevault_uuid}{$type}}) foreach my $variable (sort {$a cmp $b} keys %{$anvil->data->{sql}{scan_storcli_variables}{scan_storcli_variable_uuid}{source_table}{'scan_storcli_cachevaults'}{source_uuid}{$cachevault_uuid}{$type}})
{ {
# This variable has vanished # This variable has vanished
next if not defined $anvil->data->{sql}{scan_storcli_variables}{scan_storcli_variable_uuid}{source_table}{'scan_storcli_cachevaults'}{source_uuid}{$cachevault_uuid}{$type}{$variable}{scan_storcli_variable_value};
my $old_variable_value = $anvil->data->{sql}{scan_storcli_variables}{scan_storcli_variable_uuid}{source_table}{'scan_storcli_cachevaults'}{source_uuid}{$cachevault_uuid}{$type}{$variable}{scan_storcli_variable_value}; my $old_variable_value = $anvil->data->{sql}{scan_storcli_variables}{scan_storcli_variable_uuid}{source_table}{'scan_storcli_cachevaults'}{source_uuid}{$cachevault_uuid}{$type}{$variable}{scan_storcli_variable_value};
my $variable_uuid = $anvil->data->{sql}{scan_storcli_variables}{scan_storcli_variable_uuid}{source_table}{'scan_storcli_cachevaults'}{source_uuid}{$cachevault_uuid}{$type}{$variable}{scan_storcli_variable_uuid}; my $variable_uuid = $anvil->data->{sql}{scan_storcli_variables}{scan_storcli_variable_uuid}{source_table}{'scan_storcli_cachevaults'}{source_uuid}{$cachevault_uuid}{$type}{$variable}{scan_storcli_variable_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
@ -9070,8 +9071,8 @@ sub get_cachevault_data
($variable, $value) = ($line =~ /^(.{$split_point})(.*)$/); ($variable, $value) = ($line =~ /^(.{$split_point})(.*)$/);
$variable =~ s/\s+$//; $variable =~ s/\s+$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
variable => $variable, 's1:variable' => $variable,
value => $value, 's2:value' => $value,
}}); }});
} }

@ -1979,52 +1979,52 @@ CREATE TRIGGER trigger_temperature
AFTER INSERT OR UPDATE ON temperature AFTER INSERT OR UPDATE ON temperature
FOR EACH ROW EXECUTE PROCEDURE history_temperature(); FOR EACH ROW EXECUTE PROCEDURE history_temperature();
-- -- Enable after Yan's review
-- This is used to audit major events by user. -- This is used to audit major events by user.
--CREATE TABLE audits ( CREATE TABLE audits (
--audit_uuid uuid primary key, audit_uuid uuid primary key,
--audit_user_uuid uuid not null, -- This is the users -> user_uuid the audit is tracking audit_user_uuid uuid not null, -- This is the users -> user_uuid the audit is tracking
--audit_details text not null, -- This is the information explaining the action being audited. audit_details text not null, -- This is the information explaining the action being audited.
--modified_date timestamp with time zone not null, modified_date timestamp with time zone not null,
--FOREIGN KEY(audit_user_uuid) REFERENCES users(user_uuid) FOREIGN KEY(audit_user_uuid) REFERENCES users(user_uuid)
--); );
--ALTER TABLE audits OWNER TO admin; ALTER TABLE audits OWNER TO admin;
--CREATE TABLE history.audits ( CREATE TABLE history.audits (
--history_id bigserial, history_id bigserial,
--audit_uuid uuid, audit_uuid uuid,
--audit_user_uuid uuid, audit_user_uuid uuid,
--audit_details text, audit_details text,
--modified_date timestamp with time zone not null modified_date timestamp with time zone not null
--); );
--ALTER TABLE history.audits OWNER TO admin; ALTER TABLE history.audits OWNER TO admin;
--CREATE FUNCTION history_audits() RETURNS trigger CREATE FUNCTION history_audits() RETURNS trigger
--AS $$ AS $$
--DECLARE DECLARE
--history_audits RECORD; history_audits RECORD;
--BEGIN BEGIN
--SELECT INTO history_audits * FROM audits WHERE audit_uuid = new.audit_uuid; SELECT INTO history_audits * FROM audits WHERE audit_uuid = new.audit_uuid;
--INSERT INTO history.audits INSERT INTO history.audits
--(audit_uuid, (audit_uuid,
--audit_user_uuid, audit_user_uuid,
--audit_details, audit_details,
--modified_date) modified_date)
--VALUES VALUES
--(history_audit.audit_uuid, (history_audit.audit_uuid,
--history_audit.audit_user_uuid, history_audit.audit_user_uuid,
--history_audit.audit_details, history_audit.audit_details,
--history_audit.modified_date); history_audit.modified_date);
--RETURN NULL; RETURN NULL;
--END; END;
--$$ $$
--LANGUAGE plpgsql; LANGUAGE plpgsql;
--ALTER FUNCTION history_audits() OWNER TO admin; ALTER FUNCTION history_audits() OWNER TO admin;
--CREATE TRIGGER trigger_audits CREATE TRIGGER trigger_audits
--AFTER INSERT OR UPDATE ON audits AFTER INSERT OR UPDATE ON audits
--FOR EACH ROW EXECUTE PROCEDURE history_audits(); FOR EACH ROW EXECUTE PROCEDURE history_audits();
-- ------------------------------------------------------------------------------------------------------- -- -- ------------------------------------------------------------------------------------------------------- --

@ -3582,6 +3582,8 @@ The error was:
</key> </key>
<key name="warning_0147">[ Warning ] - The interface: [#!variable!interface!#] appears to be down (state: [#!variable!state!#]). The system uptime is: [#!variable!uptime!#], so it might be a problem where the interface didn't start on boot as it should have. So we're going to bring the interface up.</key> <key name="warning_0147">[ Warning ] - The interface: [#!variable!interface!#] appears to be down (state: [#!variable!state!#]). The system uptime is: [#!variable!uptime!#], so it might be a problem where the interface didn't start on boot as it should have. So we're going to bring the interface up.</key>
<key name="warning_0148">[ Warning ] - The IPMI stonith resource: [#!variable!resource!#] is in the role: [#!variable!role!#] (should be 'Started'). Will check the IPMI config now.</key> <key name="warning_0148">[ Warning ] - The IPMI stonith resource: [#!variable!resource!#] is in the role: [#!variable!role!#] (should be 'Started'). Will check the IPMI config now.</key>
<key name="warning_0149">[ Warning ] - Failed to find a valid IP address or password to be used to setup the DR host's IPMI.</key>
<key name="warning_0150">[ Warning ] - The test "fail file": [#!variable!fail_file!#] was found. So long as this file exists, the ocf:alteeve:server RA will return 'OCF_ERR_GENERIC' (exit code 1). Delete the file to resume normal operation.</key>
</language> </language>
<!-- 日本語 --> <!-- 日本語 -->
<language name="jp" long_name="日本語" description="Anvil! language file."> <language name="jp" long_name="日本語" description="Anvil! language file.">

@ -75,8 +75,10 @@ if ($anvil->data->{switches}{'job-uuid'})
message => "message_0217", message => "message_0217",
}); });
# Are we in an Anvil! system? # Are we in an Anvil! system or are we a DR host?
if (not $anvil->data->{sys}{anvil_uuid}) my $host_type = $anvil->Get->host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
if ((not $anvil->data->{sys}{anvil_uuid}) && ($host_type ne "dr"))
{ {
# We're not in an Anvil!. # We're not in an Anvil!.
if ($anvil->data->{switches}{'job-uuid'}) if ($anvil->data->{switches}{'job-uuid'})
@ -135,6 +137,32 @@ sub run_jobs
server_name => $server_name, server_name => $server_name,
}}); }});
# Before we start, we need to know if this server is on DR hosts. To do this, we'll parse the DRBD
# config file and look for DR hosts.
$anvil->Database->get_hosts({debug => 2});
$anvil->DRBD->gather_data({debug => 2});
foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{peer}})
{
my $peer_host_uuid = $anvil->Database->get_host_uuid_from_string({debug => 2, string => $peer});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
peer => $peer,
peer_host_uuid => $peer_host_uuid,
}});
if (($peer_host_uuid) && (exists $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}))
{
my $host_type = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{host_type};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
if ($host_type eq "dr")
{
$anvil->data->{dr_hosts}{$peer_host_uuid} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"dr_hosts::${peer_host_uuid}" => $anvil->data->{dr_hosts}{$peer_host_uuid},
}});
}
}
}
if (not $anvil->data->{job}{peer_mode}) if (not $anvil->data->{job}{peer_mode})
{ {
# Remove the server from pacemaker (stopping it, if necessary). # Remove the server from pacemaker (stopping it, if necessary).
@ -166,7 +194,6 @@ sub run_jobs
job_status => "failed", job_status => "failed",
}); });
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0228", variables => { resource => $server_name }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0228", variables => { resource => $server_name }});
$anvil->nice_exit({exit_code => 1});
} }
$anvil->Job->update_progress({ $anvil->Job->update_progress({
@ -360,11 +387,15 @@ sub remove_from_pacemaker
# We're node 2 # We're node 2
push @{$peers}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid} push @{$peers}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}
} }
if ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid})
if (exists $anvil->data->{dr_hosts})
{ {
# There's a DR host. foreach my $peer_host_uuid (keys %{$anvil->data->{dr_hosts}})
push @{$peers}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; {
push @{$peers}, $peer_host_uuid;
}
} }
my $progress = 30; my $progress = 30;
foreach my $host_uuid (@{$peers}) foreach my $host_uuid (@{$peers})
{ {

@ -2339,11 +2339,9 @@ sub load_job
# Load the anvil # Load the anvil
$anvil->data->{sys}{node1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; $anvil->data->{sys}{node1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
$anvil->data->{sys}{node2_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; $anvil->data->{sys}{node2_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
$anvil->data->{sys}{dr1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"sys::node1_host_uuid" => $anvil->data->{sys}{node1_host_uuid}, "sys::node1_host_uuid" => $anvil->data->{sys}{node1_host_uuid},
"sys::node2_host_uuid" => $anvil->data->{sys}{node2_host_uuid}, "sys::node2_host_uuid" => $anvil->data->{sys}{node2_host_uuid},
"sys::dr1_host_uuid" => $anvil->data->{sys}{dr1_host_uuid},
}}); }});
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0075,!!machine!".$anvil->data->{sys}{machine}."!!,!!manifest_uuid!".$anvil->data->{sys}{manifest_uuid}."!!"); update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0075,!!machine!".$anvil->data->{sys}{machine}."!!,!!manifest_uuid!".$anvil->data->{sys}{manifest_uuid}."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0075", variables => { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0075", variables => {

@ -242,7 +242,7 @@ ORDER BY
# Do we have a file_location_uuid? If not, there will be soon but nothing to do until # Do we have a file_location_uuid? If not, there will be soon but nothing to do until
# then. # then.
my $file_location_uuid = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; my $file_location_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
next if not $file_location_uuid; next if not $file_location_uuid;

@ -112,8 +112,6 @@ sub process_interactive
$anvil->data->{target_server}{anvil_node1_host_name} = "" if not exists $anvil->data->{target_server}{anvil_node1_host_name}; $anvil->data->{target_server}{anvil_node1_host_name} = "" if not exists $anvil->data->{target_server}{anvil_node1_host_name};
$anvil->data->{target_server}{anvil_node2_host_uuid} = "" if not exists $anvil->data->{target_server}{anvil_node2_host_uuid}; $anvil->data->{target_server}{anvil_node2_host_uuid} = "" if not exists $anvil->data->{target_server}{anvil_node2_host_uuid};
$anvil->data->{target_server}{anvil_node2_host_name} = "" if not exists $anvil->data->{target_server}{anvil_node2_host_name}; $anvil->data->{target_server}{anvil_node2_host_name} = "" if not exists $anvil->data->{target_server}{anvil_node2_host_name};
$anvil->data->{target_server}{anvil_dr1_host_uuid} = "" if not exists $anvil->data->{target_server}{anvil_dr1_host_uuid};
$anvil->data->{target_server}{anvil_dr1_host_name} = "" if not exists $anvil->data->{target_server}{anvil_dr1_host_name};
$anvil->data->{new_config}{cpu}{sockets} = ""; $anvil->data->{new_config}{cpu}{sockets} = "";
@ -216,8 +214,6 @@ sub interactive_question
$anvil->data->{target_server}{anvil_node1_host_name} = "" if not defined $anvil->data->{target_server}{anvil_node1_host_name}; $anvil->data->{target_server}{anvil_node1_host_name} = "" if not defined $anvil->data->{target_server}{anvil_node1_host_name};
$anvil->data->{target_server}{anvil_node2_host_uuid} = "" if not defined $anvil->data->{target_server}{anvil_node2_host_uuid}; $anvil->data->{target_server}{anvil_node2_host_uuid} = "" if not defined $anvil->data->{target_server}{anvil_node2_host_uuid};
$anvil->data->{target_server}{anvil_node2_host_name} = "" if not defined $anvil->data->{target_server}{anvil_node2_host_name}; $anvil->data->{target_server}{anvil_node2_host_name} = "" if not defined $anvil->data->{target_server}{anvil_node2_host_name};
$anvil->data->{target_server}{anvil_dr1_host_uuid} = "" if not defined $anvil->data->{target_server}{anvil_dr1_host_uuid};
$anvil->data->{target_server}{anvil_dr1_host_name} = "" if not defined $anvil->data->{target_server}{anvil_dr1_host_name};
### Anvil ### Anvil
# If 'switches::anvil' is set, see if it's a UUID and then set either 'anvil-uuid' or 'anvil-name'. # If 'switches::anvil' is set, see if it's a UUID and then set either 'anvil-uuid' or 'anvil-name'.
@ -405,14 +401,12 @@ sub interactive_question
my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid}; my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
my $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
anvil_uuid => $anvil_uuid, anvil_uuid => $anvil_uuid,
server_name => $server_name, server_name => $server_name,
server_host_uuid => $server_host_uuid, server_host_uuid => $server_host_uuid,
node1_host_uuid => $node1_host_uuid, node1_host_uuid => $node1_host_uuid,
node2_host_uuid => $node2_host_uuid, node2_host_uuid => $node2_host_uuid,
dr1_host_uuid => $dr1_host_uuid,
}}); }});
$anvil->data->{target_server}{anvil_uuid} = $anvil_uuid; $anvil->data->{target_server}{anvil_uuid} = $anvil_uuid;
@ -424,8 +418,6 @@ sub interactive_question
$anvil->data->{target_server}{anvil_node1_host_name} = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name}; $anvil->data->{target_server}{anvil_node1_host_name} = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
$anvil->data->{target_server}{anvil_node2_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; $anvil->data->{target_server}{anvil_node2_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
$anvil->data->{target_server}{anvil_node2_host_name} = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name}; $anvil->data->{target_server}{anvil_node2_host_name} = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
$anvil->data->{target_server}{anvil_dr1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
$anvil->data->{target_server}{anvil_dr1_host_name} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"target_server::anvil_uuid" => $anvil->data->{target_server}{anvil_uuid}, "target_server::anvil_uuid" => $anvil->data->{target_server}{anvil_uuid},
"target_server::server_name" => $anvil->data->{target_server}{server_name}, "target_server::server_name" => $anvil->data->{target_server}{server_name},
@ -435,18 +427,7 @@ sub interactive_question
"target_server::anvil_node1_host_name" => $anvil->data->{target_server}{anvil_node1_host_name}, "target_server::anvil_node1_host_name" => $anvil->data->{target_server}{anvil_node1_host_name},
"target_server::anvil_node2_host_uuid" => $anvil->data->{target_server}{anvil_node2_host_uuid}, "target_server::anvil_node2_host_uuid" => $anvil->data->{target_server}{anvil_node2_host_uuid},
"target_server::anvil_node2_host_name" => $anvil->data->{target_server}{anvil_node2_host_name}, "target_server::anvil_node2_host_name" => $anvil->data->{target_server}{anvil_node2_host_name},
"target_server::anvil_dr1_host_uuid" => $anvil->data->{target_server}{anvil_dr1_host_uuid},
"target_server::anvil_dr1_host_name" => $anvil->data->{target_server}{anvil_dr1_host_name},
}}); }});
if ($dr1_host_uuid)
{
$anvil->data->{target_server}{anvil_dr1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
$anvil->data->{target_server}{anvil_dr1_host_name} = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name};;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"target_server::anvil_dr1_host_uuid" => $anvil->data->{target_server}{anvil_dr1_host_uuid},
"target_server::anvil_dr1_host_name" => $anvil->data->{target_server}{anvil_dr1_host_name},
}});
}
### Pull out details of the server. ### Pull out details of the server.
# How much RAM are we using and how much is configured? # How much RAM are we using and how much is configured?

@ -394,6 +394,12 @@ sub write_definition
$anvil->Database->get_servers(); $anvil->Database->get_servers();
$anvil->Database->get_server_definitions(); $anvil->Database->get_server_definitions();
if (not exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid})
{
sleep 1;
next;
}
my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid}; my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
my $server_definition = exists $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid} ? $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml} : ""; my $server_definition = exists $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid} ? $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition => $server_definition }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition => $server_definition }});
@ -2231,7 +2237,6 @@ sub interactive_ask_server_ram
} }
my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
my $dr1_host_uuid = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
print $anvil->Words->string({key => "job_0167", variables => { print $anvil->Words->string({key => "job_0167", variables => {
ram_available => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}, unit => "M"}).")", ram_available => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}, unit => "M"}).")",
@ -2534,8 +2539,8 @@ sub interactive_ask_server_install_media
my $retry = 0; my $retry = 0;
while(1) while(1)
{ {
$anvil->Database->get_files(); $anvil->Database->get_files({debug => 2});
$anvil->Database->get_file_locations(); $anvil->Database->get_file_locations({debug => 2});
my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid}; my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
@ -2550,7 +2555,7 @@ sub interactive_ask_server_install_media
my $file_size = $anvil->data->{files}{file_name}{$file_name}{file_size}; my $file_size = $anvil->data->{files}{file_name}{$file_name}{file_size};
my $file_type = $anvil->data->{files}{file_name}{$file_name}{file_type}; my $file_type = $anvil->data->{files}{file_name}{$file_name}{file_type};
my $file_path = $file_directory."/".$file_name; my $file_path = $file_directory."/".$file_name;
my $file_location_uuid = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; my $file_location_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_uuid => $file_uuid, file_uuid => $file_uuid,
@ -2650,7 +2655,7 @@ sub interactive_ask_server_driver_disc
my $file_size = $anvil->data->{files}{file_name}{$file_name}{file_size}; my $file_size = $anvil->data->{files}{file_name}{$file_name}{file_size};
my $file_type = $anvil->data->{files}{file_name}{$file_name}{file_type}; my $file_type = $anvil->data->{files}{file_name}{$file_name}{file_type};
my $file_path = $file_directory."/".$file_name; my $file_path = $file_directory."/".$file_name;
my $file_location_uuid = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; my $file_location_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_uuid => $file_uuid, file_uuid => $file_uuid,
@ -2690,7 +2695,6 @@ sub interactive_ask_server_driver_disc
} }
my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
my $dr1_host_uuid = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : "";
print $anvil->Words->string({key => "job_0182"})."\n"; print $anvil->Words->string({key => "job_0182"})."\n";
print $iso_list."\n"; print $iso_list."\n";

@ -733,31 +733,19 @@ sub process_pull_file
my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
my $anvil_dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
anvil_node1_host_uuid => $anvil_node1_host_uuid, anvil_node1_host_uuid => $anvil_node1_host_uuid,
anvil_node2_host_uuid => $anvil_node2_host_uuid, anvil_node2_host_uuid => $anvil_node2_host_uuid,
anvil_dr1_host_uuid => $anvil_dr1_host_uuid,
}}); }});
# Where we pull from will depend on which machine we are and how many strikers we have. If we have # Where we pull from will depend on which machine we are and how many strikers we have. If we have
# one Anvil!, node 1 and 2 download at the same time, and DR waits. If there are two strikers, Each # one Anvil!, sub-node 1 and 2 download at the same time. If there are two strikers, Each node will
# node will download from a different striker (if possible) and DR waits. If there are 3 or more # download from a different striker (if possible).
# Strikers, DR does not wait, and downloads from a different striker than the striker's nodes use.
my $i_am = "node1"; my $i_am = "node1";
if ($anvil->Get->host_uuid eq $anvil_node2_host_uuid) if ($anvil->Get->host_uuid eq $anvil_node2_host_uuid)
{ {
$i_am = "node2"; $i_am = "node2";
} }
elsif ($anvil->Get->host_uuid eq $anvil_dr1_host_uuid)
{
$i_am = "dr1";
# As we're DR, we'll likely be pinging the nodes to seee if they're up when we wait for them
# to finish jobs. As such, load their IPs into memory.
$anvil->Network->load_ips({debug => 2, host_uuid => $anvil_node1_host_uuid});
$anvil->Network->load_ips({debug => 2, host_uuid => $anvil_node2_host_uuid});
}
my $striker_count = @{$anvil->data->{target_strikers}}; my $striker_count = @{$anvil->data->{target_strikers}};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
@ -777,158 +765,26 @@ sub process_pull_file
$anvil->nice_exit({exit_code => 2}); $anvil->nice_exit({exit_code => 2});
} }
my $use = 0; my $use = 0;
my $dr_wait = 1;
if ($striker_count >= 3) if ($striker_count >= 3)
{ {
$dr_wait = 0;
if ($i_am eq "node1") { $use = 0; } if ($i_am eq "node1") { $use = 0; }
elsif ($i_am eq "node2") { $use = 1; } elsif ($i_am eq "node2") { $use = 1; }
elsif ($i_am eq "dr1") { $use = 2; } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'use' => $use }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
'use' => $use,
dr_wait => $dr_wait,
}});
} }
elsif ($striker_count == 2) elsif ($striker_count == 2)
{ {
# Two strikers, each node will use the other Striker, DR waits for both to be done. # Two strikers, each node will use the other Striker
if ($i_am eq "node1") { $use = 0; } if ($i_am eq "node1") { $use = 0; }
elsif ($i_am eq "node2") { $use = 1; } elsif ($i_am eq "node2") { $use = 1; }
elsif ($i_am eq "dr1") { $use = 1; } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'use' => $use }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
'use' => $use,
dr_wait => $dr_wait,
}});
} }
elsif ($striker_count == 1) elsif ($striker_count == 1)
{ {
# Only 1 Striker # Only 1 Striker
if ($i_am eq "node1") { $use = 0; } if ($i_am eq "node1") { $use = 0; }
elsif ($i_am eq "node2") { $use = 0; } elsif ($i_am eq "node2") { $use = 0; }
elsif ($i_am eq "dr1") { $use = 0; } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'use' => $use }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
'use' => $use,
dr_wait => $dr_wait,
}});
}
# If I'm DR and need to wait, look for jobs on node1 and node2 and wait until both are done (or is
# offline).
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
i_am => $i_am,
dr_wait => $dr_wait,
}});
if (($i_am eq "dr1") && ($dr_wait))
{
my $node1_job_uuid = "";
my $node1_online = 1;
my $node2_job_uuid = "";
my $node2_online = 1;
my $node1_waiting = 1;
my $node2_waiting = 1;
my $query = "
SELECT
job_uuid,
job_host_uuid
FROM
jobs
WHERE
job_name = 'storage::pull_file'
AND
(
job_host_uuid = ".$anvil->Database->quote($anvil_node1_host_uuid)."
OR
job_host_uuid = ".$anvil->Database->quote($anvil_node2_host_uuid)."
)
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $job_uuid = $row->[0];
my $job_host_uuid = $row->[1];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
job_uuid => $job_uuid,
job_host_uuid => $job_host_uuid,
}});
if ($job_host_uuid eq $anvil_node1_host_uuid)
{
$node1_job_uuid = $job_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_job_uuid => $node1_job_uuid }});
}
if ($job_host_uuid eq $anvil_node2_host_uuid)
{
$node2_job_uuid = $job_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_job_uuid => $node2_job_uuid }});
}
}
my $waiting = 1;
while($waiting)
{
if ($node1_waiting)
{
if (not $node1_job_uuid)
{
$node1_waiting = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_waiting => $node1_waiting }});
}
else
{
$node1_waiting = wait_on_host($anvil, $anvil_node1_host_uuid, $node1_job_uuid);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_waiting => $node1_waiting }});
}
}
if ($node2_waiting)
{
if (not $node2_job_uuid)
{
$node2_waiting = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_waiting => $node2_waiting }});
}
else
{
$node2_waiting = wait_on_host($anvil, $anvil_node2_host_uuid, $node2_job_uuid);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_waiting => $node2_waiting }});
}
}
if ((not $node1_waiting) && (not $node2_waiting))
{
# We can proceed.
$waiting = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
}
else
{
# Sleep for a bit.
my $sleep_time = 30;
my $wait_until = $anvil->Get->date_and_time({offset => $sleep_time});
$anvil->Job->update_progress({
progress => 1,
message => "message_0199,!!strikers!".$striker_count."!!,!!node1_waiting!".$node1_waiting."!!,!!node2_waiting!".$node2_waiting."!!,!!wait_until!".$wait_until."!!",
job_status => "failed",
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0199", variables => {
strikers => $striker_count,
node1_waiting => $node1_waiting,
node2_waiting => $node2_waiting,
wait_until => $wait_until,
}});
sleep $sleep_time;
}
}
} }
# Now proceed with the download! # Now proceed with the download!

@ -75,7 +75,7 @@ sub striker_checks
### NOTE: Disabled until review complete ### NOTE: Disabled until review complete
# This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release) # This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release)
#update_audits($anvil); update_audits($anvil);
### NOTE: Disabled until review complete ### NOTE: Disabled until review complete
# This checks to make sure that the new dr_links table exists, and that existing anvil_dr1_host_uuid # This checks to make sure that the new dr_links table exists, and that existing anvil_dr1_host_uuid

@ -49,8 +49,7 @@ SELECT anv.anvil_password
FROM hosts AS hos FROM hosts AS hos
JOIN anvils AS anv JOIN anvils AS anv
ON hos.host_uuid = anv.anvil_node1_host_uuid ON hos.host_uuid = anv.anvil_node1_host_uuid
OR hos.host_uuid = anv.anvil_node2_host_uuid OR hos.host_uuid = anv.anvil_node2_host_uuid
OR hos.host_uuid = anv.anvil_dr1_host_uuid
WHERE hos.host_name = ".$anvil->Database->quote($target)." WHERE hos.host_name = ".$anvil->Database->quote($target)."
;"; ;";

@ -103,10 +103,6 @@ if ($anvil->data->{switches}{'anvil'})
$anvil_uuid = $anvil->data->{purge}{anvil_uuid}; $anvil_uuid = $anvil->data->{purge}{anvil_uuid};
push @{$anvil->data->{purge}{hosts}}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; push @{$anvil->data->{purge}{hosts}}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
push @{$anvil->data->{purge}{hosts}}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; push @{$anvil->data->{purge}{hosts}}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
if ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid})
{
push @{$anvil->data->{purge}{hosts}}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
}
} }
elsif($anvil->data->{switches}{'host'}) elsif($anvil->data->{switches}{'host'})
{ {
@ -198,14 +194,6 @@ else
}}); }});
# DR, if set. # DR, if set.
if ($anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid})
{
my $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0244", variables => {
host_name => $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name},
host_uuid => $dr1_host_uuid,
}});
}
} }
else else
{ {
@ -261,10 +249,6 @@ foreach my $host_uuid (@{$anvil->data->{purge}{hosts}})
{ {
$host_key = "anvil_node2_host_uuid"; $host_key = "anvil_node2_host_uuid";
} }
elsif ($host_uuid eq $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid})
{
$host_key = "anvil_dr1_host_uuid";
}
if ($host_key) if ($host_key)
{ {
my $query = " my $query = "

Loading…
Cancel
Save