Merge pull request #335 from ClusterLabs/anvil-tools-dev

Anvil tools dev
main
Digimer 2 years ago committed by GitHub
commit 58371d22b6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      Anvil/Tools.pm
  2. 97
      Anvil/Tools/Cluster.pm
  3. 1
      Anvil/Tools/Convert.pm
  4. 39
      Anvil/Tools/DRBD.pm
  5. 28
      Anvil/Tools/Database.pm
  6. 3
      ocf/alteeve/server
  7. 2
      scancore-agents/scan-cluster/scan-cluster
  8. 129
      scancore-agents/scan-drbd/scan-drbd
  9. 2
      scancore-agents/scan-drbd/scan-drbd.xml
  10. 113
      scancore-agents/scan-lvm/scan-lvm
  11. 3
      scancore-agents/scan-lvm/scan-lvm.xml
  12. 4
      scancore-agents/scan-server/scan-server.xml
  13. 4
      share/words.xml
  14. 1
      tools/Makefile.am
  15. 161
      tools/anvil-daemon
  16. 21
      tools/anvil-delete-server
  17. 38
      tools/anvil-provision-server
  18. 186
      tools/anvil-watch-power

@ -1095,6 +1095,7 @@ sub _set_paths
bonds => "/proc/net/bonding",
'cgi-bin' => "/var/www/cgi-bin",
drbd_resources => "/etc/drbd.d/",
drbd_kernel_proc => "/sys/kernel/debug/drbd/resources",
fence_agents => "/usr/sbin",
firewalld_services => "/usr/lib/firewalld/services",
firewalld_zones_etc => "/etc/firewalld/zones", # Changes when firewall-cmd ... --permanent is used.

@ -226,7 +226,7 @@ sub add_server
my $host_name = $anvil->Get->host_name();
my $server_state = defined $anvil->data->{server}{location}{$server_name}{status} ? $anvil->data->{server}{location}{$server_name}{status} : "";
my $server_host = defined $anvil->data->{server}{location}{$server_name}{host_name} ? $anvil->data->{server}{location}{$server_name}{host_name} : "";
my $target_role = $server_state eq "running" ? "started" : "stopped";
my $target_role = $server_host ? "started" : "stopped"; # Don't use state as it could be 'paused' if caught during initialization.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
host_name => $host_name,
server_state => $server_state,
@ -241,26 +241,91 @@ sub add_server
###
### TODO: If the target_role is 'started' because the server was running, we may need to later do an
### update to set it to 'stopped' after we've verified it's in the cluster below.
my $resource_command = $anvil->data->{path}{exe}{pcs}." resource create ".$server_name." ocf:alteeve:server name=\"".$server_name."\" meta allow-migrate=\"true\" target-role=\"".$target_role."\" op monitor interval=\"60\" start timeout=\"60\" on-fail=\"block\" stop timeout=\"300\" on-fail=\"block\" migrate_to timeout=\"600\" on-fail=\"block\" migrate_from timeout=\"600\" on-fail=\"block\"";
my $pcs_file = "/tmp/anvil.add_server.".$server_name.".cib";
my $pcs_cib_file_command = $anvil->data->{path}{exe}{pcs}." cluster cib ".$pcs_file;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { pcs_cib_file_command => $pcs_cib_file_command }});
my ($output, $return_code) = $anvil->System->call({shell_call => $pcs_cib_file_command});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
output => $output,
return_code => $return_code,
}});
undef $output;
undef $return_code;
my $resource_command = $anvil->data->{path}{exe}{pcs}." -f ".$pcs_file." resource create ".$server_name." ocf:alteeve:server name=\"".$server_name."\" meta allow-migrate=\"true\" target-role=\"".$target_role."\" op monitor interval=\"60\" start timeout=\"60\" on-fail=\"block\" stop timeout=\"300\" on-fail=\"block\" migrate_to timeout=\"600\" on-fail=\"block\" migrate_from timeout=\"600\" on-fail=\"block\"";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { resource_command => $resource_command }});
my ($output, $return_code) = $anvil->System->call({shell_call => $resource_command});
($output, $return_code) = $anvil->System->call({shell_call => $resource_command});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
output => $output,
return_code => $return_code,
}});
my $constraint_command = $anvil->data->{path}{exe}{pcs}." constraint location ".$server_name." prefers ";
if (($server_state eq "running") && ($server_host ne $host_name))
### NOTE: The higher the constraint score, the more preferred the host is.
# Which sub-node do we want to run the server on?
my $run_on_host_name = "";
my $backup_host_name = "";
my $target_host_uuid = $anvil->Cluster->get_primary_host_uuid({debug => 2, anvil_uuid => $anvil_uuid});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_uuid => $target_host_uuid }});
if ($target_role eq "started")
{
# Run on the current host.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
server_host => $server_host,
host_name => $host_name,
}});
if ($server_host eq $host_name)
{
# Run here
$run_on_host_name = $local_name;
$backup_host_name = $peer_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
run_on_host_name => $run_on_host_name,
backup_host_name => $backup_host_name,
}});
}
else
{
# Run on the
$run_on_host_name = $peer_name;
$backup_host_name = $local_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
run_on_host_name => $run_on_host_name,
backup_host_name => $backup_host_name,
}});
}
}
else
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
target_host_uuid => $target_host_uuid,
peer_host_uuid => $peer_host_uuid,
}});
if ($target_host_uuid eq $peer_host_uuid)
{
# Set the peer as primary.
$constraint_command .= $local_name."=100 ".$peer_name."=200";
# Run on the
$run_on_host_name = $peer_name;
$backup_host_name = $local_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
run_on_host_name => $run_on_host_name,
backup_host_name => $backup_host_name,
}});
}
else
{
# Set us as primary.
$constraint_command .= $local_name."=200 ".$peer_name."=100";
# Run here
$run_on_host_name = $local_name;
$backup_host_name = $peer_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
run_on_host_name => $run_on_host_name,
backup_host_name => $backup_host_name,
}});
}
}
my $constraint_command = $anvil->data->{path}{exe}{pcs}." -f ".$pcs_file." constraint location ".$server_name." prefers ".$run_on_host_name."=200 ".$backup_host_name."=100";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { constraint_command => $constraint_command }});
undef $output;
@ -271,6 +336,19 @@ sub add_server
return_code => $return_code,
}});
# Commit
my $commit_command = $anvil->data->{path}{exe}{pcs}." cluster cib-push ".$pcs_file;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { commit_command => $commit_command }});
($output, $return_code) = $anvil->System->call({shell_call => $commit_command});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
output => $output,
return_code => $return_code,
}});
# Unlink the temp CIB
unlink $pcs_file;
# Reload the CIB
($problem) = $anvil->Cluster->parse_cib({debug => 2});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }});
@ -4698,7 +4776,6 @@ sub _set_server_constraint
if ($problem)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0145", variables => { server => $server }});
}
}

@ -1006,6 +1006,7 @@ sub human_readable_to_bytes
}
# Start cleaning up the variables.
$size =~ s/\(.*?\)//; # Sometimes, the bytes with the human readible size in brackets is passed.
my $value = $size;
$size =~ s/ //g;
$type =~ s/ //g;

@ -193,7 +193,11 @@ sub allow_two_primaries
target => $target,
});
}
if ($anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{'peer-node-id'} =~ /^\d+$/)
if ((exists $anvil->data->{drbd}{status}{$host}) &&
(exists $anvil->data->{drbd}{status}{$host}{resource}{$resource}) &&
(exists $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}) &&
(defined $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{'peer-node-id'}) &&
($anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{'peer-node-id'} =~ /^\d+$/))
{
$target_node_id = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{'peer-node-id'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { target_node_id => $target_node_id }});
@ -206,7 +210,7 @@ sub allow_two_primaries
}
my $key = $set_to eq "yes" ? "log_0350" : "log_0642";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 1, key => "", variables => {
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 1, key => $key, variables => {
resource => $resource,
target_name => $peer_name,
target_node_id => $target_node_id,
@ -676,6 +680,21 @@ sub delete_resource
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0134", variables => { file_path => $definition_file }});
}
# Call scan-lvm and scan-drbd to make sure the databases are updated.
foreach my $agent ("scan-drbd", "scan-lvm")
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0740", variables => { agent => $agent }});
my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/".$agent."/".$agent.$anvil->Log->switches();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
output => $output,
return_code => $return_code,
}});
}
return(0);
}
@ -2487,11 +2506,20 @@ sub manage_resource
### TODO: When taking down a resource, check to see if any machine is SyncTarget and take it/them
### down first. See anvil-rename-server -> verify_server_is_off() for the logic.
### TODO: Sanity check the resource name and task requested.
### NOTE: For an unknown reason, sometimes a resource is left with allow-two-primary enabled. This
### can block startup, so to be safe, during start, we'll call adjust
### NOTE: If a live-migration fails, one of the nodes could have their allow-two-primaries left up.
### This ensures that they're set to 'no' before connecting.
if ($task eq "up")
{
# This generally brings up the resource
my ($return) = $anvil->DRBD->allow_two_primaries({
debug => 2,
resource => $resource,
set_to => "no",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'return' => $return }});
# Now call an adjust to make sure all other config details are loaded. It also up's the
# resource.
my $shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$resource;
my $output = "";
my $return_code = 255;
@ -2523,6 +2551,9 @@ sub manage_resource
return_code => $return_code,
}});
}
# Sleep for a moment to make sure adjust has taken hold.
sleep 1;
}
# If we 'adjust'ed above, this will likely complain that the backing disk already exists, and that's

@ -4648,6 +4648,10 @@ sub get_jobs
{
delete $anvil->data->{jobs}{running};
}
if (exists $anvil->data->{jobs}{modified_date})
{
delete $anvil->data->{jobs}{modified_date};
}
my $query = "
SELECT
@ -4757,6 +4761,30 @@ WHERE
"jobs::running::${job_uuid}::job_status" => $anvil->data->{jobs}{running}{$job_uuid}{job_status},
"jobs::running::${job_uuid}::modified_date" => $anvil->data->{jobs}{running}{$job_uuid}{modified_date},
}});
# Make it possible to sort by modified date for serial execution of similar jobs.
$anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_command} = $job_command;
$anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_data} = $job_data;
$anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_by} = $job_picked_up_by;
$anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_at} = $job_picked_up_at;
$anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_updated} = $job_updated;
$anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_name} = $job_name;
$anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_progress} = $job_progress;
$anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_title} = $job_title;
$anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_description} = $job_description;
$anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_status} = $job_status;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_command" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_command},
"jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_data" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_data},
"jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_picked_up_by" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_by},
"jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_picked_up_at" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_at},
"jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_updated" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_updated},
"jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_name" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_name},
"jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_progress" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_progress},
"jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_title" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_title},
"jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_description" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_description},
"jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_status" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_status},
}});
}
my $return_count = @{$return};

@ -620,11 +620,12 @@ sub start_server
local_name => $local_name,
peer_name => $peer_name,
}});
$anvil->Cluster->_set_server_constraint({
my $problem = $anvil->Cluster->_set_server_constraint({
debug => 2,
server => $server,
preferred_node => $local_name,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
# Success!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0309", variables => { server => $server }});

@ -298,7 +298,7 @@ sub check_resources
if ($update)
{
# Update the resource config.
my $resource_command = $anvil->data->{path}{exe}{pcs}." resource update ".$server." ocf:alteeve:server name=\"".$server."\" meta allow-migrate=\"true\" op monitor interval=\"60\" start timeout=\"60\" on-fail=\"block\" stop timeout=\"300\" on-fail=\"block\" migrate_to timeout=\"600\" on-fail=\"block\" migrate_from timeout=\"600\" on-fail=\"block\"";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0739", variables => {
server => $server,

@ -100,6 +100,8 @@ find_changes($anvil);
check_config($anvil);
fix_things($anvil);
# Shut down.
$anvil->ScanCore->agent_shutdown({agent => $THIS_FILE});
@ -108,6 +110,133 @@ $anvil->ScanCore->agent_shutdown({agent => $THIS_FILE});
# Functions #
#############################################################################################################
# This looks for issues that we can fix, and if any are found, we try to fix them.
sub fix_things
{
my ($anvil) = @_;
# If the cluster is up and both nodes are online, make sure all DRBD resources are connected.
$anvil->Database->get_hosts();
$anvil->DRBD->get_status();
my $host_uuid = $anvil->Get->host_uuid();
my $host_type = $anvil->Get->host_type();
my $short_host_name = $anvil->Get->short_host_name();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
host_uuid => $host_uuid,
host_type => $host_type,
short_host_name => $short_host_name,
}});
if ($host_type eq "node")
{
my $problem = $anvil->Cluster->parse_cib({debug => 3});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
problem => $problem,
"cib::parsed::local::ready" => $anvil->data->{cib}{parsed}{'local'}{ready},
"cib::parsed::peer::ready" => $anvil->data->{cib}{parsed}{peer}{ready},
}});
if ((not $problem) && ($anvil->data->{cib}{parsed}{peer}{ready}) && ($anvil->data->{cib}{parsed}{'local'}{ready}))
{
# Walk through all resources and make sure the peer and local disks are attached and
# the connection established.
foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource => $resource }});
# Check connection states.
foreach my $local_name (sort {$a cmp $b} keys%{$anvil->data->{drbd}{status}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { local_name => $local_name }});
foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{drbd}{status}{$local_name}{resource}{$resource}{connection}})
{
my $connection_state = lc($anvil->data->{drbd}{status}{$local_name}{resource}{$resource}{connection}{$peer}{'connection-state'});
my $peer_host_uuid = $anvil->Get->host_uuid_from_name({host_name => $peer, debug => 3});
my $peer_host_type = exists $anvil->data->{hosts}{host_uuid}{$host_uuid} ? $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:peer' => $peer,
's2:connection_state' => $connection_state,
's3:peer_host_uuid' => $peer_host_uuid,
's4:peer_host_type' => $peer_host_type,
}});
if (($peer_host_type eq "node") && ($connection_state eq "standalone"))
{
# Are either of the nodes still 'allow-two-primary',
# set it to 'no' (no migration is happening anyway,
# if we're StandAlone), and try reconnecting.
my $variables = {
resource => $resource,
peer => $peer,
};
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_drbd_message_0037", variables => $variables});
$anvil->Alert->register({alert_level => "notice", message => "scan_drbd_message_0037", variables => $variables, set_by => $THIS_FILE, sort_position => $anvil->data->{'scan-drbd'}{alert_sort}++});
$anvil->DRBD->manage_resource({
debug => 2,
resource => $resource,
task => "up",
});
}
}
}
foreach my $volume (sort {$a <=> $b} keys %{$anvil->data->{new}{resource}{$resource}{volume}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}})
{
my $peer_host_uuid = $anvil->Get->host_uuid_from_name({host_name => $peer});
my $peer_host_type = exists $anvil->data->{hosts}{host_uuid}{$host_uuid} ? $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:peer' => $peer,
's2:peer_host_uuid' => $peer_host_uuid,
's3:peer_host_type' => $peer_host_type,
}});
if ($peer_host_type eq "node")
{
my $connection_state = $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{connection_state};
my $local_role = $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{local_role};
my $peer_role = $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{peer_role};
my $local_disk_state = $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{local_disk_state};
my $peer_disk_state = $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{peer_disk_state};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:connection_state' => $connection_state,
's2:local_role' => $local_role,
's3:peer_role' => $peer_role,
's4:local_disk_state' => $local_disk_state,
's5:peer_disk_state' => $peer_disk_state,
}});
# Find the peer's host_uuid and make sure we're
# talking to a node, not a DR host.
if ($local_disk_state eq "down")
{
# Bring it up.
my $variables = {
resource => $resource,
volume => $volume,
peer => $peer,
};
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_drbd_message_0036", variables => $variables});
$anvil->Alert->register({alert_level => "notice", message => "scan_drbd_message_0036", variables => $variables, set_by => $THIS_FILE, sort_position => $anvil->data->{'scan-drbd'}{alert_sort}++});
$anvil->DRBD->manage_resource({
debug => 2,
resource => $resource,
task => "up",
});
}
}
}
}
}
}
}
return(0);
}
# In the early days, scan_drbd_peers -> scan_drbd_peer_tcp_port was type numeric. This wasn't compatible with
# drbd-proxy and had to be changed to type text to support csv port lists.
sub check_schema

@ -155,6 +155,8 @@ The global common configuration file: [#!variable!file!#] needs to be updated. T
====
</key>
<key name="scan_drbd_message_0035">The DRBD peer: [#!variable!peer_name!#] resource: [#!variable!resource_name!#] volume: [#!variable!volume_number!#] was deleted, and the corresponding LVM data no longer exists. Purging this resource's data.</key>
<key name="scan_drbd_message_0036">The cluster is up and both nodes are ready, yet the DRBD resource/volume: [#!variable!resource!#/#!variable!volume!#] is down. Bringing it up now.</key>
<key name="scan_drbd_message_0037">The cluster is up and both nodes are ready, yet the DRBD resource/volume: [#!variable!resource!#] is StandAlone. Adjusting and reconnecting now.</key>
<!-- Units -->
<key name="scan_drbd_unit_0001">Enabled</key>

@ -1000,9 +1000,35 @@ WHERE
{
# We've got an entry in the 'scan_lvm_pv' table, so now we'll look for data in the node and
# services tables.
my $scan_lvm_pv_uuid = $row->[0];
my $scan_lvm_pv_internal_uuid = $row->[1];
my $scan_lvm_pv_name = $row->[2];
$anvil->data->{sql}{scan_lvm_pvs}{scan_lvm_pv_internal_uuid}{$scan_lvm_pv_internal_uuid}{scan_lvm_pv_uuid} = $row->[0];;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
scan_lvm_pv_uuid => $scan_lvm_pv_uuid,
scan_lvm_pv_internal_uuid => $scan_lvm_pv_internal_uuid,
scan_lvm_pv_name => $scan_lvm_pv_name,
}});
# If there's an existing entry, delete this one, it's somehow a duplicate.
if (exists $anvil->data->{sql}{scan_lvm_pvs}{scan_lvm_pv_internal_uuid}{$scan_lvm_pv_internal_uuid})
{
# Duplicate!
my $variables = {
pv_name => $scan_lvm_pv_name,
pv_uuid => $scan_lvm_pv_uuid,
internal_uuid => $scan_lvm_pv_internal_uuid,
};
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_lvm_alert_0033", variables => $variables});
$anvil->Alert->register({alert_level => "notice", message => "scan_lvm_alert_0033", variables => $variables, set_by => $THIS_FILE});
my $query = "DELETE FROM scan_lvm_pvs WHERE scan_lvm_pv_uuid = ".$anvil->Database->quote($scan_lvm_pv_uuid).";\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
next;
}
# Save
$anvil->data->{sql}{scan_lvm_pvs}{scan_lvm_pv_internal_uuid}{$scan_lvm_pv_internal_uuid}{scan_lvm_pv_uuid} = $scan_lvm_pv_uuid;
$anvil->data->{sql}{scan_lvm_pvs}{scan_lvm_pv_internal_uuid}{$scan_lvm_pv_internal_uuid}{scan_lvm_pv_name} = $scan_lvm_pv_name;
$anvil->data->{sql}{scan_lvm_pvs}{scan_lvm_pv_internal_uuid}{$scan_lvm_pv_internal_uuid}{scan_lvm_pv_used_by_vg} = $row->[3];
$anvil->data->{sql}{scan_lvm_pvs}{scan_lvm_pv_internal_uuid}{$scan_lvm_pv_internal_uuid}{scan_lvm_pv_attributes} = $row->[4];
@ -1049,9 +1075,35 @@ WHERE
{
# We've got an entry in the 'scan_lvm_vg' table, so now we'll look for data in the node and
# services tables.
my $scan_lvm_vg_uuid = $row->[0];
my $scan_lvm_vg_internal_uuid = $row->[1];
my $scan_lvm_vg_name = $row->[2];
$anvil->data->{sql}{scan_lvm_vgs}{scan_lvm_vg_internal_uuid}{$scan_lvm_vg_internal_uuid}{scan_lvm_vg_uuid} = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
scan_lvm_vg_uuid => $scan_lvm_vg_uuid,
scan_lvm_vg_internal_uuid => $scan_lvm_vg_internal_uuid,
scan_lvm_vg_name => $scan_lvm_vg_name,
}});
# If there's an existing entry, delete this one, it's somehow a duplicate.
if (exists $anvil->data->{sql}{scan_lvm_vgs}{scan_lvm_vg_internal_uuid}{$scan_lvm_vg_internal_uuid})
{
# Duplicate!
my $variables = {
vg_name => $scan_lvm_vg_name,
vg_uuid => $scan_lvm_vg_uuid,
internal_uuid => $scan_lvm_vg_internal_uuid,
};
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_lvm_alert_0034", variables => $variables});
$anvil->Alert->register({alert_level => "notice", message => "scan_lvm_alert_0034", variables => $variables, set_by => $THIS_FILE});
my $query = "DELETE FROM scan_lvm_vgs WHERE scan_lvm_vg_uuid = ".$anvil->Database->quote($scan_lvm_vg_uuid).";\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
next;
}
# Save
$anvil->data->{sql}{scan_lvm_vgs}{scan_lvm_vg_internal_uuid}{$scan_lvm_vg_internal_uuid}{scan_lvm_vg_uuid} = $scan_lvm_vg_uuid;
$anvil->data->{sql}{scan_lvm_vgs}{scan_lvm_vg_internal_uuid}{$scan_lvm_vg_internal_uuid}{scan_lvm_vg_name} = $scan_lvm_vg_name;
$anvil->data->{sql}{scan_lvm_vgs}{scan_lvm_vg_internal_uuid}{$scan_lvm_vg_internal_uuid}{scan_lvm_vg_attributes} = $row->[3];
$anvil->data->{sql}{scan_lvm_vgs}{scan_lvm_vg_internal_uuid}{$scan_lvm_vg_internal_uuid}{scan_lvm_vg_extent_size} = $row->[4];
@ -1097,9 +1149,64 @@ WHERE
{
# We've got an entry in the 'scan_lvm_lv' table, so now we'll look for data in the node and
# services tables.
my $scan_lvm_lv_uuid = $row->[0];
my $scan_lvm_lv_internal_uuid = $row->[1];
my $scan_lvm_lv_name = $row->[2];
$anvil->data->{sql}{scan_lvm_lvs}{scan_lvm_lv_internal_uuid}{$scan_lvm_lv_internal_uuid}{scan_lvm_lv_uuid} = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
scan_lvm_lv_uuid => $scan_lvm_lv_uuid,
scan_lvm_lv_internal_uuid => $scan_lvm_lv_internal_uuid,
scan_lvm_lv_name => $scan_lvm_lv_name,
}});
# If there's an existing entry, delete this one, it's somehow a duplicate.
if (exists $anvil->data->{sql}{scan_lvm_lvs}{scan_lvm_lv_internal_uuid}{$scan_lvm_lv_internal_uuid})
{
# Duplicate! Is one of them DELETED?
my $existing_name = $anvil->data->{sql}{scan_lvm_lvs}{scan_lvm_lv_internal_uuid}{$scan_lvm_lv_internal_uuid}{scan_lvm_lv_name};
my $existing_uuid = $anvil->data->{sql}{scan_lvm_lvs}{scan_lvm_lv_internal_uuid}{$scan_lvm_lv_internal_uuid}{scan_lvm_lv_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
existing_name => $existing_name,
existing_uuid => $existing_uuid,
}});
if ($anvil->data->{sql}{scan_lvm_lvs}{scan_lvm_lv_internal_uuid}{$scan_lvm_lv_internal_uuid}{scan_lvm_lv_name} eq "DELETED")
{
# Delete the one we already read.
my $variables = {
lv_name => $existing_name,
lv_uuid => $existing_uuid,
internal_uuid => $scan_lvm_lv_internal_uuid,
};
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_lvm_alert_0035", variables => $variables});
$anvil->Alert->register({alert_level => "notice", message => "scan_lvm_alert_0034", variables => $variables, set_by => $THIS_FILE});
my $query = "DELETE FROM scan_lvm_lvs WHERE scan_lvm_lv_uuid = ".$anvil->Database->quote($existing_uuid).";\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
delete $anvil->data->{sql}{scan_lvm_lvs}{scan_lvm_lv_internal_uuid}{$scan_lvm_lv_internal_uuid};
}
else
{
# Delete the new one
my $variables = {
lv_name => $scan_lvm_lv_name,
lv_uuid => $scan_lvm_lv_uuid,
internal_uuid => $scan_lvm_lv_internal_uuid,
};
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_lvm_alert_0035", variables => $variables});
$anvil->Alert->register({alert_level => "notice", message => "scan_lvm_alert_0034", variables => $variables, set_by => $THIS_FILE});
my $query = "DELETE FROM scan_lvm_lvs WHERE scan_lvm_lv_uuid = ".$anvil->Database->quote($scan_lvm_lv_uuid).";\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
next;
}
}
# Save
$anvil->data->{sql}{scan_lvm_lvs}{scan_lvm_lv_internal_uuid}{$scan_lvm_lv_internal_uuid}{scan_lvm_lv_uuid} = $scan_lvm_lv_uuid;
$anvil->data->{sql}{scan_lvm_lvs}{scan_lvm_lv_internal_uuid}{$scan_lvm_lv_internal_uuid}{scan_lvm_lv_name} = $scan_lvm_lv_name;
$anvil->data->{sql}{scan_lvm_lvs}{scan_lvm_lv_internal_uuid}{$scan_lvm_lv_internal_uuid}{scan_lvm_lv_attributes} = $row->[3];
$anvil->data->{sql}{scan_lvm_lvs}{scan_lvm_lv_internal_uuid}{$scan_lvm_lv_internal_uuid}{scan_lvm_lv_on_vg} = $row->[4];

@ -220,6 +220,9 @@ The attribute bits are:
<key name="scan_lvm_alert_0030">The volume group: [#!variable!vg_name!#] (UUID: [#!variable!vg_uuid!#]) has vanished! If this is a removable device, like a USB drive, this is OK. Otherwise, it could be the sign of a disk failure.</key>
<key name="scan_lvm_alert_0031">The logical volume: [#!variable!lv_name!#] (UUID: [#!variable!lv_uuid!#]) has been lost. This is normal if the LV is on a removable medium like a USB disk. Otherwise, it could be the sign of a disk failure.</key>
<key name="scan_lvm_alert_0032">The physical volume: [#!variable!pv_name!#] (UUID: [#!variable!pv_uuid!#]) sector size changed from: [#!variable!old_pv_sector_size!#] (#!variable!old_pv_sector_size_bytes!# bytes) to: [#!variable!new_pv_sector_size!#] (#!variable!new_pv_sector_size_bytes!# bytes). This should not happen, unless the storage behind the PV was somehow rebuilt without recreating the LVM metadata.</key>
<key name="scan_lvm_alert_0033">The physical volume with the internal UUID: [#!variable!internal_uuid!#] (name: [#!variable!pv_name!#]) has a duplicate entry! Deleting the one with the scan_lvm_pv_uuid: [#!variable!pv_uuid!#].</key>
<key name="scan_lvm_alert_0034">The volume group with the internal UUID: [#!variable!internal_uuid!#] (name: [#!variable!vg_name!#]) has a duplicate entry! Deleting the one with the scan_lvm_vg_uuid: [#!variable!vg_uuid!#].</key>
<key name="scan_lvm_alert_0035">The logical volume with the internal UUID: [#!variable!internal_uuid!#] (name: [#!variable!lv_name!#]) has a duplicate entry! Deleting the one with the scan_lvm_lv_uuid: [#!variable!lv_uuid!#].</key>
<!-- Log entries -->
<key name="scan_lvm_log_0001">Starting: [#!variable!program!#].</key>

@ -77,9 +77,9 @@ A new server named: [#!variable!server!#] has been found. The definition XML is:
<key name="scan_server_alert_0011">The server: [#!variable!server!#] has is no longer marked as user-stopped.</key>
<key name="scan_server_alert_0012">The server: [#!variable!server!#] has moved from the node: [#!variable!old_host_name!#] to: [#!variable!new_host_name!#] (from: [#!variable!old_host_uuid!#] to: [#!variable!new_host_uuid!#]).</key>
<key name="scan_server_alert_0013">The amount of RAM the server: [#!variable!server!#] is currently using has changed from: [#!variable!old_ram_in_use_short!#] to: [#!variable!new_ram_in_use_short!#] (from: [#!variable!old_ram_in_use_bytes!# bytes] to: [#!variable!new_ram_in_use_bytes!# bytes]).</key>
<key name="scan_server_alert_0014">The amount of RAM configured for the server: [#!variable!server!#] has changed from: [#!variable!old_ram_in_use_short!#] to: [#!variable!new_ram_in_use_short!#] (from: [#!variable!old_ram_in_use_bytes!# bytes] to: [#!variable!new_ram_in_use_bytes!# bytes]).</key>
<key name="scan_server_alert_0014">The amount of RAM configured for the server: [#!variable!server!#] has changed from: [#!variable!old_configured_ram_short!#] to: [#!variable!new_configured_ram_short!#] (from: [#!variable!old_configured_ram_bytes!# bytes] to: [#!variable!new_configured_ram_bytes!# bytes]).</key>
<key name="scan_server_alert_0015">
The amount of RAM configured for the server: [#!variable!server!#] has changed from: [#!variable!old_ram_in_use_short!#] to: [#!variable!new_ram_in_use_short!#] (from: [#!variable!old_ram_in_use_bytes!# bytes] to: [#!variable!new_ram_in_use_bytes!# bytes]).
The amount of RAM configured for the server: [#!variable!server!#] has changed from: [#!variable!old_configured_ram_short!#] to: [#!variable!new_configured_ram_short!#] (from: [#!variable!old_configured_ram_bytes!# bytes] to: [#!variable!new_configured_ram_bytes!# bytes]).
- NOTE: This RAM will not be used until the server is power cycled!
</key>

@ -1549,6 +1549,7 @@ Note: This is a permanent action! If you protect this server again later, a full
<key name="job_0461"><![CDATA[ > File name: [#!variable!name!#], file UUID: [#!variable!uuid!#], size: [#!variable!size!#]]]></key>
<key name="job_0462"><![CDATA[ --driver-disc - (optional) A driver disc to be added as a second optical drive. Valid options are above.]]></key>
<key name="job_0463">Enabling the enable-safe-start daemon.</key>
<key name="job_0464">Calling select ScanCore scan agents to ensure the database is updated.</key>
<!-- Log entries -->
<key name="log_0001">Starting: [#!variable!program!#].</key>
@ -2390,6 +2391,9 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
<key name="log_0737">The job: [#!variable!command!#] (with job UUID: [#!variable!job_uuid!#]) is being skipped for now, already started a job (started job_uuid: [#!variable!started_job!#]) with this command on this loop.</key>
<key name="log_0738">There are no databases available at this time.</key>
<key name="log_0739">The server: [#!variable!server!#] needs it's pacemaker configuration updated. Running: [#!variable!command!#].</key>
<key name="log_0740">Running the scan-agent: [#!variable!agent!#] now to ensure that the database has an updated view of resources.</key>
<key name="log_0741">I was about to start: [#!variable!command!#] with the job UUID: [#!variable!this_job_uuid!#]. However, another job using the same command with the job UUID: [#!variable!other_job_uuid!#]. To avoid race conditions, only one process with a given command is run at the same time.</key>
<key name="log_0742">The job with the command: [#!variable!command!#] and job UUID: [#!variable!job_uuid!#] is restarting.</key>
<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>

@ -47,6 +47,7 @@ dist_sbin_SCRIPTS = \
anvil-update-system \
anvil-version-changes \
anvil-watch-bonds \
anvil-watch-power \
scancore \
striker-auto-initialize-all \
striker-boot-machine \

@ -1412,51 +1412,53 @@ sub run_jobs
# Get a list of pending or incomplete jobs.
my $ended_within = $startup ? 1 : 300;
my $return = $anvil->Database->get_jobs({ended_within => $ended_within});
my $count = @{$return};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
ended_within => $ended_within,
'return' => $return,
count => $count,
}});
foreach my $hash_ref (@{$return})
{
my $job_uuid = $hash_ref->{job_uuid};
my $job_command = $hash_ref->{job_command};
my $job_data = $hash_ref->{job_data};
my $job_picked_up_by = $hash_ref->{job_picked_up_by};
my $job_picked_up_at = $hash_ref->{job_picked_up_at};
my $job_updated = $hash_ref->{job_updated};
my $job_name = $hash_ref->{job_name};
my $job_progress = $hash_ref->{job_progress};
my $job_title = $hash_ref->{job_title};
my $job_description = $hash_ref->{job_description};
my $job_status = $hash_ref->{job_status};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { ended_within => $ended_within }});
$anvil->Database->get_jobs({ended_within => $ended_within});
foreach my $modified_date (sort {$a cmp $b} keys %{$anvil->data->{jobs}{modified_date}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { modified_date => $modified_date }});
foreach my $job_uuid (sort {$a cmp $b} keys %{$anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}})
{
# Reload the jobs so we get an updated view of them.
$anvil->Database->get_jobs({ended_within => $ended_within});
# Collect the data.
my $job_command = $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_command};
my $short_command = $job_command;
$short_command =~ s/\s.*$//;
my $job_data = $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_data};
my $job_picked_up_by = $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_by};
my $job_picked_up_at = $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_at};
my $job_updated = $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_updated};
my $job_name = $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_name};
my $job_progress = $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_progress};
my $job_title = $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_title};
my $job_description = $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_description};
my $job_status = $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_status};
my $started_seconds_ago = $job_picked_up_at ? (time - $job_picked_up_at) : 0;
my $updated_seconds_ago = $job_updated ? (time - $job_updated) : 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
job_uuid => $job_uuid,
job_command => $job_command,
job_data => $job_data,
job_picked_up_by => $job_picked_up_by,
job_picked_up_at => $job_picked_up_at,
job_updated => $job_updated,
job_name => $job_name,
job_progress => $job_progress,
job_title => $job_title,
job_description => $job_description,
job_status => $job_status,
started_seconds_ago => $started_seconds_ago,
updated_seconds_ago => $updated_seconds_ago,
's01:job_uuid' => $job_uuid,
's02:job_command' => $job_command,
's03:short_command' => $short_command,
's04:job_data' => $job_data,
's05:job_picked_up_by' => $job_picked_up_by,
's06:job_picked_up_at' => $job_picked_up_at,
's07:job_updated' => $job_updated,
's08:job_name' => $job_name,
's09:job_progress' => $job_progress,
's10:job_title' => $job_title,
's11:job_description' => $job_description,
's12:job_status' => $job_status,
's13:started_seconds_ago' => $started_seconds_ago,
's14:updated_seconds_ago' => $updated_seconds_ago,
}});
# To minimize the chance of race conditions, any given command will be called only once at a
# time. If two jobs of the same command exist, only one will be called.
# To minimize the chance of race conditions, any given command will be called only
# once at a time. If two jobs of the same command exist, only one will be called.
if ($job_progress != 100)
{
my $short_command = $job_command;
$short_command =~ s/\s.*$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { short_command => $short_command }});
if (exists $anvil->data->{sys}{started}{$short_command})
{
# Skip it.
@ -1482,7 +1484,19 @@ sub run_jobs
next;
}
if ($job_progress ne "100")
if ($job_progress == 100)
{
# This is a job that might have just completed, clear the started value.
$anvil->data->{jobs}{$job_uuid}{started} = 0;
$job_picked_up_at = 0;
$job_picked_up_by = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
job_picked_up_at => $job_picked_up_at,
job_picked_up_by => $job_picked_up_by,
"jobs::${job_uuid}::started" => $anvil->data->{jobs}{$job_uuid}{started},
}});
}
else
{
$anvil->data->{sys}{jobs_running} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::jobs_running" => $anvil->data->{sys}{jobs_running} }});
@ -1501,9 +1515,9 @@ sub run_jobs
# If the job is done, just clear the 'job_picked_up_by' and be done.
if ($job_progress ne "100")
{
# It's possible that the job updated to 100% and exited after we
# gathered the job data, so we won't restart until we've seen it not
# running and not at 100% after 5 loops.
# It's possible that the job updated to 100% and exited after
# we gathered the job data, so we won't restart until we've
# seen it not running and not at 100% after 5 loops.
if ((not exists $anvil->data->{lost_job_count}{$job_uuid}) or (not defined $anvil->data->{lost_job_count}{$job_uuid}))
{
$anvil->data->{lost_job_count}{$job_uuid} = 0;
@ -1511,7 +1525,8 @@ sub run_jobs
}
if ($anvil->data->{lost_job_count}{$job_uuid} > 5)
{
# The previous job is gone, but the job isn't finished. Start it again.
# The previous job is gone, but the job isn't
# finished. Start it again.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0007", variables => {
command => $job_command,
pid => $job_picked_up_by,
@ -1542,6 +1557,12 @@ sub run_jobs
$job_picked_up_by = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_picked_up_by => $job_picked_up_by }});
}
elsif ($job_progress ne "100")
{
# The job is running.
$anvil->data->{jobs_started}{$short_command} = $job_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "jobs_started::${short_command}" => $anvil->data->{jobs_started}{$short_command} }});
}
}
# Convert the double-banged strings into a proper message.
@ -1590,7 +1611,7 @@ sub run_jobs
$jobs_file .= $json_string.",\n";
# If the job is done, move on.
next if $job_progress eq "100";
next if $job_progress == 100;
next if $anvil->data->{switches}{'no-start'};
# If 'startup' is set, we only care if 'job_status' is 'anvil_startup'
@ -1604,9 +1625,54 @@ sub run_jobs
next;
}
# If the job is not running, start it.
# If the job is not running, and we've not started any other of the same command this
# loop, start it.
if (not $job_picked_up_by)
{
if (exists $anvil->data->{jobs_started}{$short_command})
{
# Is the job_uuid associated with this command done?
my $started_job_uuid = $anvil->data->{jobs_started}{$short_command};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { started_job_uuid => $started_job_uuid }});
if (exists $anvil->data->{jobs}{running}{$started_job_uuid})
{
# If the previously running job and this job have the same
# UUID, it failed and needs to restart.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
job_uuid => $job_uuid,
started_job_uuid => $started_job_uuid,
"jobs::running::${started_job_uuid}::job_progress" => $anvil->data->{jobs}{running}{$started_job_uuid}{job_progress},
}});
if ($started_job_uuid eq $job_uuid)
{
# We're restarting.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0741", variables => {
command => $job_command,
job_uuid => $job_uuid,
}});
}
elsif ($anvil->data->{jobs}{running}{$started_job_uuid}{job_progress} != 100)
{
# Don't start it in this pass.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0741", variables => {
command => $job_command,
this_job_uuid => $job_uuid,
other_job_uuid => $started_job_uuid,
}});
next;
}
else
{
# The previous job is done, delete it.
$anvil->data->{jobs_started}{$short_command} = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"jobs_started::${short_command}" => $anvil->data->{jobs_started}{$short_command},
}});
}
}
}
my $command = $job_command." --job-uuid ".$job_uuid;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0210", variables => { command => $command }});
@ -1648,6 +1714,11 @@ sub run_jobs
# Record that we've tried to start this job, so that we don't try to restart it for any reason for at least a minute.
$anvil->data->{jobs}{$job_uuid}{started} = time;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'jobs::$job_uuid::started' => $anvil->data->{jobs}{$job_uuid}{started} }});
# Record that a job with this command has started
$anvil->data->{jobs_started}{$short_command} = $job_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "jobs_started::${short_command}" => $anvil->data->{jobs_started}{$short_command} }});
}
}
}

@ -240,6 +240,27 @@ WHERE
unlink $resource_file;
}
# Call scan-cluster, scan-network and scan-server to make sure the databases are updated.
$anvil->Job->update_progress({
progress => 90,
message => "job_0464",
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0464"});
foreach my $agent ("scan-cluster", "scan-network", "scan-server")
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0740", variables => { agent => $agent }});
my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/".$agent."/".$agent.$anvil->Log->switches();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
}
# We're done
$anvil->Job->update_progress({
progress => 100,
message => "job_0216",

@ -324,6 +324,26 @@ sub run_jobs
# Add the server to the cluster.
add_server_to_cluster($anvil);
# Update the database by calling select scan agents
$anvil->Job->update_progress({
progress => 95,
message => "job_0464",
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0464"});
foreach my $agent ("scan-drbd", "scan-cluster", "scan-network", "scan-server")
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0740", variables => { agent => $agent }});
my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/".$agent."/".$agent.$anvil->Log->switches();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
}
# Done!
$anvil->Job->update_progress({
progress => 100,
@ -1408,6 +1428,24 @@ sub create_lv
$anvil->nice_exit({exit_code => 1});
}
# Call scan-lvm now to make sure we've recorded this LV.
$anvil->Job->update_progress({
progress => 28,
message => "job_0464",
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0464"});
my $agent = "scan-lvm";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0740", variables => { agent => $agent }});
$shell_call = $anvil->data->{path}{directories}{scan_agents}."/".$agent."/".$agent.$anvil->Log->switches();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
# Report
$anvil->Job->update_progress({
progress => 30,

@ -0,0 +1,186 @@
#!/usr/bin/perl
use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
use Text::Diff;
use Term::Cap;
use Time::Local;
$| = 1;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
my $anvil = Anvil::Tools->new();
# Get a list of all interfaces with IP addresses.
$anvil->Get->switches({debug => 3, list => []});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->data->{'say'}{yes} = $anvil->Words->string({key => "unit_0001"});
$anvil->data->{'say'}{'no'} = $anvil->Words->string({key => "unit_0002"});
$anvil->data->{'say'}{ups} = $anvil->Words->string({key => "header_0024"});
$anvil->data->{'say'}{ip_address} = $anvil->Words->string({key => "header_0025"});
$anvil->data->{'say'}{charge_percent} = $anvil->Words->string({key => "header_0107"});
$anvil->data->{'say'}{on_battery} = $anvil->Words->string({key => "header_0108"});
$anvil->data->{'say'}{estimated_runtime} = $anvil->Words->string({key => "header_0109"});
$anvil->data->{'say'}{last_updated} = $anvil->Words->string({key => "header_0110"});
my $t = Term::Cap->Tgetent;
while(1)
{
# Reload defaults, re-read the config and then connect to the database(s)
$anvil->refresh();
$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0132"});
if ($anvil->data->{sys}{database}{connections})
{
show_power_data($anvil);
}
else
{
# No databases available.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "log_0738"});
}
sleep 2;
}
$anvil->nice_exit({exit_code => 0});
sub show_power_data
{
my ($anvil) = @_;
$anvil->Database->get_upses({debug => 2});
my $longest_ups_name = length($anvil->data->{'say'}{ups});
my $longest_ip_address = length($anvil->data->{'say'}{ip_address});
my $longest_charge_percent = length($anvil->data->{'say'}{charge_percent});
my $longest_on_battery = length($anvil->data->{'say'}{on_battery});
my $longest_estimated_runtime = length($anvil->data->{'say'}{estimated_runtime});
my $longest_age_of_data = length($anvil->data->{'say'}{last_updated});
foreach my $ups_uuid (sort {$a cmp $b} keys %{$anvil->data->{upses}{ups_uuid}})
{
my $ups_name = $anvil->data->{upses}{ups_uuid}{$ups_uuid}{ups_name};
my $ups_agent = $anvil->data->{upses}{ups_uuid}{$ups_uuid}{ups_agent};
my $ups_ip_address = $anvil->data->{upses}{ups_uuid}{$ups_uuid}{ups_ip_address};
my $ups_modified_date = $anvil->data->{upses}{ups_uuid}{$ups_uuid}{modified_date_unix};
my $power_uuid = $anvil->data->{upses}{ups_uuid}{$ups_uuid}{power_uuid};
my $on_battery = $anvil->data->{power}{power_uuid}{$power_uuid}{power_on_battery};
my $seconds_left = $anvil->data->{power}{power_uuid}{$power_uuid}{power_seconds_left};
my $charge_percentage = $anvil->data->{power}{power_uuid}{$power_uuid}{power_charge_percentage};
my $power_modified_date = $anvil->data->{power}{power_uuid}{$power_uuid}{modified_date_unix};
my $last_updated = $power_modified_date > $ups_modified_date ? $power_modified_date : $ups_modified_date;
my $age_of_data = $anvil->Convert->time({'time' => (time - $last_updated), translate => 1});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's01:ups_uuid' => $ups_uuid,
's02:ups_name' => $ups_name,
's03:ups_ip_address' => $ups_ip_address,
's04:ups_modified_date' => $ups_modified_date,
's05:power_uuid' => $power_uuid,
's06:on_battery' => $on_battery,
's07:seconds_left' => $seconds_left,
's08:charge_percentage' => $charge_percentage,
's09:power_modified_date' => $power_modified_date,
's10:last_updated' => $last_updated,
's11:age_of_data' => $age_of_data,
}});
$anvil->data->{ups}{$ups_name}{ip_address} = $ups_ip_address;
$anvil->data->{ups}{$ups_name}{on_battery} = $on_battery ? $anvil->data->{'say'}{yes} : $anvil->data->{'say'}{'no'};
$anvil->data->{ups}{$ups_name}{estimated_runtime} = $anvil->Convert->time({'time' => $seconds_left, translate => 1});;
$anvil->data->{ups}{$ups_name}{charge_percentage} = $charge_percentage;
$anvil->data->{ups}{$ups_name}{age_of_data} = $age_of_data;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"s1:ups::${ups_name}::ip_address" => $anvil->data->{ups}{$ups_name}{ip_address},
"s2:ups::${ups_name}::on_battery" => $anvil->data->{ups}{$ups_name}{on_battery},
"s3:ups::${ups_name}::seconds_left" => $anvil->data->{ups}{$ups_name}{seconds_left},
"s4:ups::${ups_name}::charge_percentage" => $anvil->data->{ups}{$ups_name}{charge_percentage},
"s5:ups::${ups_name}::age_of_data" => $anvil->data->{ups}{$ups_name}{age_of_data},
}});
if (length($ups_name) > $longest_ups_name)
{
$longest_ups_name = length($ups_name);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_ups_name => $longest_ups_name }});
}
if (length($anvil->data->{ups}{$ups_name}{ip_address}) > $longest_ip_address)
{
$longest_ip_address = length($anvil->data->{ups}{$ups_name}{ip_address});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_ip_address => $longest_ip_address }});
}
if (length($anvil->data->{ups}{$ups_name}{charge_percent}) > $longest_charge_percent)
{
$longest_charge_percent = length($anvil->data->{ups}{$ups_name}{charge_percent});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_charge_percent => $longest_charge_percent }});
}
if (length($anvil->data->{ups}{$ups_name}{on_battery}) > $longest_on_battery)
{
$longest_on_battery = length($anvil->data->{ups}{$ups_name}{on_battery});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_on_battery => $longest_on_battery }});
}
if (length($anvil->data->{ups}{$ups_name}{estimated_runtime}) > $longest_estimated_runtime)
{
$longest_estimated_runtime = length($anvil->data->{ups}{$ups_name}{estimated_runtime});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_estimated_runtime => $longest_estimated_runtime }});
}
if (length($anvil->data->{ups}{$ups_name}{age_of_data}) > $longest_age_of_data)
{
$longest_age_of_data = length($anvil->data->{ups}{$ups_name}{age_of_data});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_age_of_data => $longest_age_of_data }});
}
}
system('clear');
print $t->Tgoto("cm", 0, 0);
my $header = "| ".sprintf("%-${longest_ups_name}s", $anvil->data->{'say'}{ups});
$header .= " | ".sprintf("%-${longest_ip_address}s", $anvil->data->{'say'}{ip_address});
$header .= " | ".sprintf("%-${longest_charge_percent}s", $anvil->data->{'say'}{charge_percent});
$header .= " | ".sprintf("%-${longest_on_battery}s", $anvil->data->{'say'}{on_battery});
$header .= " | ".sprintf("%-${longest_estimated_runtime}s", $anvil->data->{'say'}{estimated_runtime});
$header .= " | ".sprintf("%-${longest_age_of_data}s", $anvil->data->{'say'}{last_updated});
$header .= " |";
my $break_line = "+-".sprintf("%0${longest_ups_name}d", 0);
$break_line .= "-+-".sprintf("%0${longest_ip_address}d", 0);
$break_line .= "-+-".sprintf("%0${longest_charge_percent}d", 0);
$break_line .= "-+-".sprintf("%0${longest_on_battery}d", 0);
$break_line .= "-+-".sprintf("%0${longest_estimated_runtime}d", 0);
$break_line .= "-+-".sprintf("%0${longest_age_of_data}d", 0);
$break_line .= "-+";
$break_line =~ s/0/-/g;
print $anvil->Get->date_and_time()."\n";
print $break_line."\n";
print $header."\n";
print $break_line."\n";
foreach my $ups_name (sort {$a cmp $b} keys %{$anvil->data->{ups}})
{
my $ups_ip_address = $anvil->data->{ups}{$ups_name}{ip_address};
my $on_battery = $anvil->data->{ups}{$ups_name}{on_battery};
my $estimated_runtime = $anvil->data->{ups}{$ups_name}{estimated_runtime};
my $charge_percentage = $anvil->data->{ups}{$ups_name}{charge_percentage};
my $age_of_data = $anvil->data->{ups}{$ups_name}{age_of_data};
my $data_line = "| ".sprintf("%-${longest_ups_name}s", $ups_name);
$data_line .= " | ".sprintf("%-${longest_ip_address}s", $ups_ip_address);
$data_line .= " | ".sprintf("%-${longest_charge_percent}s", $charge_percentage);
$data_line .= " | ".sprintf("%-${longest_on_battery}s", $on_battery);
$data_line .= " | ".sprintf("%-${longest_estimated_runtime}s", $estimated_runtime);
$data_line .= " | ".sprintf("%-${longest_age_of_data}s", $age_of_data);
$data_line .= " |";
print $data_line."\n";
}
print $break_line."\n";
print $anvil->Words->string({key => "header_0061"})."\n";
return(0);
}
Loading…
Cancel
Save