seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Continued work on anvil-watch-servers; Parsed server data now.

Browse Source 
* Updated Cluster->parse_cib() to store DRBD fence node restrictions by
  server/node. Also updated to make it easier to get the server's
  preferred node.

Signed-off-by: digimer <mkelly@alteeve.ca>
main
digimer 1 year ago
parent 207a014ae0
commit 26f4446bf9
3 changed files with 274 additions and 12 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 60
      Anvil/Tools/Cluster.pm
  2. 22
      share/words.xml
  3. 204
      tools/anvil-watch-servers

60
Anvil/Tools/Cluster.pm
Unescape View File

@ -2694,7 +2694,7 @@ sub is_primary
=head2 manage_fence_delay =head2 manage_fence_delay
This method checks or sets the fence delay that controls which node survives in a network split. Generally, this is the node hosting servers, as ScanCore's C<< scan-cluster >> should set this based on where the servers are runn. This method checks or sets the fence delay that controls which node survives in a network split. Generally, this is the node hosting servers, as ScanCore's C<< scan-cluster >> should set this based on where the servers are run.
If C<< set >> is given an invalid host name, or if this is called on a node that is not a cluster member, C<< !!error!! >> is returned. Otherwise, the node with the delay favouring it is returned. If, somehow, neither node has a delay, then an empty string is returned. If C<< set >> is given an invalid host name, or if this is called on a node that is not a cluster member, C<< !!error!! >> is returned. Otherwise, the node with the delay favouring it is returned. If, somehow, neither node has a delay, then an empty string is returned.
@ -3387,8 +3387,25 @@ sub parse_cib
"cib::parsed::configuration::constraints::location::${id}::score" => $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{score}, "cib::parsed::configuration::constraints::location::${id}::score" => $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{score},
}}); }});
# If there's no 'node', this is probably a drbd fence constraint. # If there's no 'node', this is probably a drbd fence constraint. If there is
if (not $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{node}) # a node, make it easier to look up the score for each node.
if ($anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{node})
{
my $server = $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{resource};
my $node = $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{node};
my $score = $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{score};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"s1:server" => $server,
"s2:node" => $node,
"s3:score" => $score,
}});
$anvil->data->{cib}{parsed}{data}{location_constraint}{$server}{node}{$node}{score} = $score;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"cib::parsed::data::location_constraint::${server}::node::${node}::score" => $anvil->data->{cib}{parsed}{data}{location_constraint}{$server}{node}{$node}{score},
}});
}
else
{ {
foreach my $rule_id ($constraint->findnodes('./rule')) foreach my $rule_id ($constraint->findnodes('./rule'))
{ {
@ -3851,6 +3868,11 @@ sub parse_cib
"s2:node_name" => $node_name, "s2:node_name" => $node_name,
"s3:value" => $value, "s3:value" => $value,
}}); }});
$anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_node}{$node_name}{value} = $value;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"cib::parsed::data::server::${lrm_resource_id}::drbd_fence_node::${node_name}::value" => $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_node}{$node_name}{value},
}});
} }
} }
} }
@ -3865,6 +3887,37 @@ sub parse_cib
} }
} }
# Sort out which node a given server prefers.
foreach my $server (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{location_constraint}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { server => $server }});
my $highest_score = 0;
my $preferred_host = "";
foreach my $node (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{location_constraint}{$server}{node}})
{
my $this_score = $anvil->data->{cib}{parsed}{data}{location_constraint}{$server}{node}{$node}{score};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"s1:node" => $node,
"s2:this_score" => $this_score,
}});
if ($this_score > $highest_score)
{
$highest_score = $this_score;
$preferred_host = $node;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"s1:highest_score" => $highest_score,
"s2:preferred_host" => $preferred_host,
}});
}
}
$anvil->data->{cib}{parsed}{data}{location_constraint}{$server}{preferred_host} = $preferred_host;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"cib::parsed::data::location_constraint::${server}::preferred_host" => $anvil->data->{cib}{parsed}{data}{location_constraint}{$server}{preferred_host},
}});
}
# Now call 'crm_mon --output-as=xml' to determine which resource are running where. As of the time # Now call 'crm_mon --output-as=xml' to determine which resource are running where. As of the time
# of writing this (late 2020), stopped resources are not displayed. So the principle purpose of this # of writing this (late 2020), stopped resources are not displayed. So the principle purpose of this
# call is to determine what resources are running, and where they are running. # call is to determine what resources are running, and where they are running.
@ -3893,6 +3946,7 @@ sub parse_cib
# Starting # Starting
# Migrating # Migrating
# Stopping # Stopping
# Stopped
$status = $active ? "running" : "off"; $status = $active ? "running" : "off";
# If the role is NOT 'migrating', and we have a database connection, check to see if # If the role is NOT 'migrating', and we have a database connection, check to see if

22
share/words.xml
Unescape View File

@ -1180,9 +1180,15 @@ resource #!variable!server!# {
<key name="header_0115">-=] DR Hosts</key> <key name="header_0115">-=] DR Hosts</key>
<key name="header_0116">-=] Servers</key> <key name="header_0116">-=] Servers</key>
<key name="header_0117">Subnode</key> <key name="header_0117">Subnode</key>
<key name="header_0118">Host State</key> <key name="header_0118">Host Status</key>
<key name="header_0119">Pacemaker State</key> <key name="header_0119">Pacemaker Status</key>
<key name="header_0120">Maintenance Mode</key> <key name="header_0120">Maintenance Mode</key>
<key name="header_0121">Server Name</key>
<key name="header_0122">Server Status</key>
<key name="header_0123">Resource Status</key>
<key name="header_0124">Preferred Host</key>
<key name="header_0125">DRBD Fence</key>
<key name="header_0126">Booted</key>
<!-- Strings used by jobs --> <!-- Strings used by jobs -->
<key name="job_0001">Configure Network</key> <key name="job_0001">Configure Network</key>
@ -3634,6 +3640,18 @@ If you are comfortable that the target has changed for a known reason, you can s
<key name="striker_0312">Transitioning</key> <key name="striker_0312">Transitioning</key>
<key name="striker_0313">Maintenance Mode</key> <key name="striker_0313">Maintenance Mode</key>
<key name="striker_0314">Normal Operation</key> <key name="striker_0314">Normal Operation</key>
<key name="striker_0315">Unknown</key>
<key name="striker_0316">Running</key>
<key name="striker_0317">Blocked</key>
<key name="striker_0318">Paused</key>
<key name="striker_0319">Shutting Down</key>
<key name="striker_0320">Shut Off</key>
<key name="striker_0321">Crashed</key>
<key name="striker_0322">PM Suspended</key>
<key name="striker_0323">Started</key>
<key name="striker_0324">Starting</key>
<key name="striker_0325">Migrating</key>
<key name="striker_0326">Stopped</key>
<!-- These are generally units and appended to numbers --> <!-- These are generally units and appended to numbers -->
<key name="suffix_0001">#!variable!number!#/sec</key> <key name="suffix_0001">#!variable!number!#/sec</key>

204
tools/anvil-watch-servers
Unescape View File

@ -114,7 +114,7 @@ sub show_status
### TODO: Make this work outside the cluster, for cases when servers are running outside the ### TODO: Make this work outside the cluster, for cases when servers are running outside the
### pacemaker cluster stack. ### pacemaker cluster stack.
# Are we a cluster member? # Are we a cluster member?
my $problem = $anvil->Cluster->parse_cib(); my $problem = $anvil->Cluster->parse_cib({debug => 2});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
if ($problem) if ($problem)
{ {
@ -122,9 +122,21 @@ sub show_status
$anvil->nice_exit({exit_code => 1}); $anvil->nice_exit({exit_code => 1});
} }
# foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{server}})
# {
# print "DRBD Fence for Server: [".$server_name."]\n";
# foreach my $node_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_node}})
# {
# my $value = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_node}{$node_name}{value};
# print "- Node: [".$node_name."], value: [".$value."]\n";
# }
# }
# die;
# Load host information so that we can check for IPMI configs, if needed. # Load host information so that we can check for IPMI configs, if needed.
$anvil->Database->get_hosts(); $anvil->Database->get_hosts();
$anvil->Database->get_anvils(); $anvil->Database->get_anvils();
$anvil->Database->get_servers();
show_servers($anvil); show_servers($anvil);
print "\n"; print "\n";
@ -140,6 +152,189 @@ sub show_servers
# Show the server states # Show the server states
$anvil->data->{'say'}{server_name} = $anvil->Words->string({key => "header_0121"});
$anvil->data->{'say'}{server_status} = $anvil->Words->string({key => "header_0122"});
$anvil->data->{'say'}{resource_status} = $anvil->Words->string({key => "header_0123"});
$anvil->data->{'say'}{host_name} = $anvil->Words->string({key => "header_0026"});
$anvil->data->{'say'}{preferred_host} = $anvil->Words->string({key => "header_0124"});
$anvil->data->{'say'}{drbd_fence} = $anvil->Words->string({key => "header_0125"});
$anvil->data->{'say'}{boot_time} = $anvil->Words->string({key => "header_0126"});
$anvil->data->{'say'}{unknown} = $anvil->Words->string({key => "striker_0315"});
$anvil->data->{'say'}{running} = $anvil->Words->string({key => "striker_0316"});
$anvil->data->{'say'}{blocked} = $anvil->Words->string({key => "striker_0317"});
$anvil->data->{'say'}{paused} = $anvil->Words->string({key => "striker_0318"});
$anvil->data->{'say'}{shutting_down} = $anvil->Words->string({key => "striker_0319"});
$anvil->data->{'say'}{shut_off} = $anvil->Words->string({key => "striker_0320"});
$anvil->data->{'say'}{crashed} = $anvil->Words->string({key => "striker_0321"});
$anvil->data->{'say'}{pm_suspended} = $anvil->Words->string({key => "striker_0322"});
$anvil->data->{'say'}{started} = $anvil->Words->string({key => "striker_0323"});
$anvil->data->{'say'}{starting} = $anvil->Words->string({key => "striker_0324"});
$anvil->data->{'say'}{migrating} = $anvil->Words->string({key => "striker_0325"});
$anvil->data->{'say'}{stopping} = $anvil->Words->string({key => "striker_0309"});
$anvil->data->{'say'}{stopped} = $anvil->Words->string({key => "striker_0326"});
my $longest_server_name = length($anvil->data->{'say'}{server_name});
my $longest_server_status = length($anvil->data->{'say'}{server_status});
my $longest_resource_status = length($anvil->data->{'say'}{resource_status});
my $longest_host_name = length($anvil->data->{'say'}{host_name});
my $longest_preferred_host = length($anvil->data->{'say'}{preferred_host});
my $longest_drbd_fence = length($anvil->data->{'say'}{drbd_fence});
my $longest_boot_time = length($anvil->data->{'say'}{boot_time});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:longest_server_name' => $longest_server_name,
's2:longest_server_status' => $longest_server_status,
's3:longest_resource_status' => $longest_resource_status,
's4:longest_host_name' => $longest_host_name,
's5:longest_preferred_host' => $longest_preferred_host,
's6:longest_drbd_fence' => $longest_drbd_fence,
's7:longest_boot_time' => $longest_boot_time,
}});
my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
{
my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:server_name' => $server_name,
's2:server_uuid' => $server_uuid,
}});
my $server_user_stop = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_user_stop};
my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
my $server_boot_time = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_boot_time};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:server_user_stop' => $server_user_stop,
's2:server_host_uuid' => $server_host_uuid,
's3:server_state' => $server_state,
's4:server_boot_time' => $server_boot_time,
}});
# Database/virsh status
next if $server_state eq "DELETED";
my $say_server_state = $anvil->data->{'say'}{unknown};
if ($server_state eq "running") { $say_server_state = $anvil->data->{'say'}{running}; }
elsif ($server_state eq "blocked") { $say_server_state = $anvil->data->{'say'}{blocked}; }
elsif ($server_state eq "paused") { $say_server_state = $anvil->data->{'say'}{paused}; }
elsif ($server_state eq "in shutdown") { $say_server_state = $anvil->data->{'say'}{shutting_down}; }
elsif ($server_state eq "shut off") { $say_server_state = $anvil->data->{'say'}{shut_off}; }
elsif ($server_state eq "crashed") { $say_server_state = $anvil->data->{'say'}{crashed}; }
elsif ($server_state eq "pmsuspended") { $say_server_state = $anvil->data->{'say'}{pm_suspended}; }
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_state => $server_state }});
# pcs resource status
my $resource_status = "--";
my $is_failed = "--";
my $say_resource_status = $anvil->data->{'say'}{unknown};
if (exists $anvil->data->{cib}{parsed}{data}{server}{$server_name})
{
$resource_status = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{status};
$is_failed = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{failed};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
resource_status => $resource_status,
is_failed => $is_failed,
}});
if ($resource_status eq "started") { $say_resource_status = $anvil->data->{'say'}{started}; }
elsif ($resource_status eq "starting") { $say_resource_status = $anvil->data->{'say'}{starting}; }
elsif ($resource_status eq "migrating") { $say_resource_status = $anvil->data->{'say'}{migrating}; }
elsif ($resource_status eq "stopping") { $say_resource_status = $anvil->data->{'say'}{stopping}; }
elsif ($resource_status eq "stopped") { $say_resource_status = $anvil->data->{'say'}{stopped}; }
else { $say_resource_status = $resource_status; }
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource_status => $resource_status }});
# Host node.
my $say_host_name = "--";
if (($server_host_uuid) && ($server_host_uuid ne "NULL"))
{
$say_host_name = $anvil->data->{hosts}{host_uuid}{$server_host_uuid}{short_host_name};
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_host_name => $say_host_name }});
my $say_preferred_host = "";
my $preferred_host_score = 0;
if (exists $anvil->data->{cib}{parsed}{data}{location_constraint}{$server_name})
{
$say_preferred_host = $anvil->data->{cib}{parsed}{data}{location_constraint}{$server_name}{preferred_host};
$preferred_host_score = $anvil->data->{cib}{parsed}{data}{location_constraint}{$server_name}{node}{$say_preferred_host}{score};
$say_preferred_host .= " (".$preferred_host_score.")";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_preferred_host => $say_preferred_host }});
}
# DRBD fence
my $say_drbd_fence = "--";
foreach my $node_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_node}})
{
my $value = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_node}{$node_name}{value};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
node_name => $node_name,
value => $value,
}});
if ($value eq "1")
{
$say_drbd_fence .= $node_name.",";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_drbd_fence => $say_drbd_fence }});
}
}
$say_drbd_fence =~ s/,$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_drbd_fence => $say_drbd_fence }});
# Boot time
my $say_boot_time = "--";
if ($server_boot_time)
{
$say_boot_time = $anvil->Get->date_and_time({use_time => $server_boot_time});
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_boot_time => $say_boot_time }});
if (length($server_name) > $longest_server_name)
{
$longest_server_name = length($server_name);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_server_name => $longest_server_name }});
}
if (length($say_server_state) > $longest_server_status)
{
$longest_server_status = length($say_server_state);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_server_status => $longest_server_status }});
}
if (length($say_resource_status) > $longest_resource_status)
{
$longest_resource_status = length($say_resource_status);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_resource_status => $longest_resource_status }});
}
if (length($say_host_name) > $longest_host_name)
{
$longest_host_name = length($say_host_name);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_host_name => $longest_host_name }});
}
if (length($say_preferred_host) > $longest_preferred_host)
{
$longest_preferred_host = length($say_preferred_host);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_preferred_host => $longest_preferred_host }});
}
if (length($say_drbd_fence) > $longest_drbd_fence)
{
$longest_drbd_fence = length($say_drbd_fence);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_drbd_fence => $longest_drbd_fence }});
}
if (length($say_boot_time) > $longest_boot_time)
{
$longest_boot_time = length($say_boot_time);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_boot_time => $longest_boot_time }});
}
=cut
Servers:
+-------------+-----------------+-------------------+-------------+------------------+--------------+-----------+
| server name | <server status> | <resource status> | <host node> | <preferred host> | <dbrd fence> | Boot Time |
+-------------+-----------------+-------------------+-------------+------------------+--------------+-----------+
=cut
}
return(0); return(0);
} }
@ -160,7 +355,7 @@ sub show_nodes
### Strings ### Strings
# host states # host states
$anvil->data->{'say'}{unknown} = $anvil->Words->string({key => "unit_0004"}); $anvil->data->{'say'}{unknown} = $anvil->Words->string({key => "striker_0315"});
$anvil->data->{'say'}{online} = $anvil->Words->string({key => "striker_0308"}); $anvil->data->{'say'}{online} = $anvil->Words->string({key => "striker_0308"});
$anvil->data->{'say'}{powered_off} = $anvil->Words->string({key => "striker_0307"}); $anvil->data->{'say'}{powered_off} = $anvil->Words->string({key => "striker_0307"});
$anvil->data->{'say'}{stopping} = $anvil->Words->string({key => "striker_0309"}); $anvil->data->{'say'}{stopping} = $anvil->Words->string({key => "striker_0309"});
@ -270,11 +465,6 @@ Subnode Status:
| <node1> | <host status> | <Pacemaker status> | <mainteance mode> | | <node1> | <host status> | <Pacemaker status> | <mainteance mode> |
| <node2> | <Host status> | <Pacemaker status> | <mainteance mode> | | <node2> | <Host status> | <Pacemaker status> | <mainteance mode> |
+---------+---------------+--------------------+-------------------+ +---------+---------------+--------------------+-------------------+
Servers:
+-------------+----------------+------------------+-------------+-----------------+--------------+
| server name | <server state> | <resource state> | <host node> | <prefered host> | <dbrd fence> |
+-------------+----------------+------------------+-------------+-----------------+--------------+
=cut =cut
# Now look again to show the subnode states # Now look again to show the subnode states

Write Preview
Loading…
Cancel
Save