From a8b19343524476332e0e076cbf262493fd811425 Mon Sep 17 00:00:00 2001 From: digimer Date: Mon, 27 Nov 2023 13:33:19 -0500 Subject: [PATCH 1/4] Adding new anvil-watch-servers tool and man page Signed-off-by: digimer --- man/Makefile.am | 1 + man/anvil-watch-servers.8 | 0 tools/Makefile.am | 1 + tools/anvil-watch-servers | 0 4 files changed, 2 insertions(+) create mode 100644 man/anvil-watch-servers.8 create mode 100755 tools/anvil-watch-servers diff --git a/man/Makefile.am b/man/Makefile.am index 44a81a00..a6715bcd 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -51,6 +51,7 @@ dist_man8_MANS = \ anvil-watch-bonds.8 \ anvil-watch-drbd.8 \ anvil-watch-power.8 \ + anvil-watch-servers.8 \ fence_delay.8 \ fence_pacemaker.8 \ scancore.8 \ diff --git a/man/anvil-watch-servers.8 b/man/anvil-watch-servers.8 new file mode 100644 index 00000000..e69de29b diff --git a/tools/Makefile.am b/tools/Makefile.am index 200d0145..88e9ec61 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -49,6 +49,7 @@ dist_sbin_SCRIPTS = \ anvil-watch-bonds \ anvil-watch-drbd \ anvil-watch-power \ + anvil-watch-servers \ scancore \ striker-auto-initialize-all \ striker-boot-machine \ diff --git a/tools/anvil-watch-servers b/tools/anvil-watch-servers new file mode 100755 index 00000000..e69de29b From 207a014ae0026b376fc61542b95af7336a764e36 Mon Sep 17 00:00:00 2001 From: digimer Date: Mon, 27 Nov 2023 23:43:07 -0500 Subject: [PATCH 2/4] Got anvil-watch-servers showing the status of subnodes. * Updated System->maintenance_mode() to take 'host_uuid' so that the maintenance mode of remote machines can be checked/set. Signed-off-by: digimer --- Anvil/Tools/System.pm | 25 ++- man/anvil-watch-servers.8 | 30 ++++ share/words.xml | 14 +- tools/anvil-watch-power | 1 + tools/anvil-watch-servers | 370 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 433 insertions(+), 7 deletions(-) diff --git a/Anvil/Tools/System.pm b/Anvil/Tools/System.pm index beab237e..e8e07bb1 100644 --- a/Anvil/Tools/System.pm +++ b/Anvil/Tools/System.pm @@ -3597,6 +3597,10 @@ This returns C<< 1 >> if maintenance mode is enabled and C<< 0 >> if disabled. Parameters; +=head3 host_uuid (optional, default 'Get->host_uuid') + +If set, this can check or set the maintenance mode on another host. + =head3 set (optional) If this is set to C<< 1 >>, maintenance mode is enabled. If this is set to C<< 0 >>, maintenance mode is disabled. @@ -3610,8 +3614,18 @@ sub maintenance_mode my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "System->maintenance_mode()" }}); - my $set = defined $parameter->{set} ? $parameter->{set} : ""; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { set => $set }}); + my $host_uuid = defined $parameter->{host_uuid} ? $parameter->{host_uuid} : ""; + my $set = defined $parameter->{set} ? $parameter->{set} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + host_uuid => $host_uuid, + set => $set, + }}); + + if (not $host_uuid) + { + $host_uuid = $anvil->Get->host_uuid; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }}); + } if (($set) or ($set eq "0")) { @@ -3627,7 +3641,7 @@ sub maintenance_mode variable_default => "0", variable_description => "striker_0087", variable_section => "system", - variable_source_uuid => $anvil->Get->host_uuid, + variable_source_uuid => $host_uuid, variable_source_table => "hosts", }); } @@ -3641,7 +3655,7 @@ sub maintenance_mode variable_default => "0", variable_description => "striker_0087", variable_section => "system", - variable_source_uuid => $anvil->Get->host_uuid, + variable_source_uuid => $host_uuid, variable_source_table => "hosts", }); } @@ -3657,10 +3671,9 @@ sub maintenance_mode debug => $debug, variable_name => "maintenance_mode", variable_source_table => "hosts", - variable_source_uuid => $anvil->Get->host_uuid, + variable_source_uuid => $host_uuid, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - debug => $debug, maintenance_mode => $maintenance_mode, variable_uuid => $variable_uuid, modified_date => $modified_date, diff --git a/man/anvil-watch-servers.8 b/man/anvil-watch-servers.8 index e69de29b..3b351872 100644 --- a/man/anvil-watch-servers.8 +++ b/man/anvil-watch-servers.8 @@ -0,0 +1,30 @@ +.\" Manpage for the Anvil! server system manager +.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions. +.TH anvil-watch-servers "8" "November 27 2023" "Anvil! Intelligent Availability™ Platform" +.SH NAME +anvil-watch-servers \- Tool used to watch the status of servers on an Anvil! node. +.SH SYNOPSIS +.B anvil-watch-servers +\fI\, \/\fR[\fI\,options\/\fR] +.SH DESCRIPTION +anvil-watch-servers \- This is designed to be run on an Anvil! node to monitor the status of servers it is hosting. It is not designed for use on Strikers or to monitor multiple nodes (yet). +.TP +.SH OPTIONS +.TP +\-?, \-h, \fB\-\-help\fR +Show this man page. +.TP +\fB\-\-log-secure\fR +When logging, record sensitive data, like passwords. +.TP +\-v, \-vv, \-vvv +Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data. +.SS "Commands:" +.TP +\fB\-\-watch\fR +Without this switch, the state of the servers is shown as it is now, and then the program exists. With this switch, the program will stay active, refreshing every X seconds, as set with this switch. The default is to refresh every 2 seconds. +.IP +.SH AUTHOR +Written by Madison Kelly, Alteeve staff and the Anvil! project contributors. +.SH "REPORTING BUGS" +Report bugs to users@clusterlabs.org diff --git a/share/words.xml b/share/words.xml index 701ce852..81cfdebd 100644 --- a/share/words.xml +++ b/share/words.xml @@ -1179,6 +1179,10 @@ resource #!variable!server!# { Completed Jobs -=] DR Hosts -=] Servers + Subnode + Host State + Pacemaker State + Maintenance Mode Configure Network @@ -3202,7 +3206,7 @@ Proceed? [y/N] - Target: [#!variable!device_target!#], boot: [#!variable!say_boot!#], Replication Volume: [#!variable!drbd_resource!#/#!variable!drbd_volume!#] |- Resource / LV / Metadata sizes: [#!variable!resource_size!# / #!variable!lv_size!# / #!variable!metadata_size!#], free space: [#!variable!max_free_space!#] - Target: [#!variable!device_target!#], boot: [#!variable!say_boot!#], ISO: [#!variable!device_path!#] - Sub-Nodes: + Subnodes: DR Hosts: |- Name: [#!variable!host_name!#], UUID: [#!ariable!host_uuid!#] |- Volume: [#!variable!volume_number!#], backing device: [#!variable!backing_disk!#], DRBD minor: [#!variable!device_minor!#], size: [#!variable!volume_size!#] @@ -3622,6 +3626,14 @@ If you are comfortable that the target has changed for a known reason, you can s This indicates when, in unix time, the local install target data was updated. This indicates when, in unix time, the OUI data was last update. The OUI data is a list of MAC address prefixes and which companies they've been assigned to. This indicates when, in unix time, the network was last scanned. This is done to determine what IPs are used by servers on the Anvil! node, and to try to identify foundation pack devices on the network. These scans are simple ping sweeps used to get the MAC addresses of devices with IPs. + Powered Off + Online + Stopping + Booting + Offline + Transitioning + Maintenance Mode + Normal Operation #!variable!number!#/sec diff --git a/tools/anvil-watch-power b/tools/anvil-watch-power index 20dc7af5..e64ac5d0 100755 --- a/tools/anvil-watch-power +++ b/tools/anvil-watch-power @@ -43,6 +43,7 @@ while(1) if ($anvil->data->{sys}{database}{connections}) { show_power_data($anvil); + $anvil->Database->disconnect(); } else { diff --git a/tools/anvil-watch-servers b/tools/anvil-watch-servers index e69de29b..bea59836 100755 --- a/tools/anvil-watch-servers +++ b/tools/anvil-watch-servers @@ -0,0 +1,370 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use Anvil::Tools; +use Data::Dumper; +use Text::Diff; +use Term::Cap; +use Time::Local; + +$| = 1; + +my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; +my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; +if (($running_directory =~ /^\./) && ($ENV{PWD})) +{ + $running_directory =~ s/^\./$ENV{PWD}/; +} + +my $anvil = Anvil::Tools->new(); + +# Get a list of all interfaces with IP addresses. +$anvil->Get->switches({debug => 2, list => ["watch"]}); +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); + +$anvil->Database->connect(); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132" }); +if (not $anvil->data->{sys}{database}{connections}) +{ + # No databases, exit. + $anvil->Log->entry({ source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003" }); + $anvil->nice_exit({ exit_code => 1 }); +} + +our $t = Term::Cap->Tgetent; + +# One shot or continuous? +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 'switches::watch' => $anvil->data->{switches}{watch}, +}}); +if ($anvil->data->{switches}{watch}) +{ + # Disconnect before we go into the loop + $anvil->Database->disconnect(); + + # Do we have an interval? + my $interval = 2; + if ($anvil->data->{switches}{watch} =~ /^\d+$/) + { + $interval = $anvil->data->{switches}{watch}; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { interval => $interval }}); + + # Loop until terminated. + while(1) + { + $anvil->refresh(); + $anvil->Database->connect(); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0132"}); + + if ($anvil->data->{sys}{database}{connections}) + { + show_status($anvil); + $anvil->Database->disconnect(); + } + else + { + # No databases available. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "log_0738"}); + } + sleep $interval; + } +} +else +{ + # Once and exit. + $anvil->Database->connect(); + show_status($anvil); +} + +$anvil->nice_exit({exit_code => 0}); + + +############################################################################################################# +# Functions # +############################################################################################################# + +sub show_status +{ + my ($anvil) = @_; + + if ($anvil->data->{switches}{watch}) + { + system('clear'); + print $t->Tgoto("cm", 0, 0); + } + + if ($anvil->data->{switches}{watch}) + { + my $date = $anvil->Get->date_and_time(); + print "-=] Updated: ".$date." - Press ' + ' to exit\n"; + } + + ### TODO: Add support for checking/monitoring DR hosts + # Get the node states + my $host_type = $anvil->Get->host_type(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }}); + if ($host_type ne "node") + { + print "This must be run on a subnode. Exiting.\n"; + $anvil->nice_exit({exit_code => 1}); + } + + ### TODO: Make this work outside the cluster, for cases when servers are running outside the + ### pacemaker cluster stack. + # Are we a cluster member? + my $problem = $anvil->Cluster->parse_cib(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); + if ($problem) + { + print "This subnode is not in the cluster (failed to parse the CIB). Exiting.\n"; + $anvil->nice_exit({exit_code => 1}); + } + + # Load host information so that we can check for IPMI configs, if needed. + $anvil->Database->get_hosts(); + $anvil->Database->get_anvils(); + + show_servers($anvil); + print "\n"; + + show_nodes($anvil); + + return(0); +} + +sub show_servers +{ + my ($anvil) = @_; + + # Show the server states + + return(0); +} + +sub show_nodes +{ + my ($anvil) = @_; + + # Headers + $anvil->data->{'say'}{subnode} = $anvil->Words->string({key => "header_0117"}); + $anvil->data->{'say'}{host_status} = $anvil->Words->string({key => "header_0118"}); + $anvil->data->{'say'}{pacemaker_status} = $anvil->Words->string({key => "header_0119"}); + $anvil->data->{'say'}{maintenance_mode} = $anvil->Words->string({key => "header_0120"}); + + my $longest_node_name = length($anvil->data->{'say'}{subnode}); + my $longest_host_status = length($anvil->data->{'say'}{host_status}); + my $longest_pacemaker_status = length($anvil->data->{'say'}{pacemaker_status}); + my $longest_maintenance_mode = length($anvil->data->{'say'}{maintenance_mode}); + + ### Strings + # host states + $anvil->data->{'say'}{unknown} = $anvil->Words->string({key => "unit_0004"}); + $anvil->data->{'say'}{online} = $anvil->Words->string({key => "striker_0308"}); + $anvil->data->{'say'}{powered_off} = $anvil->Words->string({key => "striker_0307"}); + $anvil->data->{'say'}{stopping} = $anvil->Words->string({key => "striker_0309"}); + $anvil->data->{'say'}{booting} = $anvil->Words->string({key => "striker_0310"}); + + # Cluster states (online from above) + $anvil->data->{'say'}{offline} = $anvil->Words->string({key => "striker_0311"}); + $anvil->data->{'say'}{transitioning} = $anvil->Words->string({key => "striker_0312"}); + + # Maintenance mode. + $anvil->data->{'say'}{maintenance_mode} = $anvil->Words->string({key => "striker_0313"}); + $anvil->data->{'say'}{normal_operation} = $anvil->Words->string({key => "striker_0314"}); + + # Get the length of the node strings. + foreach my $node_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}}) + { + my $host_uuid = $anvil->Database->get_host_uuid_from_string({string => $node_name}); + my $host_status = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_status}; + my $maintenance_mode = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{node_state}{'maintenance-mode'}; + my $in_ccm = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{node_state}{in_ccm}; + my $crmd = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{node_state}{crmd}; + my $join = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{node_state}{'join'}; + my $ready = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{node_state}{ready}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:node_name' => $node_name, + 's2:host_uuid' => $host_uuid, + 's3:host_status' => $host_status, + 's4:maintenance_mode' => $maintenance_mode, + 's5:in_ccm' => $in_ccm, + 's6:crmd' => $crmd, + 's7:join' => $join, + 's8:ready' => $ready, + }}); + + # Convert the host state to a string. + my $say_host_status = $anvil->data->{'say'}{unknown}; + if ($host_status eq "online") + { + $say_host_status = $anvil->data->{'say'}{online}; + } + elsif ($host_status eq "powered off") + { + $say_host_status = $anvil->data->{'say'}{powered_off}; + } + elsif ($host_status eq "stopping") + { + $say_host_status = $anvil->data->{'say'}{stopping}; + } + elsif ($host_status eq "booting") + { + $say_host_status = $anvil->data->{'say'}{booting}; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_host_status => $say_host_status }}); + + # Convert the Pacemaker state. + my $say_pacemaker_status = $anvil->data->{'say'}{unknown}; + if ($ready) + { + $say_pacemaker_status = $anvil->data->{'say'}{online}; + } + elsif (($in_ccm) or ($crmd) or ($join)) + { + # Transitioning + $say_pacemaker_status = $anvil->data->{'say'}{transitioning}; + } + else + { + $say_pacemaker_status = $anvil->data->{'say'}{offline}; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_pacemaker_status => $say_pacemaker_status }}); + + # Maintenance mode + my $anvil_maintenance_mode = $anvil->System->maintenance_mode({host_uuid => $host_uuid}); + my $say_maintenance_mode = (($maintenance_mode) or ($anvil_maintenance_mode)) ? $anvil->data->{'say'}{maintenance_mode} : $anvil->data->{'say'}{normal_operation}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + anvil_maintenance_mode => $anvil_maintenance_mode, + say_maintenance_mode => $say_maintenance_mode, + }}); + + # Update the lengths, if needed + if (length($node_name) > $longest_node_name) + { + $longest_node_name = length($node_name); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_node_name => $longest_node_name }}); + } + if (length($say_host_status) > $longest_host_status) + { + $longest_host_status = length($say_host_status); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_host_status => $longest_host_status }}); + } + if (length($say_pacemaker_status) > $longest_pacemaker_status) + { + $longest_pacemaker_status = length($say_pacemaker_status); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_pacemaker_status => $longest_pacemaker_status }}); + } + if (length($say_maintenance_mode) > $longest_maintenance_mode) + { + $longest_maintenance_mode = length($say_maintenance_mode); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_maintenance_mode => $longest_maintenance_mode }}); + } + } +=cut +Subnode Status: ++---------+---------------+--------------------+-------------------+ +| Subnode | Host Status | Pacemaker Status | Maintenance Mode | ++---------+---------------+--------------------+-------------------+ +| | | | | +| | | | | ++---------+---------------+--------------------+-------------------+ + +Servers: ++-------------+----------------+------------------+-------------+-----------------+--------------+ +| server name | | | | | | ++-------------+----------------+------------------+-------------+-----------------+--------------+ +=cut + + # Now look again to show the subnode states + my $subnode_divider = ""; for (1..$longest_node_name) { $subnode_divider .= "-"; } + my $host_status_divider = ""; for (1..$longest_host_status) { $host_status_divider .= "-"; } + my $pacemaker_status_divider = ""; for (1..$longest_pacemaker_status) { $pacemaker_status_divider .= "-"; } + my $maintenance_mode_divider = ""; for (1..$longest_maintenance_mode) { $maintenance_mode_divider .= "-"; } + + my $say_subnode_header = $anvil->Words->center_text({string => $anvil->data->{'say'}{subnode}, width => $longest_node_name}); + my $say_host_status_header = $anvil->Words->center_text({string => $anvil->data->{'say'}{host_status}, width => $longest_host_status}); + my $say_pacemaker_status_header = $anvil->Words->center_text({string => $anvil->data->{'say'}{pacemaker_status}, width => $longest_pacemaker_status}); + my $say_maintenance_mode_header = $anvil->Words->center_text({string => $anvil->data->{'say'}{maintenance_mode}, width => $longest_maintenance_mode}); + + my $divider_line = "+-".$subnode_divider."-+-".$host_status_divider."-+-".$pacemaker_status_divider."-+-".$maintenance_mode_divider."-+\n"; + print $divider_line; + print "| ".$say_subnode_header." | ".$say_host_status_header." | ".$say_pacemaker_status_header." | ".$say_maintenance_mode_header." |\n"; + print $divider_line; + foreach my $node_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}}) + { + my $host_uuid = $anvil->Database->get_host_uuid_from_string({string => $node_name}); + my $host_status = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_status}; + my $maintenance_mode = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{node_state}{'maintenance-mode'}; + my $in_ccm = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{node_state}{in_ccm}; + my $crmd = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{node_state}{crmd}; + my $join = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{node_state}{'join'}; + my $ready = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{node_state}{ready}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:node_name' => $node_name, + 's2:host_uuid' => $host_uuid, + 's3:host_status' => $host_status, + 's4:maintenance_mode' => $maintenance_mode, + 's5:in_ccm' => $in_ccm, + 's6:crmd' => $crmd, + 's7:join' => $join, + 's8:ready' => $ready, + }}); + + # Convert the host state to a string. + my $say_host_status = $anvil->data->{'say'}{unknown}; + if ($host_status eq "online") + { + $say_host_status = $anvil->data->{'say'}{online}; + } + elsif ($host_status eq "powered off") + { + $say_host_status = $anvil->data->{'say'}{powered_off}; + } + elsif ($host_status eq "stopping") + { + $say_host_status = $anvil->data->{'say'}{stopping}; + } + elsif ($host_status eq "booting") + { + $say_host_status = $anvil->data->{'say'}{booting}; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_host_status => $say_host_status }}); + + # Convert the Pacemaker state. + my $say_pacemaker_status = $anvil->data->{'say'}{unknown}; + if ($ready) + { + $say_pacemaker_status = $anvil->data->{'say'}{online}; + } + elsif (($in_ccm) or ($crmd) or ($join)) + { + # Transitioning + $say_pacemaker_status = $anvil->data->{'say'}{transitioning}; + } + else + { + $say_pacemaker_status = $anvil->data->{'say'}{offline}; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_pacemaker_status => $say_pacemaker_status }}); + + # Maintenance mode + my $anvil_maintenance_mode = $anvil->System->maintenance_mode({host_uuid => $host_uuid}); + my $say_maintenance_mode = (($maintenance_mode) or ($anvil_maintenance_mode)) ? $anvil->data->{'say'}{maintenance_mode} : $anvil->data->{'say'}{normal_operation}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + anvil_maintenance_mode => $anvil_maintenance_mode, + say_maintenance_mode => $say_maintenance_mode, + }}); + + print "| "; + print sprintf("%-${longest_node_name}s", $node_name)." | "; + print sprintf("%-${longest_host_status}s", $say_host_status)." | "; + print sprintf("%-${longest_pacemaker_status}s", $say_pacemaker_status)." | "; + print sprintf("%-${longest_maintenance_mode}s", $say_maintenance_mode)." |\n"; + + } + print $divider_line; + + return(0); +} From 26f4446bf91147e17c349dc5af6da93ff0167d16 Mon Sep 17 00:00:00 2001 From: digimer Date: Wed, 29 Nov 2023 01:27:21 -0500 Subject: [PATCH 3/4] Continued work on anvil-watch-servers; Parsed server data now. * Updated Cluster->parse_cib() to store DRBD fence node restrictions by server/node. Also updated to make it easier to get the server's preferred node. Signed-off-by: digimer --- Anvil/Tools/Cluster.pm | 60 ++++++++++- share/words.xml | 22 +++- tools/anvil-watch-servers | 204 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 274 insertions(+), 12 deletions(-) diff --git a/Anvil/Tools/Cluster.pm b/Anvil/Tools/Cluster.pm index 35f11640..6021918e 100644 --- a/Anvil/Tools/Cluster.pm +++ b/Anvil/Tools/Cluster.pm @@ -2694,7 +2694,7 @@ sub is_primary =head2 manage_fence_delay -This method checks or sets the fence delay that controls which node survives in a network split. Generally, this is the node hosting servers, as ScanCore's C<< scan-cluster >> should set this based on where the servers are runn. +This method checks or sets the fence delay that controls which node survives in a network split. Generally, this is the node hosting servers, as ScanCore's C<< scan-cluster >> should set this based on where the servers are run. If C<< set >> is given an invalid host name, or if this is called on a node that is not a cluster member, C<< !!error!! >> is returned. Otherwise, the node with the delay favouring it is returned. If, somehow, neither node has a delay, then an empty string is returned. @@ -3387,8 +3387,25 @@ sub parse_cib "cib::parsed::configuration::constraints::location::${id}::score" => $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{score}, }}); - # If there's no 'node', this is probably a drbd fence constraint. - if (not $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{node}) + # If there's no 'node', this is probably a drbd fence constraint. If there is + # a node, make it easier to look up the score for each node. + if ($anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{node}) + { + my $server = $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{resource}; + my $node = $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{node}; + my $score = $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{score}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:server" => $server, + "s2:node" => $node, + "s3:score" => $score, + }}); + + $anvil->data->{cib}{parsed}{data}{location_constraint}{$server}{node}{$node}{score} = $score; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "cib::parsed::data::location_constraint::${server}::node::${node}::score" => $anvil->data->{cib}{parsed}{data}{location_constraint}{$server}{node}{$node}{score}, + }}); + } + else { foreach my $rule_id ($constraint->findnodes('./rule')) { @@ -3851,6 +3868,11 @@ sub parse_cib "s2:node_name" => $node_name, "s3:value" => $value, }}); + + $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_node}{$node_name}{value} = $value; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "cib::parsed::data::server::${lrm_resource_id}::drbd_fence_node::${node_name}::value" => $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_node}{$node_name}{value}, + }}); } } } @@ -3865,6 +3887,37 @@ sub parse_cib } } + # Sort out which node a given server prefers. + foreach my $server (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{location_constraint}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { server => $server }}); + my $highest_score = 0; + my $preferred_host = ""; + foreach my $node (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{location_constraint}{$server}{node}}) + { + my $this_score = $anvil->data->{cib}{parsed}{data}{location_constraint}{$server}{node}{$node}{score}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:node" => $node, + "s2:this_score" => $this_score, + }}); + + if ($this_score > $highest_score) + { + $highest_score = $this_score; + $preferred_host = $node; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:highest_score" => $highest_score, + "s2:preferred_host" => $preferred_host, + }}); + } + } + + $anvil->data->{cib}{parsed}{data}{location_constraint}{$server}{preferred_host} = $preferred_host; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "cib::parsed::data::location_constraint::${server}::preferred_host" => $anvil->data->{cib}{parsed}{data}{location_constraint}{$server}{preferred_host}, + }}); + } + # Now call 'crm_mon --output-as=xml' to determine which resource are running where. As of the time # of writing this (late 2020), stopped resources are not displayed. So the principle purpose of this # call is to determine what resources are running, and where they are running. @@ -3893,6 +3946,7 @@ sub parse_cib # Starting # Migrating # Stopping + # Stopped $status = $active ? "running" : "off"; # If the role is NOT 'migrating', and we have a database connection, check to see if diff --git a/share/words.xml b/share/words.xml index 81cfdebd..813730c9 100644 --- a/share/words.xml +++ b/share/words.xml @@ -1180,9 +1180,15 @@ resource #!variable!server!# { -=] DR Hosts -=] Servers Subnode - Host State - Pacemaker State + Host Status + Pacemaker Status Maintenance Mode + Server Name + Server Status + Resource Status + Preferred Host + DRBD Fence + Booted Configure Network @@ -3634,6 +3640,18 @@ If you are comfortable that the target has changed for a known reason, you can s Transitioning Maintenance Mode Normal Operation + Unknown + Running + Blocked + Paused + Shutting Down + Shut Off + Crashed + PM Suspended + Started + Starting + Migrating + Stopped #!variable!number!#/sec diff --git a/tools/anvil-watch-servers b/tools/anvil-watch-servers index bea59836..20a96b35 100755 --- a/tools/anvil-watch-servers +++ b/tools/anvil-watch-servers @@ -114,7 +114,7 @@ sub show_status ### TODO: Make this work outside the cluster, for cases when servers are running outside the ### pacemaker cluster stack. # Are we a cluster member? - my $problem = $anvil->Cluster->parse_cib(); + my $problem = $anvil->Cluster->parse_cib({debug => 2}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); if ($problem) { @@ -122,9 +122,21 @@ sub show_status $anvil->nice_exit({exit_code => 1}); } +# foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{server}}) +# { +# print "DRBD Fence for Server: [".$server_name."]\n"; +# foreach my $node_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_node}}) +# { +# my $value = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_node}{$node_name}{value}; +# print "- Node: [".$node_name."], value: [".$value."]\n"; +# } +# } +# die; + # Load host information so that we can check for IPMI configs, if needed. $anvil->Database->get_hosts(); $anvil->Database->get_anvils(); + $anvil->Database->get_servers(); show_servers($anvil); print "\n"; @@ -140,6 +152,189 @@ sub show_servers # Show the server states + $anvil->data->{'say'}{server_name} = $anvil->Words->string({key => "header_0121"}); + $anvil->data->{'say'}{server_status} = $anvil->Words->string({key => "header_0122"}); + $anvil->data->{'say'}{resource_status} = $anvil->Words->string({key => "header_0123"}); + $anvil->data->{'say'}{host_name} = $anvil->Words->string({key => "header_0026"}); + $anvil->data->{'say'}{preferred_host} = $anvil->Words->string({key => "header_0124"}); + $anvil->data->{'say'}{drbd_fence} = $anvil->Words->string({key => "header_0125"}); + $anvil->data->{'say'}{boot_time} = $anvil->Words->string({key => "header_0126"}); + + $anvil->data->{'say'}{unknown} = $anvil->Words->string({key => "striker_0315"}); + $anvil->data->{'say'}{running} = $anvil->Words->string({key => "striker_0316"}); + $anvil->data->{'say'}{blocked} = $anvil->Words->string({key => "striker_0317"}); + $anvil->data->{'say'}{paused} = $anvil->Words->string({key => "striker_0318"}); + $anvil->data->{'say'}{shutting_down} = $anvil->Words->string({key => "striker_0319"}); + $anvil->data->{'say'}{shut_off} = $anvil->Words->string({key => "striker_0320"}); + $anvil->data->{'say'}{crashed} = $anvil->Words->string({key => "striker_0321"}); + $anvil->data->{'say'}{pm_suspended} = $anvil->Words->string({key => "striker_0322"}); + + $anvil->data->{'say'}{started} = $anvil->Words->string({key => "striker_0323"}); + $anvil->data->{'say'}{starting} = $anvil->Words->string({key => "striker_0324"}); + $anvil->data->{'say'}{migrating} = $anvil->Words->string({key => "striker_0325"}); + $anvil->data->{'say'}{stopping} = $anvil->Words->string({key => "striker_0309"}); + $anvil->data->{'say'}{stopped} = $anvil->Words->string({key => "striker_0326"}); + + my $longest_server_name = length($anvil->data->{'say'}{server_name}); + my $longest_server_status = length($anvil->data->{'say'}{server_status}); + my $longest_resource_status = length($anvil->data->{'say'}{resource_status}); + my $longest_host_name = length($anvil->data->{'say'}{host_name}); + my $longest_preferred_host = length($anvil->data->{'say'}{preferred_host}); + my $longest_drbd_fence = length($anvil->data->{'say'}{drbd_fence}); + my $longest_boot_time = length($anvil->data->{'say'}{boot_time}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:longest_server_name' => $longest_server_name, + 's2:longest_server_status' => $longest_server_status, + 's3:longest_resource_status' => $longest_resource_status, + 's4:longest_host_name' => $longest_host_name, + 's5:longest_preferred_host' => $longest_preferred_host, + 's6:longest_drbd_fence' => $longest_drbd_fence, + 's7:longest_boot_time' => $longest_boot_time, + }}); + + my $anvil_uuid = $anvil->Cluster->get_anvil_uuid(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); + foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}}) + { + my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:server_name' => $server_name, + 's2:server_uuid' => $server_uuid, + }}); + + my $server_user_stop = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_user_stop}; + my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid}; + my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state}; + my $server_boot_time = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_boot_time}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:server_user_stop' => $server_user_stop, + 's2:server_host_uuid' => $server_host_uuid, + 's3:server_state' => $server_state, + 's4:server_boot_time' => $server_boot_time, + }}); + + # Database/virsh status + next if $server_state eq "DELETED"; + my $say_server_state = $anvil->data->{'say'}{unknown}; + if ($server_state eq "running") { $say_server_state = $anvil->data->{'say'}{running}; } + elsif ($server_state eq "blocked") { $say_server_state = $anvil->data->{'say'}{blocked}; } + elsif ($server_state eq "paused") { $say_server_state = $anvil->data->{'say'}{paused}; } + elsif ($server_state eq "in shutdown") { $say_server_state = $anvil->data->{'say'}{shutting_down}; } + elsif ($server_state eq "shut off") { $say_server_state = $anvil->data->{'say'}{shut_off}; } + elsif ($server_state eq "crashed") { $say_server_state = $anvil->data->{'say'}{crashed}; } + elsif ($server_state eq "pmsuspended") { $say_server_state = $anvil->data->{'say'}{pm_suspended}; } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_state => $server_state }}); + + # pcs resource status + my $resource_status = "--"; + my $is_failed = "--"; + my $say_resource_status = $anvil->data->{'say'}{unknown}; + if (exists $anvil->data->{cib}{parsed}{data}{server}{$server_name}) + { + $resource_status = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{status}; + $is_failed = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{failed}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + resource_status => $resource_status, + is_failed => $is_failed, + }}); + if ($resource_status eq "started") { $say_resource_status = $anvil->data->{'say'}{started}; } + elsif ($resource_status eq "starting") { $say_resource_status = $anvil->data->{'say'}{starting}; } + elsif ($resource_status eq "migrating") { $say_resource_status = $anvil->data->{'say'}{migrating}; } + elsif ($resource_status eq "stopping") { $say_resource_status = $anvil->data->{'say'}{stopping}; } + elsif ($resource_status eq "stopped") { $say_resource_status = $anvil->data->{'say'}{stopped}; } + else { $say_resource_status = $resource_status; } + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource_status => $resource_status }}); + + # Host node. + my $say_host_name = "--"; + if (($server_host_uuid) && ($server_host_uuid ne "NULL")) + { + $say_host_name = $anvil->data->{hosts}{host_uuid}{$server_host_uuid}{short_host_name}; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_host_name => $say_host_name }}); + + my $say_preferred_host = ""; + my $preferred_host_score = 0; + if (exists $anvil->data->{cib}{parsed}{data}{location_constraint}{$server_name}) + { + $say_preferred_host = $anvil->data->{cib}{parsed}{data}{location_constraint}{$server_name}{preferred_host}; + $preferred_host_score = $anvil->data->{cib}{parsed}{data}{location_constraint}{$server_name}{node}{$say_preferred_host}{score}; + $say_preferred_host .= " (".$preferred_host_score.")"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_preferred_host => $say_preferred_host }}); + } + + # DRBD fence + my $say_drbd_fence = "--"; + foreach my $node_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_node}}) + { + my $value = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_node}{$node_name}{value}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + node_name => $node_name, + value => $value, + }}); + if ($value eq "1") + { + $say_drbd_fence .= $node_name.","; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_drbd_fence => $say_drbd_fence }}); + } + } + $say_drbd_fence =~ s/,$//; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_drbd_fence => $say_drbd_fence }}); + + # Boot time + my $say_boot_time = "--"; + if ($server_boot_time) + { + $say_boot_time = $anvil->Get->date_and_time({use_time => $server_boot_time}); + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_boot_time => $say_boot_time }}); + + if (length($server_name) > $longest_server_name) + { + $longest_server_name = length($server_name); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_server_name => $longest_server_name }}); + } + if (length($say_server_state) > $longest_server_status) + { + $longest_server_status = length($say_server_state); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_server_status => $longest_server_status }}); + } + if (length($say_resource_status) > $longest_resource_status) + { + $longest_resource_status = length($say_resource_status); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_resource_status => $longest_resource_status }}); + } + if (length($say_host_name) > $longest_host_name) + { + $longest_host_name = length($say_host_name); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_host_name => $longest_host_name }}); + } + if (length($say_preferred_host) > $longest_preferred_host) + { + $longest_preferred_host = length($say_preferred_host); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_preferred_host => $longest_preferred_host }}); + } + if (length($say_drbd_fence) > $longest_drbd_fence) + { + $longest_drbd_fence = length($say_drbd_fence); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_drbd_fence => $longest_drbd_fence }}); + } + if (length($say_boot_time) > $longest_boot_time) + { + $longest_boot_time = length($say_boot_time); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_boot_time => $longest_boot_time }}); + } + + +=cut +Servers: ++-------------+-----------------+-------------------+-------------+------------------+--------------+-----------+ +| server name | | | | | | Boot Time | ++-------------+-----------------+-------------------+-------------+------------------+--------------+-----------+ +=cut + } + return(0); } @@ -160,7 +355,7 @@ sub show_nodes ### Strings # host states - $anvil->data->{'say'}{unknown} = $anvil->Words->string({key => "unit_0004"}); + $anvil->data->{'say'}{unknown} = $anvil->Words->string({key => "striker_0315"}); $anvil->data->{'say'}{online} = $anvil->Words->string({key => "striker_0308"}); $anvil->data->{'say'}{powered_off} = $anvil->Words->string({key => "striker_0307"}); $anvil->data->{'say'}{stopping} = $anvil->Words->string({key => "striker_0309"}); @@ -270,11 +465,6 @@ Subnode Status: | | | | | | | | | | +---------+---------------+--------------------+-------------------+ - -Servers: -+-------------+----------------+------------------+-------------+-----------------+--------------+ -| server name | | | | | | -+-------------+----------------+------------------+-------------+-----------------+--------------+ =cut # Now look again to show the subnode states From fd880e2fdfa514f5aa8fc3955f381289cdff8053 Mon Sep 17 00:00:00 2001 From: digimer Date: Wed, 29 Nov 2023 11:37:00 -0500 Subject: [PATCH 4/4] Finished anvil-watch-servers Signed-off-by: digimer --- share/words.xml | 5 +- tools/anvil-watch-servers | 199 ++++++++++++++++++++++++++++++-------- 2 files changed, 163 insertions(+), 41 deletions(-) diff --git a/share/words.xml b/share/words.xml index 813730c9..fef25809 100644 --- a/share/words.xml +++ b/share/words.xml @@ -458,7 +458,7 @@ Failed to parse the XML in the new definition file. The error was: Giving up. - This must be run on a node active in the cluster hosting the server being managed. Exiting. + This must be run on a subnode active in the cluster hosting the server being managed. Exiting. There are no DR hosts connected to this Anvil! node yet. Failed to find an IP we can access the DR host: [#!variable!host_name!#]. Has it been configured? Is it running? Exiting. Failed to access the DR host: [#!variable!host_name!#] using the IP: [#!variable!ip_address!#]. Is it running? Exiting. @@ -750,6 +750,8 @@ The XML that failed sanity check was: [ Error ] - The requested size: [#!variable!requested_ram!#] is less than 640 KiB, this must be a mistake. [ Error ] - The requested CPU: [#!variable!requested_cpu!#] is not valid. Valid options are '--cpu X', '--cpu Y,X' or '--cpu 'Y,X,Z' where 'X' is the number of cores per socket, 'Y' is the number of sockets and 'Z' is the number of threads per core. [ Error ] - The requested number of cores: [#!variable!requested_cores!#] (sockets: [#!variable!new_sockets!], cores per socket: [#!variable!new_cores!#], threads per core: [#!variable!new_threads!#]). + [ Error ] - This program must be run on a subnode. + [ Error ] - This subnode is not in the cluster (failed to parse the CIB). Exiting. @@ -3281,6 +3283,7 @@ proceeding. - Cores per Socket: [#!variable!old_cores!#] -> [#!variable!new_cores!#] - Threads per Core: [#!variable!old_threads!#] -> [#!variable!new_threads!#] - Total Cores: .... [#!variable!old_total_cores!#] -> [#!variable!new_total_cores!#]]]> + + ' to exit]]> Normal Password diff --git a/tools/anvil-watch-servers b/tools/anvil-watch-servers index 20a96b35..dbcf4acb 100755 --- a/tools/anvil-watch-servers +++ b/tools/anvil-watch-servers @@ -66,7 +66,7 @@ if ($anvil->data->{switches}{watch}) else { # No databases available. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "log_0738"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "log_0738"}); } sleep $interval; } @@ -89,25 +89,13 @@ sub show_status { my ($anvil) = @_; - if ($anvil->data->{switches}{watch}) - { - system('clear'); - print $t->Tgoto("cm", 0, 0); - } - - if ($anvil->data->{switches}{watch}) - { - my $date = $anvil->Get->date_and_time(); - print "-=] Updated: ".$date." - Press ' + ' to exit\n"; - } - ### TODO: Add support for checking/monitoring DR hosts # Get the node states my $host_type = $anvil->Get->host_type(); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }}); if ($host_type ne "node") { - print "This must be run on a subnode. Exiting.\n"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0478"}); $anvil->nice_exit({exit_code => 1}); } @@ -118,31 +106,36 @@ sub show_status $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); if ($problem) { - print "This subnode is not in the cluster (failed to parse the CIB). Exiting.\n"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0479"}); $anvil->nice_exit({exit_code => 1}); } -# foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{server}}) -# { -# print "DRBD Fence for Server: [".$server_name."]\n"; -# foreach my $node_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_node}}) -# { -# my $value = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_node}{$node_name}{value}; -# print "- Node: [".$node_name."], value: [".$value."]\n"; -# } -# } -# die; - # Load host information so that we can check for IPMI configs, if needed. $anvil->Database->get_hosts(); $anvil->Database->get_anvils(); $anvil->Database->get_servers(); + # It can take a second to collect the data, so we don't clear the screen until we're ready to + # display the new data + $anvil->data->{display}{status} = ""; show_servers($anvil); - print "\n"; - + $anvil->data->{display}{status} .= "\n"; show_nodes($anvil); + # Show the results. + if ($anvil->data->{switches}{watch}) + { + system('clear'); + print $t->Tgoto("cm", 0, 0); + } + + if ($anvil->data->{switches}{watch}) + { + my $date = $anvil->Get->date_and_time(); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0382", variables => { date => $date }}); + } + print $anvil->data->{display}{status}; + return(0); } @@ -151,7 +144,6 @@ sub show_servers my ($anvil) = @_; # Show the server states - $anvil->data->{'say'}{server_name} = $anvil->Words->string({key => "header_0121"}); $anvil->data->{'say'}{server_status} = $anvil->Words->string({key => "header_0122"}); $anvil->data->{'say'}{resource_status} = $anvil->Words->string({key => "header_0123"}); @@ -325,15 +317,142 @@ sub show_servers $longest_boot_time = length($say_boot_time); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_boot_time => $longest_boot_time }}); } - - + } + =cut Servers: +-------------+-----------------+-------------------+-------------+------------------+--------------+-----------+ | server name | | | | | | Boot Time | +-------------+-----------------+-------------------+-------------+------------------+--------------+-----------+ =cut + + # Now look again to show the subnode states + my $server_name_divider = ""; for (1..$longest_server_name) { $server_name_divider .= "-"; } + my $server_status_divider = ""; for (1..$longest_server_status) { $server_status_divider .= "-"; } + my $resource_status_divider = ""; for (1..$longest_resource_status) { $resource_status_divider .= "-"; } + my $host_name_divider = ""; for (1..$longest_host_name) { $host_name_divider .= "-"; } + my $preferred_host_divider = ""; for (1..$longest_preferred_host) { $preferred_host_divider .= "-"; } + my $drbd_fence_divider = ""; for (1..$longest_drbd_fence) { $drbd_fence_divider .= "-"; } + my $boot_time_divider = ""; for (1..$longest_boot_time) { $boot_time_divider .= "-"; } + + my $say_server_name_header = $anvil->Words->center_text({string => $anvil->data->{'say'}{server_name}, width => $longest_server_name}); + my $say_server_status_header = $anvil->Words->center_text({string => $anvil->data->{'say'}{server_status}, width => $longest_server_status}); + my $say_resource_status_header = $anvil->Words->center_text({string => $anvil->data->{'say'}{resource_status}, width => $longest_resource_status}); + my $say_host_name_header = $anvil->Words->center_text({string => $anvil->data->{'say'}{host_name}, width => $longest_host_name}); + my $say_preferred_host_header = $anvil->Words->center_text({string => $anvil->data->{'say'}{preferred_host}, width => $longest_preferred_host}); + my $say_drbd_fence_header = $anvil->Words->center_text({string => $anvil->data->{'say'}{drbd_fence}, width => $longest_drbd_fence}); + my $say_boot_time_header = $anvil->Words->center_text({string => $anvil->data->{'say'}{boot_time}, width => $longest_boot_time}); + + my $divider_line = "+-".$server_name_divider."-+-".$server_status_divider."-+-".$resource_status_divider."-+-".$host_name_divider."-+-".$preferred_host_divider."-+-".$drbd_fence_divider."-+-".$boot_time_divider."-+\n"; + $anvil->data->{display}{status} .= $divider_line; + $anvil->data->{display}{status} .= "| ".$say_server_name_header." | ".$say_server_status_header." | ".$say_resource_status_header." | ".$say_host_name_header." | ".$say_preferred_host_header." | ".$say_drbd_fence_header." | ".$say_boot_time_header." |\n"; + $anvil->data->{display}{status} .= $divider_line; + foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}}) + { + my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:server_name' => $server_name, + 's2:server_uuid' => $server_uuid, + }}); + + my $server_user_stop = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_user_stop}; + my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid}; + my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state}; + my $server_boot_time = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_boot_time}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:server_user_stop' => $server_user_stop, + 's2:server_host_uuid' => $server_host_uuid, + 's3:server_state' => $server_state, + 's4:server_boot_time' => $server_boot_time, + }}); + + # Database/virsh status + next if $server_state eq "DELETED"; + my $say_server_state = $anvil->data->{'say'}{unknown}; + if ($server_state eq "running") { $say_server_state = $anvil->data->{'say'}{running}; } + elsif ($server_state eq "blocked") { $say_server_state = $anvil->data->{'say'}{blocked}; } + elsif ($server_state eq "paused") { $say_server_state = $anvil->data->{'say'}{paused}; } + elsif ($server_state eq "in shutdown") { $say_server_state = $anvil->data->{'say'}{shutting_down}; } + elsif ($server_state eq "shut off") { $say_server_state = $anvil->data->{'say'}{shut_off}; } + elsif ($server_state eq "crashed") { $say_server_state = $anvil->data->{'say'}{crashed}; } + elsif ($server_state eq "pmsuspended") { $say_server_state = $anvil->data->{'say'}{pm_suspended}; } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_state => $server_state }}); + + # pcs resource status + my $resource_status = "--"; + my $is_failed = "--"; + my $say_resource_status = $anvil->data->{'say'}{unknown}; + if (exists $anvil->data->{cib}{parsed}{data}{server}{$server_name}) + { + $resource_status = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{status}; + $is_failed = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{failed}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + resource_status => $resource_status, + is_failed => $is_failed, + }}); + if ($resource_status eq "started") { $say_resource_status = $anvil->data->{'say'}{started}; } + elsif ($resource_status eq "starting") { $say_resource_status = $anvil->data->{'say'}{starting}; } + elsif ($resource_status eq "migrating") { $say_resource_status = $anvil->data->{'say'}{migrating}; } + elsif ($resource_status eq "stopping") { $say_resource_status = $anvil->data->{'say'}{stopping}; } + elsif ($resource_status eq "stopped") { $say_resource_status = $anvil->data->{'say'}{stopped}; } + else { $say_resource_status = $resource_status; } + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource_status => $resource_status }}); + + # Host node. + my $say_host_name = "--"; + if (($server_host_uuid) && ($server_host_uuid ne "NULL")) + { + $say_host_name = $anvil->data->{hosts}{host_uuid}{$server_host_uuid}{short_host_name}; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_host_name => $say_host_name }}); + + my $say_preferred_host = ""; + my $preferred_host_score = 0; + if (exists $anvil->data->{cib}{parsed}{data}{location_constraint}{$server_name}) + { + $say_preferred_host = $anvil->data->{cib}{parsed}{data}{location_constraint}{$server_name}{preferred_host}; + $preferred_host_score = $anvil->data->{cib}{parsed}{data}{location_constraint}{$server_name}{node}{$say_preferred_host}{score}; + $say_preferred_host .= " (".$preferred_host_score.")"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_preferred_host => $say_preferred_host }}); + } + + # DRBD fence + my $say_drbd_fence = "--"; + foreach my $node_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_node}}) + { + my $value = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_node}{$node_name}{value}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + node_name => $node_name, + value => $value, + }}); + if ($value eq "1") + { + $say_drbd_fence .= $node_name.","; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_drbd_fence => $say_drbd_fence }}); + } + } + $say_drbd_fence =~ s/,$//; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_drbd_fence => $say_drbd_fence }}); + + # Boot time + my $say_boot_time = "--"; + if ($server_boot_time) + { + $say_boot_time = $anvil->Get->date_and_time({use_time => $server_boot_time}); + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_boot_time => $say_boot_time }}); + + $anvil->data->{display}{status} .= "| "; + $anvil->data->{display}{status} .= sprintf("%-${longest_server_name}s", $server_name)." | "; + $anvil->data->{display}{status} .= sprintf("%-${longest_server_status}s", $say_server_state)." | "; + $anvil->data->{display}{status} .= sprintf("%-${longest_resource_status}s", $say_resource_status)." | "; + $anvil->data->{display}{status} .= sprintf("%-${longest_host_name}s", $say_host_name)." | "; + $anvil->data->{display}{status} .= sprintf("%-${longest_preferred_host}s", $say_preferred_host)." | "; + $anvil->data->{display}{status} .= sprintf("%-${longest_drbd_fence}s", $say_drbd_fence)." | "; + $anvil->data->{display}{status} .= sprintf("%-${longest_boot_time}s", $say_boot_time)." |\n"; } + $anvil->data->{display}{status} .= $divider_line; return(0); } @@ -479,9 +598,9 @@ Subnode Status: my $say_maintenance_mode_header = $anvil->Words->center_text({string => $anvil->data->{'say'}{maintenance_mode}, width => $longest_maintenance_mode}); my $divider_line = "+-".$subnode_divider."-+-".$host_status_divider."-+-".$pacemaker_status_divider."-+-".$maintenance_mode_divider."-+\n"; - print $divider_line; - print "| ".$say_subnode_header." | ".$say_host_status_header." | ".$say_pacemaker_status_header." | ".$say_maintenance_mode_header." |\n"; - print $divider_line; + $anvil->data->{display}{status} .= $divider_line; + $anvil->data->{display}{status} .= "| ".$say_subnode_header." | ".$say_host_status_header." | ".$say_pacemaker_status_header." | ".$say_maintenance_mode_header." |\n"; + $anvil->data->{display}{status} .= $divider_line; foreach my $node_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}}) { my $host_uuid = $anvil->Database->get_host_uuid_from_string({string => $node_name}); @@ -547,14 +666,14 @@ Subnode Status: say_maintenance_mode => $say_maintenance_mode, }}); - print "| "; - print sprintf("%-${longest_node_name}s", $node_name)." | "; - print sprintf("%-${longest_host_status}s", $say_host_status)." | "; - print sprintf("%-${longest_pacemaker_status}s", $say_pacemaker_status)." | "; - print sprintf("%-${longest_maintenance_mode}s", $say_maintenance_mode)." |\n"; + $anvil->data->{display}{status} .= "| "; + $anvil->data->{display}{status} .= sprintf("%-${longest_node_name}s", $node_name)." | "; + $anvil->data->{display}{status} .= sprintf("%-${longest_host_status}s", $say_host_status)." | "; + $anvil->data->{display}{status} .= sprintf("%-${longest_pacemaker_status}s", $say_pacemaker_status)." | "; + $anvil->data->{display}{status} .= sprintf("%-${longest_maintenance_mode}s", $say_maintenance_mode)." |\n"; } - print $divider_line; + $anvil->data->{display}{status} .= $divider_line; return(0); }