diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index 97ce3d4b..81a0cdc2 100644 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -1021,6 +1021,7 @@ sub _set_paths 'anvil.conf' => "/etc/anvil/anvil.conf", 'anvil.version' => "/etc/anvil/anvil.version", 'autoindex.conf' => "/etc/httpd/conf.d/autoindex.conf", + 'cib.xml' => "/var/lib/pacemaker/cib/cib.xml", 'corosync.conf' => "/etc/corosync/corosync.conf", 'dhcpd.conf' => "/etc/dhcp/dhcpd.conf", 'dnf.conf' => "/etc/dnf/dnf.conf", diff --git a/Anvil/Tools/Cluster.pm b/Anvil/Tools/Cluster.pm index c6dc5651..11e707d2 100644 --- a/Anvil/Tools/Cluster.pm +++ b/Anvil/Tools/Cluster.pm @@ -1935,6 +1935,15 @@ sub parse_cib "cib::parsed::cib::resources::primitive::${id}::type" => $anvil->data->{cib}{parsed}{cib}{resources}{primitive}{$id}{type}, "cib::parsed::cib::resources::primitive::${id}::class" => $anvil->data->{cib}{parsed}{cib}{resources}{primitive}{$id}{class}, }}); + + # If this is a stonith class, store the type as the 'agent' variable. + if ($anvil->data->{cib}{parsed}{cib}{resources}{primitive}{$id}{class} eq "stonith") + { + $anvil->data->{cib}{parsed}{data}{stonith}{primitive_id}{$id}{agent} = $anvil->data->{cib}{parsed}{cib}{resources}{primitive}{$id}{type}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "cib::parsed::data::stonith::primitive_id::${id}::agent" => $anvil->data->{cib}{parsed}{data}{stonith}{primitive_id}{$id}{agent}, + }}); + } foreach my $nvpair ($primitive->findnodes('./instance_attributes/nvpair')) { my $nvpair_id = $nvpair->{id}; @@ -2112,6 +2121,7 @@ sub parse_cib foreach my $primitive_id (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{cib}{resources}{primitive}}) { next if not $anvil->data->{cib}{parsed}{cib}{resources}{primitive}{$primitive_id}{class}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { primitive_id => $primitive_id }}); if ($anvil->data->{cib}{parsed}{cib}{resources}{primitive}{$primitive_id}{class} eq "stonith") { my $variables = {}; @@ -2121,8 +2131,9 @@ sub parse_cib my $name = $anvil->data->{cib}{parsed}{cib}{resources}{primitive}{$primitive_id}{instance_attributes}{$fence_id}{name}; my $value = $anvil->data->{cib}{parsed}{cib}{resources}{primitive}{$primitive_id}{instance_attributes}{$fence_id}{value}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - name => $name, - value => $value, + 's1:fence_id' => $fence_id, + 's2:name' => $name, + 's3:value' => $value, }}); if ($name eq "pcmk_host_list") @@ -2419,6 +2430,12 @@ sub parse_crm_mon } else { + # When called on Striker during post-scan analysis, this won't work. So to avoid noise in the + # logs, we do en explicit check if the binary exists and exit quietly if it does not. + if (not -e $anvil->data->{path}{exe}{crm_mon}) + { + return(1); + } my $shell_call = $anvil->data->{path}{exe}{crm_mon}." --output-as=xml"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }}); diff --git a/Anvil/Tools/DRBD.pm b/Anvil/Tools/DRBD.pm index 5f033578..cedd6647 100644 --- a/Anvil/Tools/DRBD.pm +++ b/Anvil/Tools/DRBD.pm @@ -731,19 +731,24 @@ sub gather_data "new::resource::${resource}::volume::${volume}::peer::${peer}::peer_role" => $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{peer_role}, }}); - # If the peer is secondary, read the device size. - if ($anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{peer_role} eq "secondary") + # Get the resource size by reading '/sys/block/drbd/size' and multiplying by '/sys/block//queue/logical_block_size' + my $drbd_device = "/sys/block/drbd".$anvil->data->{new}{resource}{$resource}{volume}{$volume}{device_minor}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_device => $drbd_device }}); + if (-d $drbd_device) { - # Get the size of the DRBD device. - my ($size, $return_code) = $anvil->System->call({secure => 1, shell_call => $anvil->data->{path}{exe}{blockdev}." --getsize64 /dev/drbd".$anvil->data->{new}{resource}{$resource}{volume}{$volume}{device_minor}}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - size => $size, - return_code => $return_code, + my $logical_block_size = $anvil->Words->clean_spaces({string => $anvil->Storage->read_file({file => $drbd_device."/queue/logical_block_size"})}); + my $sector_size = $anvil->Words->clean_spaces({string => $anvil->Storage->read_file({file => $drbd_device."/size"})}); + my $size = $logical_block_size * $sector_size; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + logical_block_size => $anvil->Convert->add_commas({number => $logical_block_size}), + sector_size => $anvil->Convert->add_commas({number => $sector_size}), + size => $anvil->Convert->add_commas({number => $size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $size}).")", }}); - if (not $return_code) + + if ($size > 0) { $anvil->data->{new}{resource}{$resource}{volume}{$volume}{size} = $size; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "new::resource::${resource}::volume::${volume}::size" => $anvil->data->{new}{resource}{$resource}{volume}{$volume}{size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{new}{resource}{$resource}{volume}{$volume}{size}}).")", }}); } @@ -765,22 +770,7 @@ sub gather_data "new::resource::${resource}::volume::${volume}::peer::${peer}::out_of_sync_size" => $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{out_of_sync_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{out_of_sync_size}}).")", }}); } -=cut - 0: cs:Established ro:Secondary/Secondary ds:Inconsistent/Inconsistent C r----- - ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:[0;0] ua:0 ap:[0;0] ep:1 wo:1 oos:0 - resync: used:0/61 hits:0 misses:0 starving:0 locked:0 changed:0 - act_log: used:0/1237 hits:0 misses:0 starving:0 locked:0 changed:0 - blocked on activity log: 0/0/0 - - 0: cs:SyncTarget ro:Secondary/Primary ds:Inconsistent/UpToDate C r----- - ns:0 nr:648960 dw:648728 dr:0 al:0 bm:0 lo:4 pe:[0;1] ua:4 ap:[0;0] ep:1 wo:1 oos:20321476 - [>....................] sync'ed: 3.2% (19844/20476)M - finish: 0:03:39 speed: 92,672 (92,936 -- 92,672) want: 2,880 K/sec - 3% sector pos: 1298032/41940408 - resync: used:1/61 hits:31926 misses:10 starving:0 locked:0 changed:5 - act_log: used:0/1237 hits:0 misses:0 starving:0 locked:0 changed:0 - blocked on activity log: 0/0/0 -=cut + if ($line =~ /sync'ed:\s+(\d.*\%)/) { $progress .= $1; @@ -1565,7 +1555,7 @@ sub get_status $anvil->data->{drbd}{status}{$host}{resource}{$resource}{devices}{volume}{$volume}{size} = $hash_ref->{devices}->[$i]->{size}; $anvil->data->{drbd}{status}{$host}{resource}{$resource}{devices}{volume}{$volume}{'upper-pending'} = $hash_ref->{devices}->[$i]->{'upper-pending'}; $anvil->data->{drbd}{status}{$host}{resource}{$resource}{devices}{volume}{$volume}{written} = $hash_ref->{devices}->[$i]->{written}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "drbd::status::${host}::resource::${resource}::devices::volume::${volume}::al-writes" => $anvil->data->{drbd}{status}{$host}{resource}{$resource}{devices}{volume}{$volume}{'al-writes'}, "drbd::status::${host}::resource::${resource}::devices::volume::${volume}::bm-writes" => $anvil->data->{drbd}{status}{$host}{resource}{$resource}{devices}{volume}{$volume}{'bm-writes'}, "drbd::status::${host}::resource::${resource}::devices::volume::${volume}::client" => $anvil->data->{drbd}{status}{$host}{resource}{$resource}{devices}{volume}{$volume}{client}, diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index e2141ac2..fcec70e7 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -87,6 +87,7 @@ my $THIS_FILE = "Database.pm"; # read_variable # refresh_timestamp # resync_databases +# update_host_status # write # _archive_table # _find_behind_database @@ -2832,17 +2833,20 @@ FROM host_ipmi => $anvil->Log->is_secure($host_ipmi), host_status => $host_status, }}); - $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_name} = $host_name; - $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_type} = $host_type; - $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_key} = $host_key; - $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_ipmi} = $host_ipmi; - $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_status} = $host_status; + $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_name} = $host_name; + $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{short_host_name} = $host_name; + $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{short_host_name} =~ s/\..*$//; + $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_type} = $host_type; + $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_key} = $host_key; + $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_ipmi} = $host_ipmi; + $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_status} = $host_status; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "machine::host_uuid::${host_uuid}::hosts::host_name" => $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_name}, - "machine::host_uuid::${host_uuid}::hosts::host_type" => $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_type}, - "machine::host_uuid::${host_uuid}::hosts::host_key" => $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_key}, - "machine::host_uuid::${host_uuid}::hosts::host_ipmi" => $anvil->Log->is_secure($anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_ipmi}), - "machine::host_uuid::${host_uuid}::hosts::host_status" => $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_status}, + "machine::host_uuid::${host_uuid}::hosts::host_name" => $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_name}, + "machine::host_uuid::${host_uuid}::hosts::short_host_name" => $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{short_host_name}, + "machine::host_uuid::${host_uuid}::hosts::host_type" => $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_type}, + "machine::host_uuid::${host_uuid}::hosts::host_key" => $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_key}, + "machine::host_uuid::${host_uuid}::hosts::host_ipmi" => $anvil->Log->is_secure($anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_ipmi}), + "machine::host_uuid::${host_uuid}::hosts::host_status" => $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_status}, }}); # If this is an Anvil! member, pull it's IP. @@ -6843,7 +6847,8 @@ This is the power state of the host. Valid values are; * C<< unknown >> - This should only be set when a node can not be reached and the previous setting was not C<< stopping >> or C<< booting >>. * C<< powered off >> - This shoule be set only when the host is confirmed off via IPMI call * C<< online >> - This is set by the host itself when it boots up and first connects to the anvil database. B<< Note >> - This does NOT indicate readiness! Only that the host is accessible -* C<< stopping >> - This is a transitional state set by the host when it begins powering off. +* C<< rebooting >> - This is a transitional state set by the host when it begins a reboot. The next step is 'online'. +* C<< stopping >> - This is a transitional state set by the host when it begins powering off. The next step is 'powered off' and will be set by a Striker. Note that if there is no host IPMI, the may stay in this state until in next powers on. * C<< booting >> - This is a transitional state set by a Striker dashboard when it is powering on a host. B<< Note >> - Given that most Striker dashboards do not have IPMI, it is expected that they will enter C<< stopping >> state and never transition to c<< powered off >>. This is OK as C<< powered off >> can only be set when a target is B<< confirmed >> off. There's no other way to ensure that a target is not stuck while shutting down. Lack of pings doesn't solve this, either, as the network can go down before the host powers off. @@ -14935,6 +14940,58 @@ sub resync_databases } +=head2 update_host_status + +This is a variant on C<< insert_or_update_hosts >> designed only to update the power status of a host. + +Parameters; + +=head3 host_uuid (optional, default Get->host_uuid) + +This is the host whose power state is being updated. + +=head3 host_status (required) + +This is the host status to set. See C<< insert_or_update_hosts -> host_status >> for valid values. + +=cut +sub update_host_status +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->update_host_status()" }}); + + my $host_uuid = defined $parameter->{host_uuid} ? $parameter->{host_uuid} : $anvil->Get->host_uuid; + my $host_status = defined $parameter->{host_status} ? $parameter->{host_status} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + host_uuid => $host_uuid, + host_status => $host_status, + }}); + + if (not $host_status) + { + # Throw an error and exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->update_host_status()", parameter => "host_status" }}); + return(""); + } + + # We're only updating the status, so we'll read in the current data to pass back in. + $anvil->Database->get_hosts({debug => $debug}); + $anvil->Database->insert_or_update_hosts({ + host_ipmi => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}, + host_key => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, + host_name => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, + host_type => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type}, + host_uuid => $host_uuid, + host_status => $host_status, + }); + + return(0); +} + + =head2 write This records data to one or all of the databases. If a UUID is passed, the query is written to one database only. Otherwise, it will be written to all DBs. diff --git a/Anvil/Tools/Network.pm b/Anvil/Tools/Network.pm index 6543d71a..84be8449 100644 --- a/Anvil/Tools/Network.pm +++ b/Anvil/Tools/Network.pm @@ -1236,7 +1236,7 @@ sub load_ips $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Network->load_ips()" }}); my $clear = defined $parameter->{clear} ? $parameter->{clear} : 1; - my $host_uuid = defined $parameter->{host_uuid} ? $parameter->{host_uuid} : $anvil->data->{sys}{host_uuid}; + my $host_uuid = defined $parameter->{host_uuid} ? $parameter->{host_uuid} : $anvil->Get->host_uuid; my $host = defined $parameter->{host} ? $parameter->{host} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { clear => $clear, diff --git a/Anvil/Tools/ScanCore.pm b/Anvil/Tools/ScanCore.pm index 2e97a569..2e9dbe07 100644 --- a/Anvil/Tools/ScanCore.pm +++ b/Anvil/Tools/ScanCore.pm @@ -647,47 +647,109 @@ sub post_scan_analysis_striker # set, or when a DR host is scheduled to boot. $anvil->Database->get_hosts_info({debug => $debug}); + # Load our IP information. + my $short_host_name = $anvil->Get->short_host_name; + $anvil->Network->load_ips({ + debug => 2, + clear => 1, + host_uuid => $anvil->Get->host_uuid, + }); + # Get a look at all nodes and DR hosts. For each, check if they're up. foreach my $host_uuid (keys %{$anvil->data->{machine}{host_uuid}}) { - # Compile data. - my $host_name = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_name}; - my $host_type = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_type}; - my $host_key = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_key}; - my $host_ipmi = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_ipmi}; - my $host_status = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_status}; - my $password = $anvil->data->{machine}{host_uuid}{$host_uuid}{password}; - my $anvil_name = $anvil->data->{machine}{host_uuid}{$host_uuid}{anvil}{name}; - my $anvil_uuid = $anvil->data->{machine}{host_uuid}{$host_uuid}{anvil}{uuid}; - my $anvil_role = $anvil->data->{machine}{host_uuid}{$host_uuid}{anvil}{role}; + # Skip outself + next if $host_uuid eq $anvil->Get->host_uuid(); + + # Compile host's data. + my $host_name = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_name}; + my $short_host_name = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_name}; + my $host_type = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_type}; + my $host_key = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_key}; + my $host_ipmi = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_ipmi}; + my $host_status = $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_status}; + my $password = $anvil->data->{machine}{host_uuid}{$host_uuid}{password}; + my $anvil_name = $anvil->data->{machine}{host_uuid}{$host_uuid}{anvil}{name}; + my $anvil_uuid = $anvil->data->{machine}{host_uuid}{$host_uuid}{anvil}{uuid}; + my $anvil_role = $anvil->data->{machine}{host_uuid}{$host_uuid}{anvil}{role}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - host_name => $host_name, - host_type => $host_type, - host_key => $host_key, - host_ipmi => $anvil->Log->is_secure($host_ipmi), - host_status => $host_status, - password => $anvil->Log->is_secure($password), - anvil_name => $anvil_name, - anvil_uuid => $anvil_uuid, - anvil_role => $anvil_role, + host_name => $host_name, + short_host_name => $short_host_name, + host_type => $host_type, + host_key => $host_key, + host_ipmi => $anvil->Log->is_secure($host_ipmi), + host_status => $host_status, + password => $anvil->Log->is_secure($password), + anvil_name => $anvil_name, + anvil_uuid => $anvil_uuid, + anvil_role => $anvil_role, }}); - ### TODO: Add an ability to mark which PDU powers a striker. If set, try logging into the - ### peer striker and if it fails, power cycle it (but only once per hour). - next if $host_type eq "striker"; - - ### TODO: Adding support for PDU resets would allow us to recover from crashed IPMI BMCs as - ### well. For now though, not 'host_ipmi' means there's nothing we can do. - if (not $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_ipmi}) + # Check to see when the last 'updated' entry was from, and it if was less than 60 seconds + # ago, skip this machine as it's likely on. + my $query = " +SELECT + round(extract(epoch from modified_date)) +FROM + updated +WHERE + updated_host_uuid = ".$anvil->Database->quote($host_uuid)." +ORDER BY + modified_date DESC +LIMIT 1;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + my $last_update = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $last_update = 0 if not defined $last_update; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { last_update => $last_update }}); + if (not $last_update) { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0560", variables => { host_name => $host_name }}); + # This machine isn't running ScanCore yet, skip it. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0597", variables => { host_name => $host_name }}); next; } + else + { + my $last_update_age = time - $last_update; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { last_update_age => $last_update_age }}); + + if ($last_update_age < 120) + { + # It was alive less than two minutes ago, we don't need to check anything. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0596", variables => { + host_name => $host_name, + difference => $last_update_age, + }}); + next; + } + } + + # Read in the unified fence data, if it's not already loaded. + my $update_fence_data = 1; + if ($anvil->data->{fence_data}{updated}) + { + my $age = time - $anvil->data->{fence_data}{updated}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { age => $age }}); + if ($age < 86400) + { + # Only refresh daily. + $update_fence_data = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { update_fence_data => $update_fence_data }}); + } + } + if ($update_fence_data) + { + $anvil->Striker->get_fence_data({debug => $debug}); + } # Check this target's power state. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0561", variables => { host_name => $host_name }}); # Do we share a network with this system? + $anvil->Network->load_ips({ + debug => 2, + clear => 1, + host_uuid => $host_uuid, + }); my $check_power = 1; my $match = $anvil->Network->find_matches({ debug => $debug, @@ -699,12 +761,11 @@ sub post_scan_analysis_striker if (not $matched_ips) { # nothing we can do with this host. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0558", variables => { host_name => $host_name }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0558", variables => { host_name => $host_name }}); next; } foreach my $interface (sort {$a cmp $b} keys %{$match->{$host_uuid}}) { - next; my $ip_address = $match->{$host_uuid}{$interface}{ip}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 's1:interface' => $interface, @@ -733,9 +794,23 @@ sub post_scan_analysis_striker if ($access) { # It's up. - $check_power = 0; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { check_power => $check_power }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0562", variables => { host_name => $host_name }}); + + $check_power = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + check_power => $check_power, + host_status => $host_status, + }}); + + # If the host_status is 'booting' or 'unknown', change it to online. + if (($host_status eq "booting") or ($host_status eq "unknown")) + { + $anvil->Database->update_host_status({ + debug => $debug, + host_uuid => $host_uuid, + host_status => "online", + }); + } last; } } @@ -748,8 +823,101 @@ sub post_scan_analysis_striker } # Do we have IPMI info? - if (not $host_ipmi) + if ((not $host_ipmi) && ($host_type eq "node") && ($anvil_uuid)) { + # No host IPMI (that we know of). Can we check using another (non PDU) fence method? + my $query = "SELECT scan_cluster_cib FROM scan_cluster WHERE scan_cluster_anvil_uuid = ".$anvil->Database->quote($anvil_uuid).";"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + my $scan_cluster_cib = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $scan_cluster_cib = "" if not defined $scan_cluster_cib; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { scan_cluster_cib => $scan_cluster_cib }}); + if ($scan_cluster_cib) + { + # Parse out the fence methods for this host. + my $problem = $anvil->Cluster->parse_cib({ + debug => $debug, + cib => $scan_cluster_cib, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }}); + if (not $problem) + { + # Parsed! Do we have a fence method we can trust to check the power + # state of this node? + my $node_name = exists $anvil->data->{cib}{parsed}{data}{node}{$short_host_name} ? $short_host_name : $host_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { node_name => $node_name }}); + foreach my $order (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{order}}) + { + my $method = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{order}{$order}{devices}; + my $agent = $anvil->data->{cib}{parsed}{data}{stonith}{primitive_id}{$method}{agent}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:order' => $order, + 's2:method' => $method, + 's3:agent' => $agent + }}); + + # We can't trust a PDU's output, so skip them. + next if $agent =~ /pdu/; + + my $shell_call = $agent." "; + foreach my $stdin_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$method}{argument}}) + { + next if $stdin_name =~ /pcmk_o\w+_action/; + my $switch = ""; + my $value = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$method}{argument}{$stdin_name}{value}; + + foreach my $this_switch (sort {$a cmp $b} keys %{$anvil->data->{fence_data}{$agent}{switch}}) + { + my $this_name = $anvil->data->{fence_data}{$agent}{switch}{$this_switch}{name}; + if ($stdin_name eq $this_name) + { + $switch = $this_switch; + my $dashes = (length($switch) > 1) ? "--" : "-"; + $shell_call .= $dashes.$switch." \"".$value."\" "; + last; + } + } + if (not $switch) + { + if ($anvil->data->{fence_data}{$agent}{switch}{$stdin_name}{name}) + { + my $dashes = (length($stdin_name) > 1) ? "--" : "-"; + $shell_call .= $dashes.$stdin_name." \"".$value."\" "; + } + } + } + $shell_call .= "--action status"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); + + my ($output, $return_code) = $anvil->System->call({debug => $debug, timeout => 30, shell_call => $shell_call}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); + foreach my $line (split/\n/, $output) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }}); + } + + if ($return_code eq "2") + { + # Node is off. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0564", variables => { host_name => $host_name }}); + $anvil->Database->update_host_status({ + debug => $debug, + host_uuid => $host_uuid, + host_status => "powered off", + }); + } + elsif ($return_code eq "0") + { + # Node is on. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0563", variables => { host_name => $host_name }}); + next; + } + } + } + } + ### TODO: Add support for power-cycling a target using PDUs. Until this, this ### will never be hit as we next on no host_ipmi, but will be useful ### when PDU support is added. @@ -758,6 +926,13 @@ sub post_scan_analysis_striker next; } + # If we're here and there's no host IPMI information, there's nothing we can do. + if (not $anvil->data->{machine}{host_uuid}{$host_uuid}{hosts}{host_ipmi}) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0560", variables => { host_name => $host_name }}); + next; + } + # Check the power state. my $shell_call = $host_ipmi; $shell_call =~ s/--action status//; @@ -775,6 +950,11 @@ sub post_scan_analysis_striker { # Node is off. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0564", variables => { host_name => $host_name }}); + $anvil->Database->update_host_status({ + debug => $debug, + host_uuid => $host_uuid, + host_status => "powered off", + }); } elsif ($return_code eq "0") { @@ -786,7 +966,7 @@ sub post_scan_analysis_striker # Still here? See if we know why the node is off. my $boot_target = 0; my $stop_reason = "unknown"; - my $query = " + $query = " SELECT variable_value FROM @@ -864,6 +1044,13 @@ AND $shell_call =~ s/--action status/ --action on/; my ($output, $return_code) = $anvil->System->call({debug => $debug, timeout => 30, shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }}); + + # Mark it as booting. + $anvil->Database->update_host_status({ + debug => $debug, + host_uuid => $host_uuid, + host_status => "booting", + }); } } } @@ -943,6 +1130,13 @@ AND $shell_call =~ s/--action status/ --action on/; my ($output, $return_code) = $anvil->System->call({debug => $debug, timeout => 30, shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }}); + + # Mark it as booting. + $anvil->Database->update_host_status({ + debug => $debug, + host_uuid => $host_uuid, + host_status => "booting", + }); } } } diff --git a/Anvil/Tools/Server.pm b/Anvil/Tools/Server.pm index 29717a32..9b4f5aeb 100644 --- a/Anvil/Tools/Server.pm +++ b/Anvil/Tools/Server.pm @@ -1262,7 +1262,7 @@ sub parse_definition }}); foreach my $hash_ref (@{$server_xml->{cpu}->[0]->{feature}}) { - my $name = $hash_ref->{name}; + my $name = $hash_ref->{name}; $anvil->data->{server}{$target}{$server}{$source}{cpu}{feature}{$name} = $hash_ref->{policy}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "server::${target}::${server}::${source}::cpu::feature::${name}" => $anvil->data->{server}{$target}{$server}{$source}{cpu}{feature}{$name}, diff --git a/Anvil/Tools/Striker.pm b/Anvil/Tools/Striker.pm index a355b898..e0998295 100644 --- a/Anvil/Tools/Striker.pm +++ b/Anvil/Tools/Striker.pm @@ -458,7 +458,10 @@ sub get_fence_data $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { action => $action }}); } } - + + # ScanCore will load this to check nodes that are not accessible. To reduce load, as this is an + # expensive call, this time is set so a caller can decide if the data should be updated. + $anvil->data->{fence_data}{updated} = time; return(0); } diff --git a/cgi-bin/get_cpu b/cgi-bin/get_cpu index e69de29b..18bf00f9 100755 --- a/cgi-bin/get_cpu +++ b/cgi-bin/get_cpu @@ -0,0 +1,188 @@ +#!/usr/bin/perl +# +# This prints JSON formated data reporting the status of CPUs. +# + +use strict; +use warnings; +use Anvil::Tools; +use Data::Dumper; +use JSON; + +$| = 1; + +my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; +my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; +if (($running_directory =~ /^\./) && ($ENV{PWD})) +{ + $running_directory =~ s/^\./$ENV{PWD}/; +} + +my $anvil = Anvil::Tools->new(); + +$anvil->Get->switches; + +$anvil->Database->connect; +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); +if (not $anvil->data->{sys}{database}{connections}) +{ + # No databases, exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003"}); + $anvil->nice_exit({exit_code => 1}); +} + +# Read in any CGI variables, if needed. +$anvil->Get->cgi(); + +$anvil->Database->get_hosts(); +$anvil->Database->get_anvils(); + +print $anvil->Template->get({file => "shared.html", name => "json_headers", show_name => 0})."\n"; + +my $target = $anvil->Get->short_host_name(); +my $hash = {}; +my $anvil_uuid = ""; +if ($anvil->data->{cgi}{anvil_uuid}{value}) +{ + $anvil_uuid = $anvil->data->{cgi}{anvil_uuid}{value}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); +} +elsif ($anvil->data->{switches}{'anvil-uuid'}) +{ + $anvil_uuid = $anvil->data->{switches}{'anvil-uuid'}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); +} +if ((not $anvil_uuid) or (not exists $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid})) +{ + $anvil->data->{anvil_status}{anvil_name} = "!!invalid!anvil_uuid!!"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'anvil_status::anvil_name' => $anvil->data->{anvil_status}{anvil_name} }}); +} +else +{ + my $node1_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; + my $node2_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; + my $dr1_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + node1_uuid => $node1_uuid, + node2_uuid => $node2_uuid, + dr1_uuid => $dr1_uuid, + }}); + + $hash->{cores} = 0; + $hash->{threads} = 0; + $hash->{allocated} = 0; + + # Do the query + my $query = " +SELECT + a.host_uuid, + a.host_name, + b.scan_hardware_cpu_cores, + b.scan_hardware_cpu_threads +FROM + hosts a, scan_hardware b +WHERE + a.host_uuid = b.scan_hardware_host_uuid +AND + ( + a.host_uuid = ".$anvil->Database->quote($node1_uuid)." + OR + a.host_uuid = ".$anvil->Database->quote($node2_uuid); + if ($dr1_uuid) + { + $query .= " + OR + a.host_uuid = ".$anvil->Database->quote($dr1_uuid); + } + $query .= " + ) +ORDER BY + a.host_name ASC +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $host_uuid = $row->[0]; + my $host_name = $row->[1]; + my $scan_hardware_cpu_cores = $row->[2]; + my $scan_hardware_cpu_threads = $row->[3]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + host_uuid => $host_uuid, + host_name => $host_name, + scan_hardware_cpu_cores => $scan_hardware_cpu_cores, + scan_hardware_cpu_threads => $scan_hardware_cpu_threads, + }}); + + if (not $hash->{cores}) + { + $hash->{cores} = $scan_hardware_cpu_cores; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'hash->cores' => $hash->{cores} }}); + } + elsif ($scan_hardware_cpu_cores < $hash->{cores}) + { + $hash->{cores} = $scan_hardware_cpu_cores; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'hash->cores' => $hash->{cores} }}); + } + if (not $hash->{threads}) + { + $hash->{threads} = $scan_hardware_cpu_threads; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'hash->threads' => $hash->{threads} }}); + } + elsif ($scan_hardware_cpu_threads < $hash->{threads}) + { + $hash->{threads} = $scan_hardware_cpu_threads; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'hash->threads' => $hash->{threads} }}); + } + } + + # Now get the servers from the Anvil! + $query = " +SELECT + a.server_uuid, + a.server_name, + b.server_definition_xml +FROM + servers a, + server_definitions b +WHERE + a.server_uuid = b.server_definition_server_uuid +AND + a.server_anvil_uuid = ".$anvil->Database->quote($anvil_uuid)." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $server_uuid = $row->[0]; + my $server_name = $row->[1]; + my $server_definition_xml = $row->[2]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:server_name' => $server_name, + 's2:server_uuid' => $server_uuid, + 's3:server_definition_xml' => $server_definition_xml, + }}); + + $anvil->Server->parse_definition({ + server => $server_name, + source => "from_db", + definition => $server_definition_xml, + }); + + $hash->{allocated} += $anvil->data->{server}{$target}{$server_name}{from_db}{cpu}{total_cores}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'hash->allocated' => $hash->{allocated} }}); + } + +} + +print JSON->new->utf8->encode($hash)."\n"; diff --git a/cgi-bin/get_memory b/cgi-bin/get_memory index e69de29b..bf91b713 100755 --- a/cgi-bin/get_memory +++ b/cgi-bin/get_memory @@ -0,0 +1,189 @@ +#!/usr/bin/perl +# +# This prints JSON formated data reporting the status of RAM on the system. +# + +use strict; +use warnings; +use Anvil::Tools; +use Data::Dumper; +use JSON; + +$| = 1; + +my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; +my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; +if (($running_directory =~ /^\./) && ($ENV{PWD})) +{ + $running_directory =~ s/^\./$ENV{PWD}/; +} + +my $anvil = Anvil::Tools->new(); + +$anvil->Get->switches; + +$anvil->Database->connect; +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); +if (not $anvil->data->{sys}{database}{connections}) +{ + # No databases, exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003"}); + $anvil->nice_exit({exit_code => 1}); +} + +# Read in any CGI variables, if needed. +$anvil->Get->cgi(); + +$anvil->Database->get_hosts(); +$anvil->Database->get_anvils(); + +print $anvil->Template->get({file => "shared.html", name => "json_headers", show_name => 0})."\n"; + +my $target = $anvil->Get->short_host_name(); +my $hash = {}; +my $anvil_uuid = ""; +if ($anvil->data->{cgi}{anvil_uuid}{value}) +{ + $anvil_uuid = $anvil->data->{cgi}{anvil_uuid}{value}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); +} +elsif ($anvil->data->{switches}{'anvil-uuid'}) +{ + $anvil_uuid = $anvil->data->{switches}{'anvil-uuid'}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); +} +if ((not $anvil_uuid) or (not exists $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid})) +{ + $anvil->data->{anvil_status}{anvil_name} = "!!invalid!anvil_uuid!!"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'anvil_status::anvil_name' => $anvil->data->{anvil_status}{anvil_name} }}); +} +else +{ + my $node1_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; + my $node2_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; + my $dr1_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + node1_uuid => $node1_uuid, + node2_uuid => $node2_uuid, + dr1_uuid => $dr1_uuid, + }}); + + $hash->{total} = 0; + $hash->{allocated} = 0; + $hash->{nodes} = []; + + my @hosts = ($node1_uuid, $node2_uuid); + if ($dr1_uuid) + { + push @hosts, $dr1_uuid; + $anvil->data->{raw}{newest_record}{$dr1_uuid} = 0; + } + + foreach my $host_uuid (@hosts) + { + # Do the query + my $query = " +SELECT + a.host_name, + b.scan_hardware_ram_total, + b.scan_hardware_memory_free, + b.scan_hardware_swap_total, + b.scan_hardware_swap_free +FROM + hosts a, scan_hardware b +WHERE + a.host_uuid = b.scan_hardware_host_uuid +AND + a.host_uuid = ".$anvil->Database->quote($host_uuid)." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $host_name = $row->[0]; + my $scan_hardware_ram_total = $row->[1]; + my $scan_hardware_memory_free = $row->[2]; + my $scan_hardware_swap_total = $row->[3]; + my $scan_hardware_swap_free = $row->[4]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + host_name => $host_name, + scan_hardware_ram_total => $anvil->Convert->add_commas({number => $scan_hardware_ram_total})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $scan_hardware_ram_total}).")", + scan_hardware_memory_free => $anvil->Convert->add_commas({number => $scan_hardware_memory_free})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $scan_hardware_memory_free}).")", + scan_hardware_swap_total => $anvil->Convert->add_commas({number => $scan_hardware_swap_total})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $scan_hardware_swap_total}).")", + scan_hardware_swap_free => $anvil->Convert->add_commas({number => $scan_hardware_swap_free})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $scan_hardware_swap_free}).")", + }}); + + + if (not $hash->{total}) + { + $hash->{total} = $scan_hardware_ram_total; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'hash->total' => $hash->{total} }}); + } + elsif ($scan_hardware_ram_total < $hash->{total}) + { + $hash->{total} = $scan_hardware_ram_total; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'hash->total' => $hash->{total} }}); + } + + push @{$hash->{nodes}}, { + host_uuid => $host_uuid, + total => $scan_hardware_ram_total, + free => $scan_hardware_memory_free, + swap_total => $scan_hardware_swap_total, + swap_used => $scan_hardware_swap_free, + }; + } + } + + # Now get the servers from the Anvil! + my $query = " +SELECT + a.server_uuid, + a.server_name, + b.server_definition_xml +FROM + servers a, + server_definitions b +WHERE + a.server_uuid = b.server_definition_server_uuid +AND + a.server_anvil_uuid = ".$anvil->Database->quote($anvil_uuid)." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $server_uuid = $row->[0]; + my $server_name = $row->[1]; + my $server_definition_xml = $row->[2]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:server_name' => $server_name, + 's2:server_uuid' => $server_uuid, + 's3:server_definition_xml' => $server_definition_xml, + }}); + + $anvil->Server->parse_definition({ + server => $server_name, + source => "from_db", + definition => $server_definition_xml, + }); + + $hash->{allocated} += $anvil->data->{server}{$target}{$server_name}{from_db}{memory}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 'hash->allocated' => $anvil->Convert->add_commas({number => $hash->{allocated}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $hash->{allocated}}).")", + }}); + } + +} + +print JSON->new->utf8->encode($hash)."\n"; diff --git a/cgi-bin/get_replicated_storage b/cgi-bin/get_replicated_storage index e69de29b..dc7037ee 100755 --- a/cgi-bin/get_replicated_storage +++ b/cgi-bin/get_replicated_storage @@ -0,0 +1,304 @@ +#!/usr/bin/perl +# +# This prints JSON formated data reporting the status of DRBD resources and volumes. +# + +use strict; +use warnings; +use Anvil::Tools; +use Data::Dumper; +use JSON; + +$| = 1; + +my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; +my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; +if (($running_directory =~ /^\./) && ($ENV{PWD})) +{ + $running_directory =~ s/^\./$ENV{PWD}/; +} + +my $anvil = Anvil::Tools->new(); + +$anvil->Get->switches; + +$anvil->Database->connect; +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); +if (not $anvil->data->{sys}{database}{connections}) +{ + # No databases, exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003"}); + $anvil->nice_exit({exit_code => 1}); +} + +# Read in any CGI variables, if needed. +$anvil->Get->cgi(); + +$anvil->Database->get_hosts(); +$anvil->Database->get_anvils(); +$anvil->DRBD->gather_data(); + +print $anvil->Template->get({file => "shared.html", name => "json_headers", show_name => 0})."\n"; + +my $hash = {}; +my $anvil_uuid = ""; +my $active_resource = ""; +my $volume_array = ""; +my $connection_array = ""; +my $target_array = ""; + $hash->{resources} = []; +if ($anvil->data->{cgi}{anvil_uuid}{value}) +{ + $anvil_uuid = $anvil->data->{cgi}{anvil_uuid}{value}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); +} +elsif ($anvil->data->{switches}{'anvil-uuid'}) +{ + $anvil_uuid = $anvil->data->{switches}{'anvil-uuid'}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); +} +if ((not $anvil_uuid) or (not exists $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid})) +{ + $anvil->data->{anvil_status}{anvil_name} = "!!invalid!anvil_uuid!!"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'anvil_status::anvil_name' => $anvil->data->{anvil_status}{anvil_name} }}); +} +else +{ + my $node1_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; + my $node2_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; + my $dr1_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + node1_uuid => $node1_uuid, + node2_uuid => $node2_uuid, + dr1_uuid => $dr1_uuid, + }}); + + $anvil->data->{raw}{newest_record}{$node1_uuid} = 0; + $anvil->data->{raw}{newest_record}{$node2_uuid} = 0; + my @hosts = ($node1_uuid, $node2_uuid); + if ($dr1_uuid) + { + push @hosts, $dr1_uuid; + $anvil->data->{raw}{newest_record}{$dr1_uuid} = 0; + } + + foreach my $host_uuid (@hosts) + { + my $host_name = $anvil->Get->host_name_from_uuid({host_uuid => $host_uuid}); + my $short_host_name = $host_name; + $short_host_name =~ s/\..*$//; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + host_uuid => $host_uuid, + host_name => $host_name, + short_host_name => $short_host_name + }}); + my $query = " +SELECT + scan_drbd_resource_uuid, + scan_drbd_resource_name, + scan_drbd_resource_up, + round(extract(epoch from modified_date)) +FROM + scan_drbd_resources +WHERE + scan_drbd_resource_host_uuid = ".$anvil->Database->quote($host_uuid)." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $scan_drbd_resource_uuid = $row->[0]; + my $scan_drbd_resource_name = $row->[1]; + my $scan_drbd_resource_up = $row->[2]; + my $modified_date = $row->[3]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + scan_drbd_resource_uuid => $scan_drbd_resource_uuid, + scan_drbd_resource_name => $scan_drbd_resource_name, + scan_drbd_resource_up => $scan_drbd_resource_up, + modified_date => $modified_date, + }}); + + if ($modified_date > $anvil->data->{raw}{newest_record}{$host_uuid}) + { + $anvil->data->{raw}{newest_record}{$host_uuid} = $modified_date; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "raw::newest_record::${host_uuid}" => $anvil->data->{raw}{newest_record}{$host_uuid}, + }}); + } + + my $volumes = []; + my $resource_hash = { + resource_name => $scan_drbd_resource_name, + resource_host_uuid => $host_uuid, + is_active => $scan_drbd_resource_up, + timestamp => $anvil->data->{raw}{newest_record}{$host_uuid}, + volumes => $volumes, + }; + + push @{$hash->{resources}}, $resource_hash; + + my $query = " +SELECT + scan_drbd_volume_uuid, + scan_drbd_volume_number, + scan_drbd_volume_device_path, + scan_drbd_volume_device_minor, + scan_drbd_volume_size, + round(extract(epoch from modified_date)) +FROM + scan_drbd_volumes +WHERE + scan_drbd_volume_scan_drbd_resource_uuid = ".$anvil->Database->quote($scan_drbd_resource_uuid)." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $scan_drbd_volume_uuid = $row->[0]; + my $scan_drbd_volume_number = $row->[1]; + my $scan_drbd_volume_device_path = $row->[2]; + my $scan_drbd_volume_device_minor = $row->[3]; + my $scan_drbd_volume_size = $row->[4]; + my $modified_date = $row->[5]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + scan_drbd_volume_uuid => $scan_drbd_volume_uuid, + scan_drbd_volume_number => $scan_drbd_volume_number, + scan_drbd_volume_device_path => $scan_drbd_volume_device_path, + scan_drbd_volume_device_minor => $scan_drbd_volume_device_minor, + scan_drbd_volume_size => $scan_drbd_volume_size, + modified_date => $modified_date, + }}); + + if ($modified_date > $anvil->data->{raw}{newest_record}{$host_uuid}) + { + $anvil->data->{raw}{newest_record}{$host_uuid} = $modified_date; + $resource_hash->{timestamp} = $modified_date; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "raw::newest_record::${host_uuid}" => $anvil->data->{raw}{newest_record}{$host_uuid}, + }}); + } + + my $connections = []; + push @{$volumes}, { + number => $scan_drbd_volume_number, + drbd_device_path => $scan_drbd_volume_device_path, + drbd_device_minor => $scan_drbd_volume_device_minor, + size => $scan_drbd_volume_size, + connections => $connections, + }; + + my $query = " +SELECT + scan_drbd_peer_host_name, + scan_drbd_peer_connection_state, + scan_drbd_peer_local_disk_state, + scan_drbd_peer_disk_state, + scan_drbd_peer_local_role, + scan_drbd_peer_role, + scan_drbd_peer_out_of_sync_size, + scan_drbd_peer_replication_speed, + scan_drbd_peer_estimated_time_to_sync, + scan_drbd_peer_ip_address, + scan_drbd_peer_tcp_port, + scan_drbd_peer_protocol, + scan_drbd_peer_fencing, + round(extract(epoch from modified_date)) +FROM + scan_drbd_peers +WHERE + scan_drbd_peer_scan_drbd_volume_uuid = ".$anvil->Database->quote($scan_drbd_volume_uuid)." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $scan_drbd_peer_host_name = $row->[0]; + my $scan_drbd_peer_connection_state = $row->[1]; + my $scan_drbd_peer_local_disk_state = $row->[2]; + my $scan_drbd_peer_disk_state = $row->[3]; + my $scan_drbd_peer_local_role = $row->[4]; + my $scan_drbd_peer_role = $row->[5]; + my $scan_drbd_peer_out_of_sync_size = $row->[6]; + my $scan_drbd_peer_replication_speed = $row->[7]; + my $scan_drbd_peer_estimated_time_to_sync = $row->[8]; + my $scan_drbd_peer_ip_address = $row->[9]; + my $scan_drbd_peer_tcp_port = $row->[10]; + my $scan_drbd_peer_protocol = $row->[11]; + my $scan_drbd_peer_fencing = $row->[12]; + my $modified_date = $row->[13]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + scan_drbd_peer_host_name => $scan_drbd_peer_host_name, + scan_drbd_peer_connection_state => $scan_drbd_peer_connection_state, + scan_drbd_peer_local_disk_state => $scan_drbd_peer_local_disk_state, + scan_drbd_peer_disk_state => $scan_drbd_peer_disk_state, + scan_drbd_peer_local_role => $scan_drbd_peer_local_role, + scan_drbd_peer_role => $scan_drbd_peer_role, + scan_drbd_peer_out_of_sync_size => $scan_drbd_peer_out_of_sync_size, + scan_drbd_peer_replication_speed => $scan_drbd_peer_replication_speed, + scan_drbd_peer_estimated_time_to_sync => $scan_drbd_peer_estimated_time_to_sync, + scan_drbd_peer_ip_address => $scan_drbd_peer_ip_address, + scan_drbd_peer_tcp_port => $scan_drbd_peer_tcp_port, + scan_drbd_peer_protocol => $scan_drbd_peer_protocol, + scan_drbd_peer_fencing => $scan_drbd_peer_fencing, + modified_date => $modified_date, + }}); + if ($modified_date > $anvil->data->{raw}{newest_record}{$host_uuid}) + { + $anvil->data->{raw}{newest_record}{$host_uuid} = $modified_date; + $resource_hash->{timestamp} = $modified_date; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "raw::newest_record::${host_uuid}" => $anvil->data->{raw}{newest_record}{$host_uuid}, + }}); + } + + push @{$connections}, { + protocol => "async_".lc($scan_drbd_peer_protocol), + connection => $scan_drbd_peer_connection_state, + ip_address => $scan_drbd_peer_ip_address, + tcp_port => $scan_drbd_peer_tcp_port, + fencing => $scan_drbd_peer_fencing, + targets => [ + # Local + { + target_name => $short_host_name, + target_host_uuid => $host_uuid, + role => $scan_drbd_peer_local_role, + disk_states => $scan_drbd_peer_local_disk_state, + }, + # Peer + { + target_name => $scan_drbd_peer_host_name, + target_host_uuid => $anvil->Get->host_uuid_from_name({host_name => $scan_drbd_peer_host_name}), + role => $scan_drbd_peer_role, + disk_states => $scan_drbd_peer_disk_state, + }, + ], + resync => { + rate => $scan_drbd_peer_replication_speed, # Bytes / second + percent_complete => (($scan_drbd_peer_out_of_sync_size / $scan_drbd_volume_size) * 100), + oos_size => $scan_drbd_peer_out_of_sync_size, + time_remain => $scan_drbd_peer_estimated_time_to_sync, + }, + }; + } + } + } + } +} + +print JSON->new->utf8->encode($hash)."\n"; diff --git a/cgi-bin/get_shared_storage b/cgi-bin/get_shared_storage index 9a5479d1..6319f971 100755 --- a/cgi-bin/get_shared_storage +++ b/cgi-bin/get_shared_storage @@ -1,6 +1,6 @@ #!/usr/bin/perl # -# This prints JSON formated data reporting the status of an Anvil! system and it's member hosts. +# This prints JSON formated data reporting the status of an file systems on nodes and DR hosts. # use strict; diff --git a/cgi-bin/get_status b/cgi-bin/get_status index e69de29b..051a71a4 100755 --- a/cgi-bin/get_status +++ b/cgi-bin/get_status @@ -0,0 +1,172 @@ +#!/usr/bin/perl +# +# This prints JSON formated data reporting the status of an Anvil!'s nodes. +# + +use strict; +use warnings; +use Anvil::Tools; +use Data::Dumper; +use JSON; + +$| = 1; + +my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; +my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; +if (($running_directory =~ /^\./) && ($ENV{PWD})) +{ + $running_directory =~ s/^\./$ENV{PWD}/; +} + +my $anvil = Anvil::Tools->new(); + +$anvil->Get->switches; + +$anvil->Database->connect; +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); +if (not $anvil->data->{sys}{database}{connections}) +{ + # No databases, exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003"}); + $anvil->nice_exit({exit_code => 1}); +} + +# Read in any CGI variables, if needed. +$anvil->Get->cgi(); + +$anvil->Database->get_hosts(); +$anvil->Database->get_anvils(); + +print $anvil->Template->get({file => "shared.html", name => "json_headers", show_name => 0})."\n"; + +my $hash = {}; +my $anvil_uuid = ""; +if ($anvil->data->{cgi}{anvil_uuid}{value}) +{ + $anvil_uuid = $anvil->data->{cgi}{anvil_uuid}{value}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); +} +elsif ($anvil->data->{switches}{'anvil-uuid'}) +{ + $anvil_uuid = $anvil->data->{switches}{'anvil-uuid'}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); +} +if ((not $anvil_uuid) or (not exists $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid})) +{ + $anvil->data->{anvil_status}{anvil_name} = "!!invalid!anvil_uuid!!"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'anvil_status::anvil_name' => $anvil->data->{anvil_status}{anvil_name} }}); +} +else +{ + my $node1_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; + my $node2_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; + my $dr1_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + node1_uuid => $node1_uuid, + node2_uuid => $node2_uuid, + dr1_uuid => $dr1_uuid, + }}); + + my $query = " +SELECT + +FROM + hosts a, + scan_filesystems b +WHERE + a.host_uuid = b.scan_filesystem_host_uuid +AND + ( + a.host_uuid = ".$anvil->Database->quote($node1_uuid)." + OR + a.host_uuid = ".$anvil->Database->quote($node2_uuid); + if ($dr1_uuid) + { + $query .= " + OR + a.host_uuid = ".$anvil->Database->quote($dr1_uuid); + } + $query .= " + ) +ORDER BY + a.host_name ASC, + b.scan_filesystem_mount_point DESC +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $host_uuid = $row->[0]; + my $host_name = $row->[1]; + my $mount_point = $row->[2]; + my $size = $row->[3]; + my $used = $row->[4]; + my $free = $size - $used; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + host_uuid => $host_uuid, + host_name => $host_name, + mount_point => $mount_point, + size => $anvil->Convert->add_commas({number => $size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $size}).")",, + used => $anvil->Convert->add_commas({number => $used})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $used}).")",, + free => $anvil->Convert->add_commas({number => $free})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $free}).")",, + }}); + + $anvil->data->{raw}{file_systems}{$mount_point}{nodes}{$host_uuid}{host_name} = $host_name; + $anvil->data->{raw}{file_systems}{$mount_point}{nodes}{$host_uuid}{total} = $size; + $anvil->data->{raw}{file_systems}{$mount_point}{nodes}{$host_uuid}{free} = $free; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "raw::file_systems::${mount_point}::nodes::${host_uuid}::host_name" => $anvil->data->{raw}{file_systems}{$mount_point}{nodes}{$host_uuid}{host_name}, + "raw::file_systems::${mount_point}::nodes::${host_uuid}::total" => $anvil->Convert->add_commas({number => $anvil->data->{raw}{file_systems}{$mount_point}{nodes}{$host_uuid}{total}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{raw}{file_systems}{$mount_point}{nodes}{$host_uuid}{total}}).")",, + "raw::file_systems::${mount_point}::nodes::${host_uuid}::free" => $anvil->Convert->add_commas({number => $anvil->data->{raw}{file_systems}{$mount_point}{nodes}{$host_uuid}{free}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{raw}{file_systems}{$mount_point}{nodes}{$host_uuid}{free}}).")",, + }}); + } + + $anvil->data->{file_systems} = []; + + foreach my $mount_point (sort {$a cmp $b} keys %{$anvil->data->{raw}{file_systems}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mount_point => $mount_point }}); + $hash->{mount_point} = $mount_point; + $hash->{nodes} = []; + my $nodes = [$node1_uuid, $node2_uuid]; + if ($dr1_uuid) + { + push @{$nodes}, $dr1_uuid; + } + foreach my $host_uuid (@{$nodes}) + { + my $host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + host_uuid => $host_uuid, + host_name => $host_name, + }}); + if (exists $anvil->data->{raw}{file_systems}{$mount_point}{nodes}{$host_uuid}) + { + push @{$hash->{nodes}}, { + host_uuid => $host_uuid, + host_name => $anvil->data->{raw}{file_systems}{$mount_point}{nodes}{$host_uuid}{host_name}, + is_mounted => 1, + total => $anvil->data->{raw}{file_systems}{$mount_point}{nodes}{$host_uuid}{total}, + free => $anvil->data->{raw}{file_systems}{$mount_point}{nodes}{$host_uuid}{free}, + } + } + else + { + push @{$hash->{nodes}}, { + host_uuid => $host_uuid, + host_name => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, + is_mounted => 0, + total => 0, + free => 0, + } + } + } + } +} + +print JSON->new->utf8->encode($hash)."\n"; diff --git a/notes b/notes index 298d7758..1d336b34 100644 --- a/notes +++ b/notes @@ -831,6 +831,7 @@ OS10(conf-range-eth1/1/1-1/1/24)# exit OS10(config)# interface range ethernet 1/1/1-1/1/24,1/1/27-1/1/30 OS10(conf-range-eth1/1/1-1/1/24,1/1/27-1/1/30)# exit + # Configure management IP address OS10# configure terminal OS10(config)# interface mgmt 1/1/1 @@ -847,6 +848,12 @@ OS10(config)# OS10(config)# exit OS10# write memory +### Set hostname: +OS10# configure terminal +OS10(config)# hostname zo-switch02 +zo-switch02(config)# + +======] VLT Config [======= ### Stacking is not a thing anymore, but VLT is its replacement. # On both switches; @@ -868,8 +875,72 @@ OS10(conf-vlt-1)# discovery-interface ethernet 1/1/25-1/1/26 <165>1 2021-04-02T12:31:18.996716+00:00 OS10 dn_alm 803 - - Node.1-Unit.1:PRI [event], Dell EMC (OS10) %VLT_ELECTION_ROLE: VLT unit 1 is elected as secondary <165>1 2021-04-02T12:31:19.087695+00:00 OS10 dn_alm 803 - - Node.1-Unit.1:PRI [event], Dell EMC (OS10) %IFM_OSTATE_UP: Interface operational state is up :vlan1 + # Configure the same MAC address to the VLT on both switches: OS10# configure terminal OS10(config)# vlt-domain 1 OS10(conf-vlt-1)# vlt-mac 00:00:00:00:00:02 +# show vlt 1 mismatch +(If no issues, VLT is OK) + +# See how I am and my role (* == switch you're on) +zo-switch02(config)# show vlt 1 role +VLT Unit ID Role +------------------------ +* 1 secondary + 2 primary + +=====] VLAN Config [======== + + +zo-switch02# configure terminal +zo-switch02(config)# interface vlan 100 +zo-switch02(conf-if-vl-100)# description BCN1 +zo-switch02(conf-if-vl-100)# interface range ethernet 1/1/1-1/1/10 +zo-switch02(conf-range-eth1/1/1-1/1/10)# switchport access vlan 100 +zo-switch02(conf-range-eth1/1/1-1/1/10)# exot +% Error: Unrecognized command. +zo-switch02(conf-range-eth1/1/1-1/1/10)# exit +zo-switch02(config)# interface range ethernet 1/1/11-1/1/14 +zo-switch02(conf-range-eth1/1/11-1/1/14)# switchport access vlan 200 +zo-switch02(conf-range-eth1/1/11-1/1/14)# exit +zo-switch02(config)# interface range ethernet 1/1/15-1/1/24 +zo-switch02(conf-range-eth1/1/15-1/1/24)# switchport access vlan 300 +ezo-switch02(conf-range-eth1/1/15-1/1/24)# exit +zo-switch02(config)# show vlan +Codes: * - Default VLAN, M - Management VLAN, R - Remote Port Mirroring VLANs, + @ – Attached to Virtual Network, P - Primary, C - Community, I - Isolated +Q: A - Access (Untagged), T - Tagged + NUM Status Description Q Ports +* 1 Active A Eth1/1/27-1/1/30 + A Po1000 + 100 Active BCN1 T Po1000 + A Eth1/1/1-1/1/10 + 200 Active T Po1000 + A Eth1/1/11-1/1/14 + 300 Active T Po1000 + A Eth1/1/15-1/1/24 + 4094 Active T Po1000 + + +### Delete a VLAN: + +zo-switch02(config)# no interface vlan 3400 +zo-switch02(config)# show vlan + + +# Configure VLANs. +OS10(config)# interface range vlan 100,200,300 +OS10(conf-range-vl-100,200,300)# exit +OS10(config)# interface range ethernet 1/1/1-1/1/10 +OS10(conf-range-eth1/1/1-1/1/10)# switchport access vlan 100 +OS10(conf-range-eth1/1/1-1/1/10)# exit +OS10(config)# interface range ethernet 1/1/11-1/1/14 +OS10(conf-range-eth1/1/11-1/1/14)# switchport access vlan 200 +OS10(conf-range-eth1/1/11-1/1/14)# exit +OS10(config)# interface range ethernet 1/1/15-1/1/24 +OS10(conf-range-eth1/1/15-1/1/24)# switchport access vlan 300 +OS10(conf-range-eth1/1/15-1/1/24)# exit + + diff --git a/scancore-agents/scan-cluster/scan-cluster b/scancore-agents/scan-cluster/scan-cluster index 336641c6..d7e27a72 100755 --- a/scancore-agents/scan-cluster/scan-cluster +++ b/scancore-agents/scan-cluster/scan-cluster @@ -71,9 +71,10 @@ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => " if ($anvil->data->{switches}{purge}) { # This can be called when doing bulk-database purges. + my $schema_file = $anvil->data->{path}{directories}{scan_agents}."/".$THIS_FILE."/".$THIS_FILE.".sql"; $anvil->Database->purge_data({ debug => 2, - tables => $anvil->data->{scancore}{'scan-cluster'}{tables}, + tables => $anvil->Database->get_tables_from_schema({schema_file => $schema_file}), }); $anvil->nice_exit({exit_code => 0}); } @@ -120,14 +121,25 @@ sub find_changes stonith_max_attempts => $stonith_max_attempts, }}); + # If we're a full cluster member, read the CIB as well. + my $cluster_cib = ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "cib::parsed::local::ready" => $anvil->data->{cib}{parsed}{'local'}{ready} }}); + if ($anvil->data->{cib}{parsed}{'local'}{ready}) + { + $cluster_cib = $anvil->Storage->read_file({cache => 0, force_read => 1, debug => 2, file => $anvil->data->{path}{configs}{'cib.xml'}}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cluster_cib => $cluster_cib }}); + } + if (exists $anvil->data->{sql}{anvil_uuid}{$scan_cluster_anvil_uuid}) { # Check for a name change $scan_cluster_uuid = $anvil->data->{sql}{anvil_uuid}{$scan_cluster_anvil_uuid}; my $old_cluster_name = $anvil->data->{sql}{scan_cluster}{scan_cluster_uuid}{$scan_cluster_uuid}{scan_cluster_name}; + my $old_cluster_cib = $anvil->data->{sql}{scan_cluster}{scan_cluster_uuid}{$scan_cluster_uuid}{scan_cluster_cib}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { scan_cluster_uuid => $scan_cluster_uuid, old_cluster_name => $old_cluster_name, + old_cluster_cib => $old_cluster_cib, }}); if ($cluster_name ne $old_cluster_name) { @@ -144,6 +156,27 @@ WHERE $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); + my $variables = { + new_cluster_name => $cluster_name, + old_cluster_name => $old_cluster_name, + }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_cluster_alert_0002", variables => $variables}); + $anvil->Alert->register({debug => 2, alert_level => "notice", message => "scan_cluster_alert_0002", variables => $variables, set_by => $THIS_FILE}); + } + if (($cluster_cib) && ($cluster_cib ne $old_cluster_cib)) + { + my $query = " +UPDATE + scan_cluster +SET + scan_cluster_cib = ".$anvil->Database->quote($cluster_name).", + modified_date = ".$anvil->Database->quote($anvil->data->{sys}{database}{timestamp})." +WHERE + scan_cluster_uuid = ".$anvil->Database->quote($scan_cluster_uuid)." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); + my $variables = { new_cluster_name => $cluster_name, old_cluster_name => $old_cluster_name, @@ -163,11 +196,13 @@ INSERT INTO scan_cluster_uuid, scan_cluster_anvil_uuid, scan_cluster_name, + scan_cluster_cib, modified_date ) VALUES ( ".$anvil->Database->quote($scan_cluster_uuid).", ".$anvil->Database->quote($scan_cluster_anvil_uuid).", ".$anvil->Database->quote($cluster_name).", + ".$anvil->Database->quote($cluster_cib).", ".$anvil->Database->quote($anvil->data->{sys}{database}{timestamp})." );"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); @@ -386,7 +421,8 @@ sub read_last_scan SELECT scan_cluster_uuid, scan_cluster_anvil_uuid, - scan_cluster_name + scan_cluster_name, + scan_cluster_cib FROM scan_cluster ;"; @@ -405,18 +441,22 @@ FROM my $scan_cluster_uuid = $row->[0]; my $scan_cluster_anvil_uuid = $row->[1]; my $scan_cluster_name = $row->[2]; + my $scan_cluster_cib = $row->[3]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "scan_cluster_uuid" => $scan_cluster_uuid, - "scan_cluster_anvil_uuid" => $scan_cluster_anvil_uuid, - "scan_cluster_name" => $scan_cluster_name, + "s1:scan_cluster_uuid" => $scan_cluster_uuid, + "s2:scan_cluster_anvil_uuid" => $scan_cluster_anvil_uuid, + "s3:scan_cluster_name" => $scan_cluster_name, + "s4:scan_cluster_cib" => $scan_cluster_cib, }}); # Store the old data now. $anvil->data->{sql}{scan_cluster}{scan_cluster_uuid}{$scan_cluster_uuid}{scan_cluster_name} = $scan_cluster_name; $anvil->data->{sql}{scan_cluster}{scan_cluster_uuid}{$scan_cluster_uuid}{scan_cluster_anvil_uuid} = $scan_cluster_anvil_uuid; + $anvil->data->{sql}{scan_cluster}{scan_cluster_uuid}{$scan_cluster_uuid}{scan_cluster_cib} = $scan_cluster_cib; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sql::scan_cluster::scan_cluster_uuid::${scan_cluster_uuid}::scan_cluster_name" => $anvil->data->{sql}{scan_cluster}{scan_cluster_uuid}{$scan_cluster_uuid}{scan_cluster_name}, "sql::scan_cluster::scan_cluster_uuid::${scan_cluster_uuid}::scan_cluster_anvil_uuid" => $anvil->data->{sql}{scan_cluster}{scan_cluster_uuid}{$scan_cluster_uuid}{scan_cluster_anvil_uuid}, + "sql::scan_cluster::scan_cluster_uuid::${scan_cluster_uuid}::scan_cluster_cib" => $anvil->data->{sql}{scan_cluster}{scan_cluster_uuid}{$scan_cluster_uuid}{scan_cluster_cib}, }}); # Make it easy to look up the cluster_uuid from the anvil_uuid. diff --git a/scancore-agents/scan-cluster/scan-cluster.sql b/scancore-agents/scan-cluster/scan-cluster.sql index 448dca6c..13338531 100644 --- a/scancore-agents/scan-cluster/scan-cluster.sql +++ b/scancore-agents/scan-cluster/scan-cluster.sql @@ -8,6 +8,7 @@ CREATE TABLE scan_cluster ( scan_cluster_uuid uuid primary key, scan_cluster_anvil_uuid uuid not null, -- The Anvil! UUID this cluster is associated with. scan_cluster_name text not null, -- The name of the cluster + scan_cluster_cib text not null, -- This is the CIB from disk, only updated when a node is a full member of the cluster. modified_date timestamp with time zone not null ); ALTER TABLE scan_cluster OWNER TO admin; @@ -17,6 +18,7 @@ CREATE TABLE history.scan_cluster ( scan_cluster_uuid uuid, scan_cluster_anvil_uuid uuid, scan_cluster_name text, + scan_cluster_cib text, modified_date timestamp with time zone not null ); ALTER TABLE history.scan_cluster OWNER TO admin; @@ -31,11 +33,13 @@ BEGIN (scan_cluster_uuid, scan_cluster_anvil_uuid, scan_cluster_name, + scan_cluster_cib, modified_date) VALUES (history_scan_cluster.scan_cluster_uuid, history_scan_cluster.scan_cluster_anvil_uuid, history_scan_cluster.scan_cluster_name, + history_scan_cluster.scan_cluster_cib, history_scan_cluster.modified_date); RETURN NULL; END; diff --git a/share/anvil.sql b/share/anvil.sql index bee4d7ae..45eb156f 100644 --- a/share/anvil.sql +++ b/share/anvil.sql @@ -61,7 +61,7 @@ CREATE TABLE hosts ( host_type text not null, -- Either 'node' or 'dashboard' or 'dr'. It is left empty until the host is configured. host_key text not null, -- This is the host's key used to authenticate it when other machines try to ssh to it. host_ipmi text not null default '', -- This is an optional string, in 'fence_ipmilan' format, that tells how to access/fence this host. - host_status text not null default 'unknown', -- This is the power state of the host. Default is 'unknown', and can be "powered off", "online", "stopping" and "booting. + host_status text not null default 'unknown', -- This is the power state of the host. Default is 'unknown', and can be "powered off", "stopping", "online" and "booting". modified_date timestamp with time zone not null ); ALTER TABLE hosts OWNER TO admin; diff --git a/share/words.xml b/share/words.xml index 47a6a22f..eb253c63 100644 --- a/share/words.xml +++ b/share/words.xml @@ -1414,7 +1414,7 @@ The file: [#!variable!file!#] needs to be updated. The difference is: Unable to find an install manifest for the Anvil! [#!variable!anvil_name!#]. As such, unable to determine what UPSes power the machine: [#!variable!host_name!#]. Unable to determine if the power feeding this node is OK or not. Unable to parse the install manifest uuid: [#!variable!manifest_uuid!#] for the Anvil! [#!variable!anvil_name!#]. As such, unable to determine what UPSes power the machine: [#!variable!host_name!#]. Unable to determine if the power feeding this node is OK or not. The UPS referenced by the 'power_uuid': [#!variable!power_uuid!#] under the host: [#!variable!host_name!#] has no record of being on mains power, so we can't determine how long it's been on batteries. Setting the "shortest time on batteries" to zero seconds. - Clearing the host's stop reason. + Marking the host as 'online' and clearing the host's stop reason. The host: [#!variable!host_name!#] is off, but there appears to be a problem translating the 'fence_ipmilan' into a workable 'ipmitool' command. Unable to check the thermal data of the host, and so, unable to determine if it's safe to boot the node. The host: [#!variable!host_name!#] was powered off because of power loss. Power is back and the UPSes are sufficiently charged. Booting it back up now. The host: [#!variable!host_name!#] was powered off for thermal reasons. All available thermal sensors read as OK now. Booting it back up now. @@ -1438,6 +1438,8 @@ The file: [#!variable!file!#] needs to be updated. The difference is: The job: [#!variable!command!#] with UUID: [#!variable!job_uuid!#] is a start-time job, not running it now. The lvm.conf already has the filter: [#!variable!filter!#], will not change it. Updated the lvm.conf file to add the filter: [#!variable!filter!#] to prevent LVM from seeing the DRBD devices as LVM devices. + The host: [#!variable!host_name!#] last updated the database: [#!variable!difference!#] seconds ago, skipping power checks. + The host: [#!variable!host_name!#] has no entries in the 'updated' table, so ScanCore has likely never run. Skipping this host for now. The host name: [#!variable!target!#] does not resolve to an IP address. diff --git a/tools/anvil-daemon b/tools/anvil-daemon index c00dc9f3..ce70dd90 100755 --- a/tools/anvil-daemon +++ b/tools/anvil-daemon @@ -749,30 +749,61 @@ AND "s3:uptime" => $uptime, "s4:difference" => $difference, }}); - if (($reboot_needed) && ($uptime < $changed_seconds_ago)) + if ($reboot_needed) { - # Clear the reboot request. - $reboot_needed = $anvil->System->reboot_needed({set => 0}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { reboot_needed => $reboot_needed }}); - - # Check to see if there was a reboot job in progress. If so, finish it off. - my $job_uuid = $anvil->Job->get_job_uuid({program => "anvil-manage-power"}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { job_uuid => $job_uuid }}); - - if ($job_uuid) + ($uptime < $changed_seconds_ago) { - # Update the percentage to '100' and then clear the old PID. - my $date_time = $anvil->Get->date_and_time(); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { date_time => $date_time }}); + # Clear the reboot request. + $reboot_needed = $anvil->System->reboot_needed({set => 0}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { reboot_needed => $reboot_needed }}); - $anvil->Job->update_progress({ - progress => 100, - message => "message_0064,!!date_and_time!".$date_time."!!", - job_uuid => $job_uuid, - picked_up_by => 0, - }); + # Check to see if there was a reboot job in progress. If so, finish it off. + my $job_uuid = $anvil->Job->get_job_uuid({program => "anvil-manage-power"}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { job_uuid => $job_uuid }}); + + if ($job_uuid) + { + # Update the percentage to '100' and then clear the old PID. + my $date_time = $anvil->Get->date_and_time(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { date_time => $date_time }}); + + $anvil->Job->update_progress({ + progress => 100, + message => "message_0064,!!date_and_time!".$date_time."!!", + job_uuid => $job_uuid, + picked_up_by => 0, + }); + } } } + else + { + # Update our status + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0572"}); + + $anvil->Database->get_hosts({debug => 3}); + my $host_uuid = $anvil->Get->host_uuid(); + $anvil->Database->insert_or_update_hosts({ + host_ipmi => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}, + host_key => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, + host_name => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, + host_type => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type}, + host_uuid => $host_uuid, + host_status => "online", + }); + + # Make sure our stop reason is cleared. + my $variable_uuid = $anvil->Database->insert_or_update_variables({ + variable_name => 'system::stop_reason', + variable_value => '', + variable_default => '', + variable_description => 'striker_0279', + variable_section => 'system', + variable_source_uuid => '4c4c4544-0043-4210-8042-c3c04f523533', + variable_source_table => 'hosts', + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }}); + } # Now look for jobs that have a job status of 'scancore_startup' run_jobs($anvil, 1); diff --git a/tools/anvil-join-anvil b/tools/anvil-join-anvil index ed0e11cc..de735fe3 100755 --- a/tools/anvil-join-anvil +++ b/tools/anvil-join-anvil @@ -202,7 +202,7 @@ sub configure_pacemaker update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0094,!!daemon!libvirtd.service!!"); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "libvirtd.service" }}); - # Disabled and stop the libvirtd daemon. + # Disabled and stop the drbd daemon. ($return_code) = $anvil->System->disable_daemon({daemon => "drbd.service"}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }}); $return_code = undef; @@ -212,6 +212,16 @@ sub configure_pacemaker update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!drbd.service!!"); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "drbd.service" }}); + # Disabled and stop the ksm and ksmtuned daemon. + ($return_code) = $anvil->System->disable_daemon({daemon => "ksm.service"}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }}); + $return_code = undef; + ($return_code) = $anvil->System->stop_daemon({daemon => "ksmtuned.service"}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }}); + $return_code = undef; + update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!ksm.service!!"); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "drbd.service" }}); + # If there is no corosync.conf, see if the peer has it. If so, copy it. If not, we'll initialize the # cluster shortly. if (not -e $anvil->data->{path}{configs}{'corosync.conf'}) diff --git a/tools/anvil-manage-power b/tools/anvil-manage-power index a37929df..6029b689 100755 --- a/tools/anvil-manage-power +++ b/tools/anvil-manage-power @@ -246,6 +246,13 @@ sub do_poweroff # that it is starting post-reboot and clear it. $reboot_needed = $anvil->System->reboot_needed({debug => 2, set => 1}); + # Mark our power state. + $anvil->Database->update_host_status({ + debug => $debug, + host_uuid => $anvil->Get->host_uuid, + host_status => $task eq "poweroff" ? "rebooting" : "stopping";, + }); + # Now do the deed. my $shell_call = $anvil->data->{path}{exe}{systemctl}." ".$task; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); diff --git a/tools/anvil-provision-server b/tools/anvil-provision-server index 5c2a129f..d3685500 100755 --- a/tools/anvil-provision-server +++ b/tools/anvil-provision-server @@ -797,8 +797,9 @@ sub create_md { my ($anvil) = @_; + ### NOTE: The '--max-peers=3' is needed to make space for future DR additions # Create the DRBD metadata - my $shell_call = $anvil->data->{path}{exe}{drbdadm}." -- --force create-md ".$anvil->data->{job}{server_name}; + my $shell_call = $anvil->data->{path}{exe}{drbdadm}." -- --force create-md --max-peers=3 ".$anvil->data->{job}{server_name}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); diff --git a/tools/scancore b/tools/scancore index d2aee96d..4d0c2505 100755 --- a/tools/scancore +++ b/tools/scancore @@ -306,8 +306,21 @@ sub startup_tasks { my ($anvil) = @_; - # Make sure our stop reason is cleared. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0572"}); + + # Update our status + $anvil->Database->get_hosts({debug => 3}); + my $host_uuid = $anvil->Get->host_uuid(); + $anvil->Database->insert_or_update_hosts({ + host_ipmi => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}, + host_key => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key}, + host_name => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}, + host_type => $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type}, + host_uuid => $host_uuid, + host_status => "online", + }); + + # Make sure our stop reason is cleared. my $variable_uuid = $anvil->Database->insert_or_update_variables({ variable_name => 'system::stop_reason', variable_value => '',