diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index 8fb5c639..6ec4c1be 100644 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -925,7 +925,7 @@ sub _set_defaults # This is the list of OSes short in the user's short list of OS types to # optimize for. The full the list is available by running: # /usr/bin/osinfo-query os - os_short_list => "rhel5.11, rhel6.10, rhel7.9, rhel8.3, win10, win2k16, win2k19", + os_short_list => "rhel8.7,rhel9.1,win10,win2k19,win2k22", }, terminal => { columns => 80, @@ -1056,6 +1056,7 @@ sub _set_paths 'httpd.conf' => "/etc/httpd/conf/httpd.conf", 'journald_anvil' => "/etc/systemd/journald.conf.d/anvil.conf", 'journald.conf' => "/etc/systemd/journald.conf", + 'logind.conf' => "/etc/systemd/logind.conf", 'lvm.conf' => "/etc/lvm/lvm.conf", 'pg_hba.conf' => "/var/lib/pgsql/data/pg_hba.conf", 'postgresql.conf' => "/var/lib/pgsql/data/postgresql.conf", @@ -1074,10 +1075,11 @@ sub _set_paths '.htpasswd' => "/etc/httpd/.htpasswd", 'chrony.conf' => "/etc/chrony.conf", group => "/etc/group", - issue => "/etc/issue", httpd_conf => "/etc/httpd/conf/httpd.conf", + host_configured => "/etc/anvil/host.configured", host_ssh_key => "/etc/ssh/ssh_host_ecdsa_key.pub", host_uuid => "/etc/anvil/host.uuid", + issue => "/etc/issue", network_cache => "/tmp/network_cache.anvil", passwd => "/etc/passwd", 'redhat-release' => "/etc/redhat-release", @@ -1151,6 +1153,7 @@ sub _set_paths 'anvil-update-files' => "/usr/sbin/anvil-update-files", 'anvil-update-states' => "/usr/sbin/anvil-update-states", 'anvil-update-system' => "/usr/sbin/anvil-update-system", + 'anvil-version-changes' => "/usr/sbin/anvil-version-changes", augtool => "/usr/bin/augtool", base64 => "/usr/bin/base64", blockdev => "/usr/sbin/blockdev", @@ -1167,6 +1170,7 @@ sub _set_paths createdb => "/usr/bin/createdb", createrepo_c => "/usr/bin/createrepo_c", createuser => "/usr/bin/createuser", + crm_attribute => "/usr/sbin/crm_attribute", crm_error => "/usr/sbin/crm_error", crm_resource => "/usr/sbin/crm_resource", crm_mon => "/usr/sbin/crm_mon", @@ -1250,6 +1254,7 @@ sub _set_paths rpm => "/usr/bin/rpm", rsync => "/usr/bin/rsync", sed => "/usr/bin/sed", + setsid => "/usr/bin/setsid", # See: https://serverfault.com/questions/1105733/virsh-command-hangs-when-script-runs-in-the-background 'shutdown' => "/usr/sbin/shutdown", snmpget => "/usr/bin/snmpget", snmpset => "/usr/bin/snmpset", @@ -1280,6 +1285,7 @@ sub _set_paths tput => "/usr/bin/tput", 'tr' => "/usr/bin/tr", uname => "/usr/bin/uname", + unfence_pacemaker => "/usr/sbin/unfence_pacemaker", unzip => "/usr/bin/unzip", useradd => "/usr/sbin/useradd", usermod => "/usr/sbin/usermod", diff --git a/Anvil/Tools/Cluster.pm b/Anvil/Tools/Cluster.pm index 1db6b883..7649ff34 100644 --- a/Anvil/Tools/Cluster.pm +++ b/Anvil/Tools/Cluster.pm @@ -8,6 +8,7 @@ use warnings; use Data::Dumper; use Scalar::Util qw(weaken isweak); use String::ShellQuote; +use Text::Diff; use XML::LibXML; use XML::Simple qw(:strict); @@ -21,6 +22,7 @@ my $THIS_FILE = "Cluster.pm"; # check_node_status # check_server_constraints # check_stonith_config +# configure_logind # delete_server # get_fence_methods # get_anvil_name @@ -646,6 +648,8 @@ sub boot_server } } + ### TODO: If we don't have a node, pick the node with the most VMs already running (by total RAM + ### count) if ($node) { $anvil->Cluster->_set_server_constraint({ @@ -654,6 +658,8 @@ sub boot_server }); } + ### TODO: Make sure that the drbd fence rule exists in pacemaker and add it, if not. + # Now boot the server. my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{pcs}." resource enable ".$server}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { @@ -1622,6 +1628,109 @@ sub check_stonith_config } +=head2 configure_logind + +This configures logind to ensure it doesn't try to do a graceful shutdown when being fenced via acpid power-button events. + +See: https://access.redhat.com/solutions/1578823 + +This method takes no parameters + +=cut +sub configure_logind +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Cluster->configure_logind()" }}); + + # Only run this on nodes. + my $host_type = $anvil->Get->host_type({debug => $debug}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_type => $host_type }}); + if ($host_type ne "node") + { + return(0); + } + + # Read in the file. + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 'path::configs::logind.conf' => $anvil->data->{path}{configs}{'logind.conf'}, + }}); + if (not -e $anvil->data->{path}{configs}{'logind.conf'}) + { + # wtf? + return(0); + } + + my $added = 0; + my $new_body = ""; + my $old_body = $anvil->Storage->read_file({debug => $debug, file => $anvil->data->{path}{configs}{'logind.conf'}}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { old_body => $old_body }}); + + if ($old_body eq "!!error!!") + { + return(0); + } + + # If we don't see 'HandlePowerKey=ignore', we need to add it. + foreach my $line (split/\n/, $old_body) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { old_body => $old_body }}); + $new_body .= $line."\n"; + if ($line =~ /^HandlePowerKey=(.*)$/) + { + # It's been set. No matter how it's set, we don't change it again. + my $set_to = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { set_to => $set_to }}); + return(0); + } + if ($line =~ /^#HandlePowerKey=/) + { + # Add line under the commented out one. + $new_body .= "HandlePowerKey=ignore\n"; + $added = 1; + } + } + + if (not $added) + { + # Append it. + $new_body .= "HandlePowerKey=ignore\n"; + $added = 1; + } + + # Still here? We almost certainly want to save then, but lets look for a difference just the same. + my $difference = diff \$old_body, \$new_body, { STYLE => 'Unified' }; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + added => $added, + difference => $difference, + }}); + if ($added) + { + # Write it out. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0732"}); + $anvil->Storage->write_file({ + file => $anvil->data->{path}{configs}{'logind.conf'}, + body => $new_body, + backup => 1, + overwrite => 1, + }); + + sleep 1; + + # Restart the daemon. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0733", variables => { daemon => "systemd-logind.service" }}); + $anvil->System->restart_daemon({ + debug => $debug, + daemon => "systemd-logind.service", + }); + } + + return(0); +} + + =head2 delete_server This takes a server (resource) name and deletes it from pacemaker. If there is a problem, C<< !!error!! >> is returned. Otherwise, C<< 0 >> is removed either once the resource is deleted, or if the resource didn't exist in the first place. @@ -3079,6 +3188,7 @@ sub parse_cib foreach my $node ($dom->findnodes('/cib/configuration/nodes/node')) { my $node_id = $node->{id}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { node_id => $node_id }}); foreach my $variable (sort {$a cmp $b} keys %{$node}) { next if $variable eq "id"; @@ -3111,6 +3221,7 @@ sub parse_cib foreach my $instance_attributes ($node->findnodes('./instance_attributes')) { my $instance_attributes_id = $instance_attributes->{id}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { instance_attributes_id => $instance_attributes_id }}); foreach my $nvpair ($instance_attributes->findnodes('./nvpair')) { my $id = $nvpair->{id}; @@ -3180,14 +3291,37 @@ sub parse_cib foreach my $constraint ($dom->findnodes('/cib/configuration/constraints/rsc_location')) { my $id = $constraint->{id}; - $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{node} = $constraint->{node}; + $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{node} = $constraint->{node} ? $constraint->{node} : ""; $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{resource} = $constraint->{rsc}; - $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{score} = $constraint->{score}; + $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{score} = $constraint->{score} ? $constraint->{score} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "cib::parsed::configuration::constraints::location::${id}::node" => $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{node}, "cib::parsed::configuration::constraints::location::${id}::resource" => $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{resource}, "cib::parsed::configuration::constraints::location::${id}::score" => $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{score}, }}); + + # If there's no 'node', this is probably a drbd fence constraint. + if (not $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{node}) + { + foreach my $rule_id ($constraint->findnodes('./rule')) + { + my $constraint_id = $rule_id->{id}; + $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{constraint}{$constraint_id}{score} = $rule_id->{score}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "cib::parsed::configuration::constraints::location::${id}::constraint::${constraint_id}::score" => $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{constraint}{$constraint_id}{score}, + }}); + foreach my $expression_id ($rule_id->findnodes('./expression')) + { + my $attribute = $expression_id->{attribute}; + $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{constraint}{$constraint_id}{attribute}{$attribute}{operation} = $expression_id->{operation}; + $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{constraint}{$constraint_id}{attribute}{$attribute}{value} = $expression_id->{value}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "cib::parsed::configuration::constraints::location::${id}::constraint::${constraint_id}::attribute::${attribute}::operation" => $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{constraint}{$constraint_id}{attribute}{$attribute}{operation}, + "cib::parsed::configuration::constraints::location::${id}::constraint::${constraint_id}::attribute::${attribute}::value" => $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{constraint}{$constraint_id}{attribute}{$attribute}{value}, + }}); + } + } + } } foreach my $node_state ($dom->findnodes('/cib/status/node_state')) { @@ -3529,17 +3663,17 @@ sub parse_cib foreach my $lrm_resource_id (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{cib}{status}{node_state}{$id}{lrm_id}{$lrm_id}{lrm_resource}}) { my $lrm_resource_operations_count = keys %{$anvil->data->{cib}{parsed}{cib}{status}{node_state}{$id}{lrm_id}{$lrm_id}{lrm_resource}{$lrm_resource_id}{lrm_rsc_op_id}}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { lrm_resource_operations_count => $lrm_resource_operations_count }}); foreach my $lrm_rsc_op_id (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{cib}{status}{node_state}{$id}{lrm_id}{$lrm_id}{lrm_resource}{$lrm_resource_id}{lrm_rsc_op_id}}) { my $type = $anvil->data->{cib}{parsed}{cib}{status}{node_state}{$id}{lrm_id}{$lrm_id}{lrm_resource}{$lrm_resource_id}{type}; my $class = $anvil->data->{cib}{parsed}{cib}{status}{node_state}{$id}{lrm_id}{$lrm_id}{lrm_resource}{$lrm_resource_id}{class}; my $operation = $anvil->data->{cib}{parsed}{cib}{status}{node_state}{$id}{lrm_id}{$lrm_id}{lrm_resource}{$lrm_resource_id}{lrm_rsc_op_id}{$lrm_rsc_op_id}{operation}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - lrm_resource_operations_count => $lrm_resource_operations_count, - type => $type, - class => $class, - operation => $operation, - lrm_rsc_op_id => $lrm_rsc_op_id, + 's1:lrm_rsc_op_id' => $lrm_rsc_op_id, + 's2:type' => $type, + 's3:class' => $class, + 's4:operation' => $operation, }}); # Skip unless it's a server. @@ -3569,6 +3703,76 @@ sub parse_cib "cib::parsed::data::server::${lrm_resource_id}::role" => $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{role}, }}); } + + # Do we have a DRBD fence rule? + $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_rule}{'exists'} = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "cib::parsed::data::server::${lrm_resource_id}::drbd_fence_rule::exists" => $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_rule}{'exists'}, + }}); + foreach my $id (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{configuration}{constraints}{location}}) + { + my $node = $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{node}; + my $resource = $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{resource}; + my $score = $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{score}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:id" => $id, + "s2:node" => $node, + "s3:resource" => $resource, + "s4:score" => $score, + }}); + + # Is this the server? + next if $resource ne $lrm_resource_id; + next if not exists $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{constraint}; + foreach my $constraint_id (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{constraint}}) + { + my $score = $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{constraint}{$constraint_id}{score}; + foreach my $attribute (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{constraint}{$constraint_id}{attribute}}) + { + my $operation = $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{constraint}{$constraint_id}{attribute}{$attribute}{operation}; + my $value = $anvil->data->{cib}{parsed}{configuration}{constraints}{location}{$id}{constraint}{$constraint_id}{attribute}{$attribute}{value}; + my $test_key = "location-".$resource."-rule"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:constraint_id' => $constraint_id, + 's2:score' => $score, + 's3:attribute' => $attribute, + 's4:operation' => $operation, + 's5:value' => $value, + 's6:test_key' => $test_key, + }}); + + if ($constraint_id eq $test_key) + { + $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_rule}{'exists'} = 1; + $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_rule}{attribute} = $attribute; + $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_rule}{operation} = $operation; + $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_rule}{value} = $value; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:cib::parsed::data::server::${lrm_resource_id}::drbd_fence_rule::exists" => $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_rule}{'exists'}, + "s2:cib::parsed::data::server::${lrm_resource_id}::drbd_fence_rule::attribute" => $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_rule}{attribute}, + "s3:cib::parsed::data::server::${lrm_resource_id}::drbd_fence_rule::operation" => $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_rule}{operation}, + "s4:cib::parsed::data::server::${lrm_resource_id}::drbd_fence_rule::value" => $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_rule}{value}, + }}); + + # Is this refereneced by any node attributes? + foreach my $node_id (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{cib}{node_state}}) + { + my $node_name = $anvil->data->{cib}{parsed}{configuration}{nodes}{$node_id}{uname}; + my $value = defined $anvil->data->{cib}{parsed}{cib}{node_state}{$node_id}{$attribute} ? $anvil->data->{cib}{parsed}{cib}{node_state}{$node_id}{$attribute} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:node_id" => $node_id, + "s2:node_name" => $node_name, + "s3:value" => $value, + }}); + } + } + } + last if $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_rule}{'exists'}; + } + + # Did we find it? + last if $anvil->data->{cib}{parsed}{data}{server}{$lrm_resource_id}{drbd_fence_rule}{'exists'}; + } } } } diff --git a/Anvil/Tools/Convert.pm b/Anvil/Tools/Convert.pm index f901a4e8..9fe65413 100644 --- a/Anvil/Tools/Convert.pm +++ b/Anvil/Tools/Convert.pm @@ -238,9 +238,26 @@ sub bytes_to_human_readable # Die if either the 'time' or 'float' has a non-digit character in it. if ($human_readable_size =~ /\D/) { + # See if this is already human readable. + my $bytes = $anvil->Convert->human_readable_to_bytes({ + debug => $debug, + size => $human_readable_size, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'bytes' => $bytes }}); + if ($bytes =~ /^\d+$/) + { + # This is fine, convert to our standard size and return. + my $new_human_readable_size = $anvil->Convert->bytes_to_human_readable({ + debug => 2, + 'bytes' => $bytes, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { new_human_readable_size => $new_human_readable_size }}); + return($new_human_readable_size); + } + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0116", variables => { method => "Convert->bytes_to_human_readable()", - parameter => "hostnmae", + parameter => "bytes", value => $human_readable_size, }}); return ("!!error!!"); @@ -982,6 +999,12 @@ sub human_readable_to_bytes type => $type, }}); + # If we were passed nothing, return nothing. + if ($size eq "") + { + return(""); + } + # Start cleaning up the variables. my $value = $size; $size =~ s/ //g; diff --git a/Anvil/Tools/DRBD.pm b/Anvil/Tools/DRBD.pm index f2776ae9..20cefadb 100644 --- a/Anvil/Tools/DRBD.pm +++ b/Anvil/Tools/DRBD.pm @@ -1396,9 +1396,9 @@ SELECT FROM scan_drbd WHERE - scan_drbd_host_uuid = '618e8007-3a0b-4bbf-a616-a64fd7d2dc30' + scan_drbd_host_uuid = ".$anvil->Database->quote($node1_host_uuid)." OR - scan_drbd_host_uuid = '75070e21-a0e3-4ba5-b4f7-476bf5d08107' + scan_drbd_host_uuid = ".$anvil->Database->quote($node2_host_uuid)." ORDER BY modified_date DESC LIMIT 1 ;"; @@ -1547,9 +1547,9 @@ LIMIT 1 ### TODO: Handle external metadata my $this_host = $host_href->{name}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - this_host => $this_host, - '$anvil->Get->host_name' => $anvil->Get->host_name, - '$anvil->Get->short_host_name' => $anvil->Get->short_host_name, + this_host => $this_host, + 'Get->host_name' => $anvil->Get->host_name, + 'Get->short_host_name' => $anvil->Get->short_host_name, }}); if (($this_host eq $anvil->Get->host_name) or ($this_host eq $anvil->Get->short_host_name)) { @@ -2755,6 +2755,7 @@ sub update_global_common my $usage_count_seen = 0; my $udev_always_use_vnr_seen = 0; my $fence_peer_seen = 0; + my $unfence_peer_seen = 0; my $auto_promote_seen = 0; my $disk_flushes_seen = 0; my $md_flushes_seen = 0; @@ -2778,6 +2779,7 @@ sub update_global_common # These values will be used to track where we are in processing the config file and what values are needed. my $say_usage_count = $usage_count ? "yes" : "no"; my $say_fence_peer = $anvil->data->{path}{exe}{fence_pacemaker}; + my $say_unfence_peer = $anvil->data->{path}{exe}{unfence_pacemaker}; my $say_auto_promote = "yes"; my $say_flushes = $use_flushes ? "yes" : "no"; my $say_allow_two_primaries = "no"; @@ -2865,6 +2867,17 @@ sub update_global_common $new_global_common .= $new_line."\n"; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0518", variables => { file => $anvil->data->{path}{configs}{'global-common.conf'}, line => $line }}); } + if (not $unfence_peer_seen) + { + $update = 1; + my $new_line = "\t\tunfence-peer ".$say_unfence_peer.";"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:update' => $update, + 's2:new_line' => $new_line, + }}); + $new_global_common .= $new_line."\n"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0518", variables => { file => $anvil->data->{path}{configs}{'global-common.conf'}, line => $line }}); + } } elsif ($in_startup) { @@ -3091,7 +3104,37 @@ sub update_global_common } if ($in_handlers) { - if ($line =~ /(\s*)fence-peer(\s+)(.*?)(;.*)$/) + if ($line =~ /(\s*)unfence-peer(\s+)(.*?)(;.*)$/) + { + my $left_space = $1; + my $middle_space = $2; + my $value = $3; + my $right_side = $4; + $unfence_peer_seen = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:left_space' => $left_space, + 's2:middle_space' => $middle_space, + 's3:value' => $value, + 's4:right_side' => $right_side, + 's5:unfence_peer_seen' => $unfence_peer_seen, + 's6:say_unfence_peer' => $say_unfence_peer, + }}); + + if ($value ne $say_unfence_peer) + { + $update = 1; + my $new_line = $left_space."unfence-peer".$middle_space.$say_unfence_peer.$right_side; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:update' => $update, + 's2:new_line' => $new_line, + }}); + + $new_global_common .= $new_line.$comment."\n"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0518", variables => { file => $anvil->data->{path}{configs}{'global-common.conf'}, line => $line }}); + next; + } + } + elsif ($line =~ /(\s*)fence-peer(\s+)(.*?)(;.*)$/) { my $left_space = $1; my $middle_space = $2; @@ -3104,6 +3147,7 @@ sub update_global_common 's3:value' => $value, 's4:right_side' => $right_side, 's5:fence_peer_seen' => $fence_peer_seen, + 's6:say_fence_peer' => $say_fence_peer, }}); if ($value ne $say_fence_peer) diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index b869f8ea..8a0ad3e4 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -29,6 +29,7 @@ my $THIS_FILE = "Database.pm"; # get_alerts # get_anvils # get_bridges +# get_dr_links # get_fences # get_file_locations # get_files @@ -53,6 +54,7 @@ my $THIS_FILE = "Database.pm"; # insert_or_update_anvils # insert_or_update_bridges # insert_or_update_bonds +# insert_or_update_dr_links # insert_or_update_fences # insert_or_update_file_locations # insert_or_update_files @@ -1849,11 +1851,13 @@ sub connect } # If we're a striker and no connections were found, start our database. + my $configured_databases = keys %{$anvil->data->{database}}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { local_host_type => $local_host_type, "sys::database::connections" => $anvil->data->{sys}{database}{connections}, + configured_databases => $configured_databases, }}); - if (($local_host_type eq "striker") && (not $anvil->data->{sys}{database}{connections})) + if (($local_host_type eq "striker") && (not $anvil->data->{sys}{database}{connections}) && ($configured_databases > 2)) { # Tell the user we're going to try to load and start. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0650"}); @@ -2648,6 +2652,9 @@ sub get_anvils $anvil->Database->get_files({debug => $debug}); $anvil->Database->get_file_locations({debug => $debug}); + # Also pull in DRs so we can link them. + $anvil->Database->get_dr_links({debug => $debug}); + my $query = " SELECT anvil_uuid, @@ -2757,6 +2764,7 @@ WHERE "anvils::host_uuid::${anvil_node2_host_uuid}::role" => $anvil->data->{anvils}{host_uuid}{$anvil_node2_host_uuid}{role}, }}); } + ### TODO: Remove this once the switch over to 'dr_links' is done. if ($anvil_dr1_host_uuid) { $anvil->data->{anvils}{host_uuid}{$anvil_dr1_host_uuid}{anvil_name} = $anvil_name; @@ -2810,6 +2818,32 @@ WHERE "anvils::anvil_uuid::${anvil_uuid}::file_name::${file_name}::file_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}, }}); } + + # Process DR hosts this Anvil! is allowed to use. + if (exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}) + { + foreach my $dr_link_host_uuid (sort {$a cmp $b} keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}}) + { + my $dr_link_uuid = $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$dr_link_host_uuid}{dr_link_uuid}; + my $dr_link_note = $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_note}; + my $dr_link_short_host_name = $anvil->data->{hosts}{host_uuid}{$dr_link_host_uuid}{short_host_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:dr_link_host_uuid" => $dr_link_host_uuid, + "s2:dr_link_uuid" => $dr_link_uuid, + "s3:dr_link_note" => $dr_link_note, + "s4:dr_link_short_host_name" => $dr_link_short_host_name, + }}); + + next if $dr_link_note eq "DELETED"; + + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{dr_host}{$dr_link_host_uuid}{dr_link_uuid} = $dr_link_uuid; + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{dr_host}{$dr_link_host_uuid}{short_host_name} = $dr_link_short_host_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:anvils::anvil_uuid::${anvil_uuid}::dr_host::${dr_link_host_uuid}::dr_link_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{dr_host}{$dr_link_host_uuid}{dr_link_uuid}, + "s2:anvils::anvil_uuid::${anvil_uuid}::dr_host::${dr_link_host_uuid}::short_host_name" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{dr_host}{$dr_link_host_uuid}{short_host_name}, + }}); + } + } } return(0); @@ -2925,6 +2959,112 @@ WHERE } +=head2 get_dr_links + +This loads the known dr_link devices into the C<< anvil::data >> hash at: + +* dr_links::dr_link_uuid::::dr_link_host_uuid +* dr_links::dr_link_uuid::::dr_link_anvil_uuid +* dr_links::dr_link_uuid::::dr_link_note +* dr_links::dr_link_uuid::::modified_date + +To simplify finding links by host or Anvil! UUID, links to C<< dr_link_uuid >> are stored in these hashes; + +* dr_links::by_anvil_uuid::::dr_link_host_uuid::::dr_link_uuid +* dr_links::by_host_uuid::::dr_link_anvil_uuid::::dr_link_uuid + +If the hash was already populated, it is cleared before repopulating to ensure no stale data remains. + +B<>: Deleted links (ones where C<< dr_link_note >> is set to C<< DELETED >>) are ignored. See the C<< include_deleted >> parameter to include them. + +Parameters; + +=head3 include_deleted (Optional, default 0) + +If set to C<< 1 >>, deleted links are included when loading the data. When C<< 0 >> is set, the default, any dr_link agent with C<< dr_link_note >> set to C<< DELETED >> is ignored. + +=cut +sub get_dr_links +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->get_dr_links()" }}); + + my $include_deleted = defined $parameter->{include_deleted} ? $parameter->{include_deleted} : 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + include_deleted => $include_deleted, + }}); + + if (exists $anvil->data->{dr_links}) + { + delete $anvil->data->{dr_links}; + } + + my $query = " +SELECT + dr_link_uuid, + dr_link_host_uuid, + dr_link_anvil_uuid, + dr_link_note, + modified_date +FROM + dr_links "; + if (not $include_deleted) + { + $query .= " +WHERE + dr_link_note != 'DELETED'"; + } + $query .= " +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $dr_link_uuid = $row->[0]; + my $dr_link_host_uuid = $row->[1]; + my $dr_link_anvil_uuid = $row->[2]; + my $dr_link_note = defined $row->[3] ? $row->[3] : ""; + my $modified_date = $row->[4]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + dr_link_uuid => $dr_link_uuid, + dr_link_host_uuid => $dr_link_host_uuid, + dr_link_anvil_uuid => $dr_link_anvil_uuid, + dr_link_note => $dr_link_note, + modified_date => $modified_date, + }}); + + # Record the data in the hash, too. + $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_host_uuid} = $dr_link_host_uuid; + $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_anvil_uuid} = $dr_link_anvil_uuid; + $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_note} = $dr_link_note; + $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{modified_date} = $modified_date; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "dr_links::dr_link_uuid::${dr_link_uuid}::dr_link_host_uuid" => $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_host_uuid}, + "dr_links::dr_link_uuid::${dr_link_uuid}::dr_link_anvil_uuid" => $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_anvil_uuid}, + "dr_links::dr_link_uuid::${dr_link_uuid}::dr_link_note" => $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_note}, + "dr_links::dr_link_uuid::${dr_link_uuid}::modified_date" => $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{modified_date}, + }}); + + $anvil->data->{dr_links}{by_anvil_uuid}{$dr_link_anvil_uuid}{dr_link_host_uuid}{$dr_link_host_uuid}{dr_link_uuid} = $dr_link_uuid; + $anvil->data->{dr_links}{by_host_uuid}{$dr_link_host_uuid}{dr_link_anvil_uuid}{$dr_link_anvil_uuid}{dr_link_uuid} = $dr_link_uuid; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "dr_links::by_anvil_uuid::${dr_link_anvil_uuid}::dr_link_host_uuid::${dr_link_host_uuid}::dr_link_uuid" => $anvil->data->{dr_links}{by_anvil_uuid}{$dr_link_anvil_uuid}{dr_link_host_uuid}{$dr_link_host_uuid}{dr_link_uuid}, + "dr_links::by_host_uuid::${dr_link_host_uuid}::dr_link_anvil_uuid::${dr_link_anvil_uuid}::dr_link_uuid" => $anvil->data->{dr_links}{by_host_uuid}{$dr_link_host_uuid}{dr_link_anvil_uuid}{$dr_link_anvil_uuid}{dr_link_uuid}, + }}); + } + + return(0); +} + + =head2 get_fences This loads the known fence devices into the C<< anvil::data >> hash at: @@ -7168,6 +7308,241 @@ WHERE } +=head2 insert_or_update_dr_links + +This updates (or inserts) a record in the 'dr_links' table. The C<< dr_link_uuid >> UUID will be returned. + +If there is an error, an empty string is returned. + +Parameters; + +=head3 uuid (optional) + +If set, only the corresponding database will be written to. + +=head3 file (optional) + +If set, this is the file name logged as the source of any INSERTs or UPDATEs. + +=head3 line (optional) + +If set, this is the file line number logged as the source of any INSERTs or UPDATEs. + +=head3 delete (optional) + +If this is set to C<< 1 >>, the record will be deleted. Specifiically, C<< dr_link_note >> is set to C<< DELETED >>. + +=head3 dr_link_uuid (optional, usually) + +This is the specific record to update. If C<< delete >> is set, then either this OR both C<< dr_link_host_uuid >> and C<< dr_link_anvil_uuid >> are required. + +=head3 dr_link_host_uuid (required, must by a host_type -> dr) + +This is the DR host's c<< hosts >> -> C<< host_uuid >>. The host_type is checked and only hosts with C<< host_type >> = C<< dr >> are allowed. + +=head3 dr_link_anvil_uuid (required) + +This is the C<< anvils >> -> C<< anvil_uuid >> that will be allowed to use this DR host. + +=head3 dr_link_note (optional) + +This is an optional note that can be used to store anything. If this is set to C<< DELETED >>, the DR to Anvil! link is severed. + +=cut +sub insert_or_update_dr_links +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_dr_links()" }}); + + my $uuid = defined $parameter->{uuid} ? $parameter->{uuid} : ""; + my $file = defined $parameter->{file} ? $parameter->{file} : ""; + my $line = defined $parameter->{line} ? $parameter->{line} : ""; + my $delete = defined $parameter->{'delete'} ? $parameter->{'delete'} : ""; + my $dr_link_uuid = defined $parameter->{dr_link_uuid} ? $parameter->{dr_link_uuid} : ""; + my $dr_link_host_uuid = defined $parameter->{dr_link_host_uuid} ? $parameter->{dr_link_host_uuid} : ""; + my $dr_link_anvil_uuid = defined $parameter->{dr_link_anvil_uuid} ? $parameter->{dr_link_anvil_uuid} : ""; + my $dr_link_note = defined $parameter->{dr_link_note} ? $parameter->{dr_link_note} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + uuid => $uuid, + file => $file, + line => $line, + dr_link_host_uuid => $dr_link_host_uuid, + dr_link_anvil_uuid => $dr_link_anvil_uuid, + dr_link_note => $dr_link_note, + }}); + + # Make sure that the UUIDs are valid. + $anvil->Database->get_hosts({deubg => $debug}); + $anvil->Database->get_dr_links({debug => $debug}); + + # If deleting, and if we have a valid 'dr_link_uuid' UUID, delete now and be done, + if ($delete) + { + # Do we have a valid dr_link_uuid? + if ($dr_link_uuid) + { + # + if (not exists $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}) + { + # Invalid, can't delete. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0397", variables => { uuid => $dr_link_uuid }}); + return(""); + } + + # If we're here, delete it if it isn't already. + if ($anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_note} ne "DELETED") + { + my $query = " +UPDATE + dr_links +SET + dr_link_node = 'DELETED', + modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." +WHERE + dr_link_uuid = ".$anvil->Database->quote($dr_link_uuid)." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + $anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); + } + return($dr_link_uuid) + } + } + + # Still here? Make sure we've got sane parameters + if (not $dr_link_host_uuid) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_dr_links()", parameter => "dr_link_host_uuid" }}); + return(""); + } + if (not $dr_link_anvil_uuid) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_dr_links()", parameter => "dr_link_anvil_uuid" }}); + return(""); + } + + # We've got UUIDs, but are they valid? + if (not exists $anvil->data->{hosts}{host_uuid}{$dr_link_host_uuid}) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0394", variables => { uuid => $dr_link_host_uuid }}); + return(""); + } + elsif ($anvil->data->{hosts}{host_uuid}{$dr_link_host_uuid}{host_type} ne "dr") + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0395", variables => { + uuid => $dr_link_host_uuid, + name => $anvil->data->{hosts}{host_uuid}{$dr_link_host_uuid}{host_name}, + type => $anvil->data->{hosts}{host_uuid}{$dr_link_host_uuid}{host_type}, + }}); + return(""); + } + if (not exists $anvil->data->{anvils}{anvil_uuid}{$dr_link_anvil_uuid}) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0396", variables => { uuid => $dr_link_anvil_uuid }}); + return(""); + } + + my $dr_host_name = $anvil->data->{hosts}{host_uuid}{$dr_link_host_uuid}{host_name}; + my $anvil_name = $anvil->data->{anvils}{anvil_uuid}{$dr_link_anvil_uuid}{anvil_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + dr_host_name => $dr_host_name, + anvil_name => $anvil_name, + }}); + + # Get the dr_link_uuid, if one exists. + if (not $dr_link_uuid) + { + if ((exists $anvil->data->{dr_links}{by_anvil_uuid}{$dr_link_anvil_uuid}) && + (exists $anvil->data->{dr_links}{by_anvil_uuid}{$dr_link_anvil_uuid}{dr_link_host_uuid}{$dr_link_host_uuid})) + { + $dr_link_uuid = $anvil->data->{dr_links}{by_anvil_uuid}{$dr_link_anvil_uuid}{dr_link_host_uuid}{$dr_link_host_uuid}{dr_link_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { dr_link_uuid => $dr_link_uuid }}); + } + } + + # If we're deleting and we found a dr_link_uuid, DELETE now and return. + if ($delete) + { + if (($dr_link_uuid) && ($anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_note} ne "DELETED")) + { + my $query = " +UPDATE + dr_links +SET + dr_link_node = 'DELETED', + modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." +WHERE + dr_link_uuid = ".$anvil->Database->quote($dr_link_uuid)." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + $anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); + } + return($dr_link_uuid) + } + + # Do we have a UUID? + if ($dr_link_uuid) + { + # Yup. Has something changed? + my $old_dr_link_anvil_uuid = $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_anvil_uuid}; + my $old_dr_link_host_uuid = $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_host_uuid}; + my $old_dr_link_note = $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_note}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + old_dr_link_anvil_uuid => $old_dr_link_anvil_uuid, + old_dr_link_host_uuid => $old_dr_link_host_uuid, + old_dr_link_note => $old_dr_link_note, + }}); + if (($old_dr_link_anvil_uuid ne $dr_link_anvil_uuid) or + ($old_dr_link_host_uuid ne $dr_link_host_uuid) or + ($old_dr_link_note ne $dr_link_note)) + { + # Clear the stop data. + my $query = " +UPDATE + dr_links +SET + dr_link_host_uuid = ".$anvil->Database->quote($dr_link_host_uuid).", + dr_link_anvil_uuid = ".$anvil->Database->quote($dr_link_anvil_uuid).", + dr_link_note = ".$anvil->Database->quote($dr_link_note).", + modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." +WHERE + dr_link_uuid = ".$anvil->Database->quote($dr_link_uuid)." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + $anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); + } + } + else + { + # No, INSERT. + $dr_link_uuid = $anvil->Get->uuid(); + my $query = " +INSERT INTO + dr_links +( + dr_link_uuid, + dr_link_host_uuid, + dr_link_anvil_uuid, + dr_link_note, + modified_date +) VALUES ( + ".$anvil->Database->quote($dr_link_uuid).", + ".$anvil->Database->quote($dr_link_host_uuid).", + ".$anvil->Database->quote($dr_link_anvil_uuid).", + ".$anvil->Database->quote($dr_link_note).", + ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." +); +"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + $anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); + } + + return($dr_link_uuid); +} + + =head2 insert_or_update_fences This updates (or inserts) a record in the 'fences' table. The C<< fence_uuid >> UUID will be returned. @@ -11826,9 +12201,9 @@ WHERE { my $difference = diff \$old_server_definition_xml, \$server_definition_xml, { STYLE => 'Unified' }; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0556", variables => { - server_name => $server_name, + server_name => $server_name, server_definition_server_uuid => $server_definition_server_uuid, - difference => $difference, + difference => $difference, }}); } @@ -14828,7 +15203,7 @@ INSERT INTO { ### NOTE: There is no history schema for states. # The lock is stale. - my $query = "DELETE FROM states WHERE state_uuid = ".$state_uuid.";"; + my $query = "DELETE FROM states WHERE state_uuid = ".$anvil->Database->quote($state_uuid).";"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Database->write({debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__}); } diff --git a/Anvil/Tools/Get.pm b/Anvil/Tools/Get.pm index dfaeef30..bbc1a040 100644 --- a/Anvil/Tools/Get.pm +++ b/Anvil/Tools/Get.pm @@ -42,6 +42,8 @@ my $THIS_FILE = "Get.pm"; # uptime # users_home # uuid +# virsh_list_net +# virsh_list_os # _salt # _wrap_to @@ -466,12 +468,9 @@ Data is store in the following hashes; anvil_resources::::ram::allocated anvil_resources::::ram::hardware anvil_resources::::bridges::::on_nodes - anvil_resources::::bridges::::on_dr anvil_resources::::storage_group::::group_name anvil_resources::::storage_group::::vg_size anvil_resources::::storage_group::::free_size - anvil_resources::::storage_group::::vg_size_on_dr - anvil_resources::::storage_group::::available_on_dr All sizes are stored in bytes. @@ -517,8 +516,7 @@ sub available_resources SELECT anvil_name, anvil_node1_host_uuid, - anvil_node2_host_uuid, - anvil_dr1_host_uuid + anvil_node2_host_uuid FROM anvils WHERE @@ -539,20 +537,13 @@ WHERE } # Get the details. - my $anvil_name = $results->[0]->[0]; - my $node1_host_uuid = $results->[0]->[1]; - my $node2_host_uuid = $results->[0]->[2]; - my $dr1_host_uuid = defined $results->[0]->[3] ? $results->[0]->[3] : ""; + my $anvil_name = $results->[0]->[0]; + my $node1_host_uuid = $results->[0]->[1]; + my $node2_host_uuid = $results->[0]->[2]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_name => $anvil_name, node1_host_uuid => $node1_host_uuid, node2_host_uuid => $node2_host_uuid, - dr1_host_uuid => $dr1_host_uuid, - }}); - - $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} = $dr1_host_uuid ? 1 : 0; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "anvil_resources::${anvil_uuid}::has_dr" => $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr}, }}); # Load hosts and network bridges @@ -570,13 +561,13 @@ WHERE $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads} = 0; $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{hardware} = 0; - foreach my $host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid) + foreach my $host_uuid ($node1_host_uuid, $node2_host_uuid) { - # If DR isn't defined, it'll be blank. - next if not $host_uuid; my $this_is = "node1"; - if ($host_uuid eq $node2_host_uuid) { $this_is = "node2"; } - elsif ($host_uuid eq $dr1_host_uuid) { $this_is = "dr1"; } + if ($host_uuid eq $node2_host_uuid) + { + $this_is = "node2"; + } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { this_is => $this_is }}); # Start collecting data. @@ -650,34 +641,30 @@ WHERE "anvil_resources::${anvil_uuid}::host_uuid::${host_uuid}::ram::hardware" => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$host_uuid}{ram}{hardware}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$host_uuid}{ram}{hardware}}).")", }}); - # For available resources, we only care about nodes. - if ($this_is !~ /^dr/) + # How many cores? + if ((not $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{cores}) or + ($scan_hardware_cpu_cores < $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{cores})) { - # How many cores? - if ((not $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{cores}) or - ($scan_hardware_cpu_cores < $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{cores})) - { - $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{cores} = $scan_hardware_cpu_cores; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "anvil_resources::${anvil_uuid}::cpu::cores" => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{cores}, - }}); - } - if ((not $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}) or - ($scan_hardware_cpu_threads < $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads})) - { - $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads} = $scan_hardware_cpu_threads; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "anvil_resources::${anvil_uuid}::cpu::threads" => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}, - }}); - } - if ((not $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}) or - ($scan_hardware_ram_total < $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{hardware})) - { - $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available} = $scan_hardware_ram_total; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "anvil_resources::${anvil_uuid}::ram::available" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")", - }}); - } + $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{cores} = $scan_hardware_cpu_cores; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "anvil_resources::${anvil_uuid}::cpu::cores" => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{cores}, + }}); + } + if ((not $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}) or + ($scan_hardware_cpu_threads < $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads})) + { + $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads} = $scan_hardware_cpu_threads; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "anvil_resources::${anvil_uuid}::cpu::threads" => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}, + }}); + } + if ((not $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}) or + ($scan_hardware_ram_total < $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{hardware})) + { + $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available} = $scan_hardware_ram_total; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "anvil_resources::${anvil_uuid}::ram::available" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")", + }}); } } @@ -750,7 +737,6 @@ ORDER BY foreach my $bridge_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{bridges}}) { $anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}{on_nodes} = 0; - $anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}{on_dr} = 0; if (($anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}{on}{$node1_host_uuid}) && ($anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}{on}{$node2_host_uuid})) { @@ -759,13 +745,6 @@ ORDER BY "anvil_resources::${anvil_uuid}::bridges::${bridge_name}::on_nodes" => $anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}{on_nodes}, }}); } - if ($anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}{on}{$dr1_host_uuid}) - { - $anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}{on_dr} = 1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "anvil_resources::${anvil_uuid}::bridges::${bridge_name}::on_dr" => $anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}{on_dr}, - }}); - } } foreach my $storage_group_uuid (keys %{$anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}}) @@ -779,8 +758,6 @@ ORDER BY my $node1_vg_free = 0; my $node2_vg_size = 0; my $node2_vg_free = 0; - my $dr1_vg_size = 0; - my $dr1_vg_free = 0; if (exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node1_host_uuid}) { $node1_vg_size = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$node1_host_uuid}{vg_size}; @@ -799,25 +776,12 @@ ORDER BY node2_vg_free => $node2_vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $node2_vg_free}).")", }}); } - if (exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr1_host_uuid}) - { - $dr1_vg_size = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr1_host_uuid}{vg_size}; - $dr1_vg_free = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr1_host_uuid}{vg_free}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - dr1_vg_size => $dr1_vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr1_vg_size}).")", - dr1_vg_free => $dr1_vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr1_vg_free}).")", - }}); - } - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size} = $node2_vg_size < $node1_vg_size ? $node2_vg_size : $node1_vg_size; - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size} = $node2_vg_free < $node1_vg_free ? $node2_vg_free : $node1_vg_free; - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr} = $dr1_vg_size; - $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr} = $dr1_vg_free; + $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size} = $node2_vg_size < $node1_vg_size ? $node2_vg_size : $node1_vg_size; + $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size} = $node2_vg_free < $node1_vg_free ? $node2_vg_free : $node1_vg_free; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "anvil_resources::${anvil_uuid}::storage_group::${storage_group_uuid}::vg_size" => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size}}).")", - "anvil_resources::${anvil_uuid}::storage_group::${storage_group_uuid}::free_size" => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}}).")", - "anvil_resources::${anvil_uuid}::storage_group::${storage_group_uuid}::vg_size_on_dr" => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr}}).")", - "anvil_resources::${anvil_uuid}::storage_group::${storage_group_uuid}::available_on_dr" => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr}}).")", + "anvil_resources::${anvil_uuid}::storage_group::${storage_group_uuid}::vg_size" => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size}}).")", + "anvil_resources::${anvil_uuid}::storage_group::${storage_group_uuid}::free_size" => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}}).")", }}); # Make it easy to sort by group name @@ -2691,6 +2655,194 @@ sub uuid return($uuid); } +=head2 virsh_list_net + +This parses the output from C<< osinfo-query device class=net >> and populated the hash; + + osinfo::net::::vendor = Company name + osinfo::net::::vendor_id = Vendor ID hex value + osinfo::net::::product = Device friendly name + osinfo::net::::product_id = Product ID hex value + +This method takes no parameters. + +=cut +sub virsh_list_net +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Get->virsh_list_net()" }}); + + my $shell_call = $anvil->data->{path}{exe}{'osinfo-query'}." device class=net"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }}); + my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + output => $output, + return_code => $return_code, + }}); + + if (exists $anvil->data->{osinfo}{net}) + { + delete $anvil->data->{osinfo}{net}; + } + + my $last_vendor = ""; + foreach my $line (split/\n/, $output) + { + $line = $anvil->Words->clean_spaces({string => $line}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }}); + + next if $line =~ /Vendor \| Vendor ID/; + next if $line =~ /----+----/; + if ($line =~ /(.*?) \| (.*?) \| (.*?) \| (.*?) \| (.*?) \| (.*?) \| (.*?) \| (.*)$/) + { + my $vendor = $1; + my $vendor_id = $2; + my $product = $3; + my $product_id = $4; + my $name = $5; + my $class = $6; + my $bus = $7; + my $id = $8; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:vendor' => $vendor, + 's2:vendor_id' => $vendor_id, + 's3:product' => $product, + 's4:product_id' => $product_id, + 's5:name' => $name, + 's6:class' => $class, + 's7:bus' => $bus, + 's8:id' => $id, + }}); + + if ($vendor) + { + $last_vendor = $vendor; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { last_vendor => $last_vendor }}); + } + else + { + $vendor = $last_vendor; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { vendor => $vendor }}); + } + next if $bus eq "xen"; + + $anvil->data->{osinfo}{net}{$name}{vendor} = $vendor; + $anvil->data->{osinfo}{net}{$name}{vendor_id} = $vendor_id; + $anvil->data->{osinfo}{net}{$name}{product} = $product; + $anvil->data->{osinfo}{net}{$name}{product_id} = $product_id; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:osinfo::net::${name}::vendor" => $anvil->data->{osinfo}{net}{$name}{vendor}, + "s2:osinfo::net::${name}::vendor_id" => $anvil->data->{osinfo}{net}{$name}{vendor_id}, + "s3:osinfo::net::${name}::product" => $anvil->data->{osinfo}{net}{$name}{product}, + "s4:osinfo::net::${name}::product_id" => $anvil->data->{osinfo}{net}{$name}{product_id}, + }}); + } + } + + # If there's only a 'virtio-net', create a 'virtio' alias. + if ((not exists $anvil->data->{osinfo}{net}{virtio}) && (exists $anvil->data->{osinfo}{net}{'virtio-net'})) + { + $anvil->data->{osinfo}{net}{virtio}{vendor} = $anvil->data->{osinfo}{net}{'virtio-net'}{vendor}; + $anvil->data->{osinfo}{net}{virtio}{vendor_id} = $anvil->data->{osinfo}{net}{'virtio-net'}{vendor_id}; + $anvil->data->{osinfo}{net}{virtio}{product} = $anvil->data->{osinfo}{net}{'virtio-net'}{product}; + $anvil->data->{osinfo}{net}{virtio}{product_id} = $anvil->data->{osinfo}{net}{'virtio-net'}{product_id}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:osinfo::net::virtio::vendor" => $anvil->data->{osinfo}{net}{virtio}{vendor}, + "s2:osinfo::net::virtio::vendor_id" => $anvil->data->{osinfo}{net}{virtio}{vendor_id}, + "s3:osinfo::net::virtio::product" => $anvil->data->{osinfo}{net}{virtio}{product}, + "s4:osinfo::net::virtio::product_id" => $anvil->data->{osinfo}{net}{virtio}{product_id}, + }}); + } + + return(0); +} + +=head2 virsh_list_os + +This parses the output from C<< osinfo-query os >> and populates the hash; + + osinfo::os-list::::name = Operating System name + osinfo::os-list::::version = OS Version + osinfo::os-list::::id = ID URL + +It also loads the OSes into the strings as 'os_list_' = OS + +This method takes no parameters. + +=cut +sub virsh_list_os +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Get->virsh_list_os()" }}); + + my $shell_call = $anvil->data->{path}{exe}{'osinfo-query'}." os"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }}); + my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + output => $output, + return_code => $return_code, + }}); + + if (exists $anvil->data->{osinfo}{'os-list'}) + { + delete $anvil->data->{osinfo}{'os-list'}; + } + + my $last_vendor = ""; + foreach my $line (split/\n/, $output) + { + $line = $anvil->Words->clean_spaces({string => $line}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }}); + + next if $line =~ /Short ID \| Name/; + next if $line =~ /----+----/; + if ($line =~ /(.*?) \| (.*?) \| (.*?) \| (.*)$/) + { + my $short_id = $1; + my $name = $2; + my $version = $3; + my $id = $4; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:short_id' => $short_id, + 's2:name' => $name, + 's3:version' => $version, + 's4:id' => $id, + }}); + + $version = "unknown" if $version eq ""; + + $anvil->data->{osinfo}{'os-list'}{$short_id}{name} = $name; + $anvil->data->{osinfo}{'os-list'}{$short_id}{version} = $version; + $anvil->data->{osinfo}{'os-list'}{$short_id}{id} = $id; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:osinfo::os-list::${short_id}::name" => $anvil->data->{osinfo}{'os-list'}{$short_id}{name}, + "s2:osinfo::os-list::${short_id}::version" => $anvil->data->{osinfo}{'os-list'}{$short_id}{version}, + "s3:osinfo::os-list::${short_id}::id" => $anvil->data->{osinfo}{'os-list'}{$short_id}{id}, + }}); + + my $key = "os_list_".$short_id; + foreach my $file (sort {$a cmp $b} keys %{$anvil->data->{words}}) + { + foreach my $language (sort {$a cmp $b} keys %{$anvil->data->{words}{$file}{language}}) + { + $anvil->data->{words}{$file}{language}{$language}{key}{$key}{content} = $name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "words::${file}::language::${language}::key::${key}::content" => $anvil->data->{words}{$file}{language}{$language}{key}{$key}{content}, + }}); + } + } + } + } + + return(0); +} + # =head3 # # Private Functions; diff --git a/Anvil/Tools/ScanCore.pm b/Anvil/Tools/ScanCore.pm index 7e6cf3bf..968c5ba7 100644 --- a/Anvil/Tools/ScanCore.pm +++ b/Anvil/Tools/ScanCore.pm @@ -1843,7 +1843,7 @@ sub post_scan_analysis_node $anvil->Email->send_alerts(); # Pull the server. - my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all".$anvil->Log->switches; + my $shell_call = $anvil->data->{path}{exe}{'anvil-migrate-server'}." --target local --server all".$anvil->Log->switches; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }}); $anvil->System->call({shell_call => $shell_call}); @@ -1967,7 +1967,7 @@ sub post_scan_analysis_node $anvil->Email->send_alerts(); # Pull the server. - my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all".$anvil->Log->switches; + my $shell_call = $anvil->data->{path}{exe}{'anvil-migrate-server'}." --target local --server all".$anvil->Log->switches; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }}); $anvil->System->call({shell_call => $shell_call}); @@ -2122,7 +2122,7 @@ sub post_scan_analysis_node } $anvil->Email->send_alerts(); - my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all".$anvil->Log->switches; + my $shell_call = $anvil->data->{path}{exe}{'anvil-migrate-server'}." --target local --server all".$anvil->Log->switches; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }}); $anvil->System->call({shell_call => $shell_call}); @@ -2185,7 +2185,7 @@ sub post_scan_analysis_node } $anvil->Email->send_alerts(); - my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all".$anvil->Log->switches; + my $shell_call = $anvil->data->{path}{exe}{'anvil-migrate-server'}." --target local --server all".$anvil->Log->switches; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }}); $anvil->System->call({shell_call => $shell_call}); @@ -2220,7 +2220,7 @@ sub post_scan_analysis_node } $anvil->Email->send_alerts(); - my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all".$anvil->Log->switches; + my $shell_call = $anvil->data->{path}{exe}{'anvil-migrate-server'}." --target local --server all".$anvil->Log->switches; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }}); $anvil->System->call({shell_call => $shell_call}); @@ -2272,7 +2272,7 @@ sub post_scan_analysis_node } $anvil->Email->send_alerts(); - my $shell_call = $anvil->data->{path}{exe}{'anvil-migate-server'}." --target local --server all".$anvil->Log->switches; + my $shell_call = $anvil->data->{path}{exe}{'anvil-migrate-server'}." --target local --server all".$anvil->Log->switches; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0011", variables => { shell_call => $shell_call }}); $anvil->System->call({shell_call => $shell_call}); @@ -2488,7 +2488,12 @@ LIMIT 1;"; } } - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { check_power => $check_power }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + check_power => $check_power, + short_host_name => $short_host_name, + host_ipmi => $host_ipmi, + host_status => $host_status, + }}); if (not $check_power) { next; @@ -2681,7 +2686,6 @@ LIMIT 1;"; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0672", variables => { host_name => $host_name }}); # Check power - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0567", variables => { host_name => $host_name }}); my ($power_health, $shortest_time_on_batteries, $highest_charge_percentage, $estimated_hold_up_time) = $anvil->ScanCore->check_power({ debug => $debug, anvil_uuid => $anvil_uuid, diff --git a/Anvil/Tools/Server.pm b/Anvil/Tools/Server.pm index fffcf80f..76bfec78 100644 --- a/Anvil/Tools/Server.pm +++ b/Anvil/Tools/Server.pm @@ -24,6 +24,31 @@ my $THIS_FILE = "Server.pm"; # migrate_virsh # shutdown_virsh +=cut TODO + +Move all virsh calls over to using Sys::Virt; + +Example; + + #!/usr/bin/perl + use strict; + use warnings; + use Sys::Virt; + + # https://metacpan.org/pod/Sys::Virt::Domain + # https://libvirt.org/api.html + + my $uri = "qemu:///system"; + my $connection = Sys::Virt->new(uri => $uri); + my @domains = $connection->list_domains(); + foreach my $domain (@domains) + { + print $log_fh "Domain: [".$domain->get_name."], UUID: [".$domain->get_uuid_string()."]\n"; + print $log_fh "Definition: [".$domain->get_xml_description."]\n"; + } + +=cut + =pod =encoding utf8 @@ -173,7 +198,7 @@ sub boot_virsh # Is this a local call or a remote call? my ($output, $return_code) = $anvil->System->call({ debug => $debug, - shell_call => $anvil->data->{path}{exe}{virsh}." create ".$definition, + shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." create ".$definition, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output, @@ -286,7 +311,7 @@ sub count_servers my $count = 0; if (-e $anvil->data->{path}{exe}{virsh}) { - my $shell_call = $anvil->data->{path}{exe}{virsh}." list"; + my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }}); my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, debug => $debug}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { @@ -396,12 +421,13 @@ sub find my $host_type = $anvil->Get->host_type({debug => $debug}); my $host_name = $anvil->Get->host_name; + my $virsh_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list --all"; my $virsh_output = ""; my $return_code = ""; if ($anvil->Network->is_local({host => $target})) { # Local call - ($virsh_output, my $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." list --all"}); + ($virsh_output, my $return_code) = $anvil->System->call({shell_call => $virsh_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { virsh_output => $virsh_output, return_code => $return_code, @@ -425,7 +451,7 @@ sub find ($virsh_output, $error, $return_code) = $anvil->Remote->call({ debug => 2, password => $password, - shell_call => $anvil->data->{path}{exe}{virsh}." list --all", + shell_call => $virsh_call, target => $target, remote_user => "root", }); @@ -661,12 +687,19 @@ sub get_status }); # Is this a local call or a remote call? - my $shell_call = $anvil->data->{path}{exe}{virsh}." dumpxml --inactive ".$server; + my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." dumpxml --inactive ".$server; my $this_host = $anvil->Get->short_host_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + shell_call => $shell_call, + this_host => $this_host, + }}); if ($anvil->Network->is_local({host => $target})) { # Local. - ($anvil->data->{server}{$host}{$server}{from_virsh}{xml}, $anvil->data->{server}{$host}{$server}{from_virsh}{return_code}) = $anvil->System->call({shell_call => $shell_call}); + ($anvil->data->{server}{$host}{$server}{from_virsh}{xml}, $anvil->data->{server}{$host}{$server}{from_virsh}{return_code}) = $anvil->System->call({ + debug => $debug, + shell_call => $shell_call, + }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "server::${host}::${server}::from_virsh::xml" => $anvil->data->{server}{$host}{$server}{from_virsh}{xml}, "server::${host}::${server}::from_virsh::return_code" => $anvil->data->{server}{$host}{$server}{from_virsh}{return_code}, @@ -865,7 +898,7 @@ sub map_network # NOTE: We don't use 'Server->find' as the hassle of tracking hosts to target isn't worth it. # Get a list of servers. - my $shell_call = $anvil->data->{path}{exe}{virsh}." list"; + my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list"; my $output = ""; if ($anvil->Network->is_local({host => $target})) { @@ -1032,6 +1065,9 @@ sub migrate_virsh ### NOTE: This method is called by ocf:alteeve:server, which operates without database access. As ### such, queries need to be run only if we've got one or more DB connections. # Mark this server as being in a migration state. + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "sys::database::connections" => $anvil->data->{sys}{database}{connections}, + }}); if ($anvil->data->{sys}{database}{connections}) { $anvil->Database->get_servers({debug => 2}); @@ -1039,14 +1075,24 @@ sub migrate_virsh my $migation_started = time; my $server_uuid = ""; my $old_server_state = ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { migation_started => $migation_started }}); foreach my $this_server_uuid (keys %{$anvil->data->{servers}{server_uuid}}) { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + this_server_uuid => $this_server_uuid, + "servers::server_uuid::${this_server_uuid}::server_name" => $anvil->data->{servers}{server_uuid}{$this_server_uuid}{server_name}, + }}); if ($server eq $anvil->data->{servers}{server_uuid}{$this_server_uuid}{server_name}) { $server_uuid = $this_server_uuid; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { server_uuid => $server_uuid }}); last; } } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + server_uuid => $server_uuid, + "sys::database::connections" => $anvil->data->{sys}{database}{connections}, + }}); if (($server_uuid) && ($anvil->data->{sys}{database}{connections})) { if ($anvil->data->{servers}{server_uuid}{$server_uuid}{server_state} ne "migrating") @@ -1066,12 +1112,15 @@ WHERE } } - # The virsh command switches host names to IPs and needs to have both the source and target IPs in - # the known_hosts file to work. - my $live_migrate = ""; - if (($server_uuid) && ($anvil->data->{servers}{server_uuid}{$server_uuid}{server_live_migration})) + # We default to live migrations, but will remove that switch if it's been set to false. + my $live_migrate = "--live"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + server_uuid => $server_uuid, + "servers::server_uuid::${server_uuid}::server_live_migration" => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_live_migration}, + }}); + if (($server_uuid) && (not $anvil->data->{servers}{server_uuid}{$server_uuid}{server_live_migration})) { - $live_migrate = "--live"; + $live_migrate = ""; } my $target_ip = $anvil->Convert->host_name_to_ip({debug => $debug, host_name => $target}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { @@ -1087,7 +1136,7 @@ WHERE }); } - my $migration_command = $anvil->data->{path}{exe}{virsh}." migrate --undefinesource --tunnelled --p2p ".$live_migrate." ".$server." qemu+ssh://".$target."/system"; + my $migration_command = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." migrate --undefinesource --tunnelled --p2p ".$live_migrate." ".$server." qemu+ssh://".$target."/system"; if ($source) { my $source_ip = $anvil->Convert->host_name_to_ip({debug => $debug, host_name => $source}); @@ -1101,7 +1150,7 @@ WHERE }); } - $migration_command = $anvil->data->{path}{exe}{virsh}." -c qemu+ssh://root\@".$source."/system migrate --undefinesource --tunnelled --p2p ".$live_migrate." ".$server." qemu+ssh://".$target."/system"; + $migration_command = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." -c qemu+ssh://root\@".$source."/system migrate --undefinesource --tunnelled --p2p ".$live_migrate." ".$server." qemu+ssh://".$target."/system"; } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { migration_command => $migration_command }}); @@ -1962,7 +2011,7 @@ sub shutdown_virsh ### TODO: No, don't do this! The server might be migrating # The server is paused. Resume it, wait a few, then proceed with the shutdown. # $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0314", variables => { server => $server }}); -# my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." resume $server"}); +# my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." resume $server"}); # if ($return_code) # { # # Looks like virsh isn't running. @@ -1980,7 +2029,7 @@ sub shutdown_virsh { # The server is suspended. Resume it, wait a few, then proceed with the shutdown. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0317", variables => { server => $server }}); - my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." dompmwakeup $server"}); + my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." dompmwakeup $server"}); if ($return_code) { # Looks like virsh isn't running. @@ -2062,7 +2111,7 @@ WHERE $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0520", variables => { server => $server }}); my ($output, $return_code) = $anvil->System->call({ debug => $debug, - shell_call => $anvil->data->{path}{exe}{virsh}." ".$task." ".$server, + shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." ".$task." ".$server, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output, diff --git a/Anvil/Tools/Storage.pm b/Anvil/Tools/Storage.pm index f81c485b..ec31d35b 100644 --- a/Anvil/Tools/Storage.pm +++ b/Anvil/Tools/Storage.pm @@ -4359,7 +4359,13 @@ sub search_directories # Set a default if nothing was passed. my $array = defined $parameter->{directories} ? $parameter->{directories} : ""; my $initialize = defined $parameter->{initialize} ? $parameter->{initialize} : ""; - + + # If PATH isn't set, set it (could have been scrubbed by a caller). + if (not $ENV{PATH}) + { + $ENV{PATH} = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin"; + } + # If the array is a CSV of directories, convert it now. if ($array =~ /,/) { @@ -5269,12 +5275,12 @@ sub _wait_if_changing if (not $delay) { - $delay = 2; + $delay = 10; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { delay => $delay }}); } elsif (($delay =~ /\D/) or ($delay == 0)) { - $delay = 2; + $delay = 10; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { delay => $delay }}); } diff --git a/Anvil/Tools/System.pm b/Anvil/Tools/System.pm index 1841a339..3e319ef7 100644 --- a/Anvil/Tools/System.pm +++ b/Anvil/Tools/System.pm @@ -294,6 +294,7 @@ sub call }}); } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { timeout => $timeout }}); if ($timeout) { # Prepend a timeout. @@ -301,6 +302,7 @@ sub call $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { shell_call => $shell_call }}); } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { background => $background }}); if ($background) { # Prepend '/tmp/' to STDOUT and/or STDERR output files, if needed. @@ -350,8 +352,10 @@ sub call } else { - $output = ""; - open (my $file_handle, $shell_call.$redirect."; ".$anvil->data->{path}{exe}{echo}." return_code:\$? |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => $secure, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }}); + $output = ""; + my $call_string = $shell_call.$redirect."; ".$anvil->data->{path}{exe}{echo}." return_code:\$? |"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { call_string => $call_string }}); + open (my $file_handle, $call_string) or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => $secure, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }}); while(<$file_handle>) { chomp; @@ -605,6 +609,55 @@ sub check_if_configured $configured = 0 if not defined $configured; $configured = 0 if $configured eq ""; + + if ((not $configured) && (-f $anvil->data->{path}{data}{host_configured})) + { + # See if there's a configured file. + my $body = $anvil->Storage->read_file({debug => $debug, file => $anvil->data->{path}{data}{host_configured}}); + foreach my $line (split/\n/, $body) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }}); + if ($line =~ /^(.*)=(.*)$/) + { + my $variable = $anvil->Words->clean_spaces({string => $1}); + my $value = $anvil->Words->clean_spaces({string => $2}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + variable => $variable, + value => $value, + }}); + + if (($variable eq "system::configured") && ($value eq "1")) + { + # Write the database entry. + my $variable_uuid = $anvil->Database->insert_or_update_variables({ + variable_name => "system::configured", + variable_value => 1, + variable_default => "", + variable_description => "striker_0048", + variable_section => "system", + variable_source_uuid => $anvil->data->{sys}{host_uuid}, + variable_source_table => "hosts", + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { variable_uuid => $variable_uuid }}); + + # mark it as configured. + $configured = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { configured => $configured }}); + } + } + } + } + + if (($configured) && (not -f $anvil->data->{path}{data}{host_configured})) + { + my $failed = $anvil->Storage->write_file({ + debug => $debug, + file => $anvil->data->{path}{data}{host_configured}, + body => "system::configured = 1\n", + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }}); + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { configured => $configured }}); return($configured); } @@ -1458,6 +1511,12 @@ sub collect_ipmi_data return('!!error!!'); } + # Take the double-quotes off the password. + $ipmi_password =~ s/^"(.*)"$/$1/; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + ipmi_password => $anvil->Log->is_secure($ipmi_password), + }}); + my $read_start_time = time; # If there is a password, write it to a temp file. diff --git a/anvil.conf b/anvil.conf index 15eb02e8..06f55fc6 100644 --- a/anvil.conf +++ b/anvil.conf @@ -196,8 +196,8 @@ sys::manage::firewall = 1 ### Server related options # This is the "short list" of servers shown when provisioning a new server. To see the full list of options, -# run '/usr/bin/osinfo-query os' on any machine in the Anvil!. -#sys::servers::os_short_list = debian10,fedora32,freebsd12.1,gentoo,macosx10.7,msdos6.22,openbsd6.7,opensuse15.2,rhel5.11,rhel6.10,rhel7.9,rhel8.3,sles12sp5,solaris11,ubuntu20.04,win10,win2k16,win2k19 +# run '/usr/bin/osinfo-query os' and use here the 'Short ID' entries on any machine in the Anvil!. +#sys::servers::os_short_list = rhel8.7,rhel9.1,win10,win2k19,win2k22 ### Scan agent options diff --git a/cgi-bin/striker b/cgi-bin/striker index 12e39b89..b862492f 100755 --- a/cgi-bin/striker +++ b/cgi-bin/striker @@ -2017,10 +2017,12 @@ sub run_manifest my $bcn_count = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{count}{bcn}; my $sn_count = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{count}{sn}; my $ifn_count = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{count}{ifn}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + my $mn_count = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{count}{mn}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bcn_count => $bcn_count, sn_count => $sn_count, ifn_count => $ifn_count, + mn_count => $mn_count, }}); # If confirmed, run! @@ -2355,7 +2357,7 @@ sub run_manifest # Show the networks. my $networks = ""; my $default_seen = 0; - foreach my $network ("bcn", "sn", "ifn") + foreach my $network ("bcn", "sn", "mn", "ifn") { my $count = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{count}{$network}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { @@ -2389,6 +2391,7 @@ sub run_manifest my $network_key = "header_0036"; if ($network eq "sn") { $network_key = "header_0037"; } elsif ($network eq "ifn") { $network_key = "header_0038"; } + elsif ($network eq "mn") { $network_key = "header_0106"; } $networks .= $anvil->Template->get({file => "anvil.html", name => "run-manifest-network", variables => { name => $anvil->Words->string({key => $network_key, variables => { number => $i }}), network => $network_range, @@ -2402,7 +2405,7 @@ sub run_manifest # Pull out the IPs that will be assigned to servers. my $machine_ips = ""; - foreach my $network ("bcn", "sn", "ifn") + foreach my $network ("bcn", "sn", "mn", "ifn") { if ($network eq "sn") { @@ -2440,6 +2443,7 @@ sub run_manifest my $network_key = "header_0036"; if ($network eq "sn") { $network_key = "header_0037"; } elsif ($network eq "ifn") { $network_key = "header_0038"; } + elsif ($network eq "mn") { $network_key = "header_0106"; } $machine_ips .= $anvil->Template->get({file => "anvil.html", name => "run-manifest-ip", variables => { name => $anvil->Words->string({key => $network_key, variables => { number => $i }}), node1 => $node1_ip ? $node1_ip : "--", @@ -2533,6 +2537,8 @@ sub handle_manifest $anvil->data->{cgi}{bcn_count}{alert} = 0 if not defined $anvil->data->{cgi}{bcn_count}{alert}; $anvil->data->{cgi}{sn_count}{value} = 0 if not defined $anvil->data->{cgi}{sn_count}{value}; $anvil->data->{cgi}{sn_count}{alert} = 0 if not defined $anvil->data->{cgi}{sn_count}{alert}; + $anvil->data->{cgi}{mn_count}{value} = 0 if not defined $anvil->data->{cgi}{mn_count}{value}; + $anvil->data->{cgi}{mn_count}{alert} = 0 if not defined $anvil->data->{cgi}{mn_count}{alert}; $anvil->data->{cgi}{ifn_count}{value} = 0 if not defined $anvil->data->{cgi}{ifn_count}{value}; $anvil->data->{cgi}{ifn_count}{alert} = 0 if not defined $anvil->data->{cgi}{ifn_count}{alert}; $anvil->data->{cgi}{dns}{value} = "8.8.8.8,8.8.4.4" if not defined $anvil->data->{cgi}{dns}{value}; @@ -2549,6 +2555,7 @@ sub handle_manifest "cgi::sequence::value" => $anvil->data->{cgi}{sequence}{value}, "cgi::bcn_count::value" => $anvil->data->{cgi}{bcn_count}{value}, "cgi::sn_count::value" => $anvil->data->{cgi}{sn_count}{value}, + "cgi::mn_count::value" => $anvil->data->{cgi}{mn_count}{value}, "cgi::ifn_count::value" => $anvil->data->{cgi}{ifn_count}{value}, "cgi::dns::value" => $anvil->data->{cgi}{dns}{value}, "cgi::ntp::value" => $anvil->data->{cgi}{ntp}{value}, @@ -2749,6 +2756,11 @@ sub handle_manifest $anvil->data->{cgi}{sn_count}{value} = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{count}{sn}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "cgi::sn_count::value" => $anvil->data->{cgi}{sn_count}{value} }}); } + if (not $anvil->data->{cgi}{mn_count}{value}) + { + $anvil->data->{cgi}{mn_count}{value} = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{count}{mn}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "cgi::mn_count::value" => $anvil->data->{cgi}{mn_count}{value} }}); + } if (not $anvil->data->{cgi}{bcn_count}{value}) { $anvil->data->{cgi}{bcn_count}{value} = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{count}{bcn}; @@ -2773,7 +2785,7 @@ sub handle_manifest $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "cgi::mtu::value" => $anvil->data->{cgi}{mtu}{value} }}); } - foreach my $network ("bcn", "sn", "ifn") + foreach my $network ("bcn", "sn", "mn", "ifn") { my $count_key = $network."_count"; foreach my $i (1..$anvil->data->{cgi}{$count_key}{value}) @@ -2812,7 +2824,7 @@ sub handle_manifest $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "cgi::${ipmi_ip_key}::value" => $anvil->data->{cgi}{$ipmi_ip_key}{value} }}); } - foreach my $network ("bcn", "sn", "ifn") + foreach my $network ("bcn", "sn", "mn", "ifn") { my $count_key = $network."_count"; foreach my $i (1..$anvil->data->{cgi}{$count_key}{value}) @@ -2921,6 +2933,11 @@ sub handle_manifest $anvil->data->{cgi}{sn_count}{value} = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "cgi::sn_count::value" => $anvil->data->{cgi}{sn_count}{value} }}); } + if (not $anvil->data->{cgi}{mn_count}{value}) + { + $anvil->data->{cgi}{mn_count}{value} = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "cgi::mn_count::value" => $anvil->data->{cgi}{mn_count}{value} }}); + } if (not $anvil->data->{cgi}{bcn_count}{value}) { $anvil->data->{cgi}{bcn_count}{value} = 1; @@ -2941,9 +2958,11 @@ sub handle_manifest sequence_class => $anvil->data->{cgi}{sequence}{alert} ? "input_alert" : "", ifn_count => $anvil->data->{cgi}{ifn_count}{value}, ifn_count_class => $anvil->data->{cgi}{ifn_count}{alert} ? "input_alert" : "", - sn_count => $anvil->data->{cgi}{sn_count}{value}, + sn_count => $anvil->data->{cgi}{sn_count}{value}, sn_count_class => $anvil->data->{cgi}{sn_count}{alert} ? "input_alert" : "", - bcn_count => $anvil->data->{cgi}{bcn_count}{value}, + mn_count => $anvil->data->{cgi}{mn_count}{value}, + mn_count_class => $anvil->data->{cgi}{mn_count}{alert} ? "input_alert" : "", + bcn_count => $anvil->data->{cgi}{bcn_count}{value}, bcn_count_class => $anvil->data->{cgi}{bcn_count}{alert} ? "input_alert" : "", }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'form::body' => $anvil->data->{form}{body} }}); @@ -3000,29 +3019,29 @@ sub handle_manifest subnet => $select, }}); } - - # There's only ever 1 SN. Just in case we change our mind later, we'll set it up as if it's + + # There's only ever 1 SN. Just in case we change our mind later, we'll set it up as if it's # variable. - foreach my $i (1..$anvil->data->{cgi}{bcn_count}{value}) + foreach my $i (1..$anvil->data->{cgi}{sn_count}{value}) { my $say_sn = $anvil->Words->string({key => "striker_0020", variables => { number => '1' }}); my $network_key = "sn".$i."_network"; my $subnet_key = "sn".$i."_subnet"; my $gateway_key = "sn".$i."_gateway"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - say_sn => $say_sn, - network_key => $network_key, - subnet_key => $subnet_key, - gateway_key => $gateway_key, + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + say_sn => $say_sn, + network_key => $network_key, + subnet_key => $subnet_key, + gateway_key => $gateway_key, }}); - + $anvil->data->{cgi}{$network_key}{value} = "10.10".$i.".0.0" if not defined $anvil->data->{cgi}{$network_key}{value}; $anvil->data->{cgi}{$network_key}{alert} = 0 if not defined $anvil->data->{cgi}{$network_key}{alert}; $anvil->data->{cgi}{$subnet_key}{value} = "255.255.0.0" if not defined $anvil->data->{cgi}{$subnet_key}{value}; $anvil->data->{cgi}{$subnet_key}{alert} = 0 if not defined $anvil->data->{cgi}{$subnet_key}{alert}; $anvil->data->{cgi}{$gateway_key}{value} = "" if not defined $anvil->data->{cgi}{$gateway_key}{value}; $anvil->data->{cgi}{$gateway_key}{alert} = 0 if not defined $anvil->data->{cgi}{$gateway_key}{alert}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "cgi::${network_key}::value" => $anvil->data->{cgi}{$network_key}{value}, "cgi::${network_key}::alert" => $anvil->data->{cgi}{$network_key}{alert}, "cgi::${subnet_key}::value" => $anvil->data->{cgi}{$subnet_key}{value}, @@ -3030,17 +3049,60 @@ sub handle_manifest "cgi::${gateway_key}::value" => $anvil->data->{cgi}{$gateway_key}{value}, "cgi::${gateway_key}::alert" => $anvil->data->{cgi}{$gateway_key}{alert}, }}); - + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "cgi::sn1_network::value" => $anvil->data->{cgi}{sn1_network}{value} }}); $network_form .= $anvil->Template->get({file => "anvil.html", name => "manifest-step2-network-entry", variables => { network => $say_sn, - network_name => $network_key, + network_name => $network_key, network_class => $anvil->data->{cgi}{$network_key}{alert} ? "input_alert" : "", - network_value => $anvil->data->{cgi}{$network_key}{value}, + network_value => $anvil->data->{cgi}{$network_key}{value}, subnet => '255.255.0.0 ', }}); } - + + # There's only ever 1 MN. Just in case we change our mind later, we'll set it up as if it's + # variable. + if ($anvil->data->{cgi}{mn_count}{value}) + { + foreach my $i (1..$anvil->data->{cgi}{mn_count}{value}) + { + my $say_mn = $anvil->Words->string({key => "striker_0299", variables => { number => '1' }}); + my $network_key = "mn".$i."_network"; + my $subnet_key = "mn".$i."_subnet"; + my $gateway_key = "mn".$i."_gateway"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + say_mn => $say_mn, + network_key => $network_key, + subnet_key => $subnet_key, + gateway_key => $gateway_key, + }}); + + $anvil->data->{cgi}{$network_key}{value} = "10.199.0.0" if not defined $anvil->data->{cgi}{$network_key}{value}; + $anvil->data->{cgi}{$network_key}{alert} = 0 if not defined $anvil->data->{cgi}{$network_key}{alert}; + $anvil->data->{cgi}{$subnet_key}{value} = "255.255.0.0" if not defined $anvil->data->{cgi}{$subnet_key}{value}; + $anvil->data->{cgi}{$subnet_key}{alert} = 0 if not defined $anvil->data->{cgi}{$subnet_key}{alert}; + $anvil->data->{cgi}{$gateway_key}{value} = "" if not defined $anvil->data->{cgi}{$gateway_key}{value}; + $anvil->data->{cgi}{$gateway_key}{alert} = 0 if not defined $anvil->data->{cgi}{$gateway_key}{alert}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "cgi::${network_key}::value" => $anvil->data->{cgi}{$network_key}{value}, + "cgi::${network_key}::alert" => $anvil->data->{cgi}{$network_key}{alert}, + "cgi::${subnet_key}::value" => $anvil->data->{cgi}{$subnet_key}{value}, + "cgi::${subnet_key}::alert" => $anvil->data->{cgi}{$subnet_key}{alert}, + "cgi::${gateway_key}::value" => $anvil->data->{cgi}{$gateway_key}{value}, + "cgi::${gateway_key}::alert" => $anvil->data->{cgi}{$gateway_key}{alert}, + }}); + + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "cgi::mn1_network::value" => $anvil->data->{cgi}{mn1_network}{value} }}); + $network_form .= $anvil->Template->get({file => "anvil.html", name => "manifest-step2-network-entry", variables => { + network => $say_mn, + network_name => $network_key, + network_class => $anvil->data->{cgi}{$network_key}{alert} ? "input_alert" : "", + network_value => $anvil->data->{cgi}{$network_key}{value}, + subnet => '255.255.0.0 ', + }}); + } + } + # Now IFNs foreach my $i (1..$anvil->data->{cgi}{ifn_count}{value}) { @@ -3091,7 +3153,7 @@ sub handle_manifest gateway_value => $anvil->data->{cgi}{$gateway_key}{value}, }}); } - + my $back_link = $anvil->data->{sys}{cgi_string}; $back_link =~ s/step=2/step=1/; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { back_link => $back_link}}); @@ -3125,7 +3187,7 @@ sub handle_manifest my $network_form = ""; my $network_note = ""; - foreach my $network ("bcn", "sn", "ifn") + foreach my $network ("bcn", "sn", "mn", "ifn") { if ($network eq "sn") { @@ -3146,6 +3208,7 @@ sub handle_manifest my $say_network_code = "striker_0018"; if ($network eq "sn") { $say_network_code = "striker_0020"; } elsif ($network eq "ifn") { $say_network_code = "striker_0022"; } + elsif ($network eq "ifn") { $say_network_code = "striker_0299"; } my $say_network = $anvil->Words->string({key => $say_network_code, variables => { number => $i }}); my $network_key = $network.$i."_network"; @@ -3423,7 +3486,7 @@ sub sanity_check_manifest_step3 { # It's a valid IP. Does it match any BCN or IFN network? my $match_found = 0; - foreach my $network ("bcn", "sn", "ifn") + foreach my $network ("bcn", "sn", "mn", "ifn") { my $count_key = $network."_count"; foreach my $i (1..$anvil->data->{cgi}{$count_key}{value}) @@ -3464,7 +3527,7 @@ sub sanity_check_manifest_step3 } # Now check that the IPs are sane. - foreach my $network ("bcn", "sn", "ifn") + foreach my $network ("bcn", "sn", "mn", "ifn") { my $count_key = $network."_count"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { @@ -3477,6 +3540,7 @@ sub sanity_check_manifest_step3 my $say_network_code = "striker_0018"; if ($network eq "sn") { $say_network_code = "striker_0020"; } elsif ($network eq "ifn") { $say_network_code = "striker_0022"; } + elsif ($network eq "mn") { $say_network_code = "striker_0299"; } my $say_network = $anvil->Words->string({key => $say_network_code, variables => { number => $i }}); my $network_key = $network.$i."_network"; @@ -3598,7 +3662,18 @@ sub sanity_check_manifest_step2 $sane = check_network($anvil, $sane, $say_sn, "sn1_network", "sn1_subnet", "sn1_gateway"); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sane => $sane }}); - + + # There's only ever 1 MN + my $say_mn = $anvil->Words->string({key => "striker_0299", variables => { number => '1' }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mn => $say_mn }}); + + # The migration network always uses the subnet /16 and has no gateway. + $anvil->data->{cgi}{mn1_subnet}{value} = "255.255.0.0" if not defined $anvil->data->{cgi}{mn1_subnet}{value}; + $anvil->data->{cgi}{mn1_gateway}{value} = "" if not defined $anvil->data->{cgi}{mn1_gateway}{value}; + + $sane = check_network($anvil, $sane, $say_mn, "mn1_network", "mn1_subnet", "mn1_gateway"); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sane => $sane }}); + # Now IFNs foreach my $i (1..$anvil->data->{cgi}{ifn_count}{value}) { @@ -5356,10 +5431,13 @@ sub process_prep_network process_anvil_menu($anvil); return(0); } - + my $host_uuid = $anvil->data->{cgi}{host_uuid}{value}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_uuid => $host_uuid }}); + # Pull the host's data out of the JSON file. $anvil->Striker->parse_all_status_json(); + my $host_type = ""; my $host_name = ""; foreach my $host (sort {$a cmp $b} keys %{$anvil->data->{json}{all_status}{hosts}}) { @@ -5371,7 +5449,11 @@ sub process_prep_network { # Found it. $host_name = $host; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_name => $host_name }}); + $host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + host_name => $host_name, + host_type => $host_type, + }}); last; } } @@ -5404,7 +5486,7 @@ sub process_prep_network foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{json}{all_status}{hosts}{$host_name}{network_interface}{interface}}) { # if any interfaces are called 'virbrX-nic', ignore it as it will be removed. Likewise, ignore any 'vnetX' devices. - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { interface => $interface }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { interface => $interface }}); if (($interface =~ /^virbr\d+-nic/) or ($interface =~ /^vnet\d+/)) { # Ignore it. @@ -5417,35 +5499,37 @@ sub process_prep_network # Store the mac address . $interfaces->{$interface} = $mac_address; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "interfaces->{".$interface."}" => $interfaces->{$interface} }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "interfaces->{".$interface."}" => $interfaces->{$interface} }}); } # Get the interface count my $interface_count = keys %{$interfaces}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { interface_count => $interface_count }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { interface_count => $interface_count }}); - if ($interface_count < 6) + if (($host_type eq "node") && ($interface_count < 6)) { # Not enough interfaces. $anvil->data->{form}{error_massage} = $anvil->Template->get({file => "main.html", name => "error_message", variables => { error_message => $anvil->Words->string({key => "warning_0015", variables => { interface_count => $interface_count } }) }}); $anvil->data->{cgi}{task}{value} = ""; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "cgi::task::value" => $anvil->data->{cgi}{task}{value}, - "form::error_massage" => $anvil->data->{form}{error_massage}, + "form::error_massage" => $anvil->data->{form}{error_massage}, }}); process_anvil_menu($anvil); } - # TODO: For now, we're only allowing one BCN and SN. So at this time, we'll show one BCN pair, one SN - # pair and N-IFN pairs. + # TODO: For now, we're only allowing one BCN, SN and MN. So at this time, we'll show one BCN pair, + # one SN pair, one MN pair and N-IFN pairs. my $bcn_pair_count = 1; my $sn_pair_count = 1; - my $ifn_pair_count = int(($interface_count - 4) / 2); + my $mn_pair_count = $interface_count > 6 ? 1 : 0; + my $ifn_pair_count = int(($interface_count - 6) / 2); $ifn_pair_count = 1 if $ifn_pair_count < 1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bcn_pair_count => $bcn_pair_count, sn_pair_count => $sn_pair_count, ifn_pair_count => $ifn_pair_count, + mn_pair_count => $mn_pair_count, }}); ### NOTE: The weird 'form::config_step2::::value is from reusing the logic used back when @@ -5548,7 +5632,7 @@ sub process_prep_network }); } } - foreach my $network ("bcn", "sn", "ifn") + foreach my $network ("bcn", "sn", "mn", "ifn") { my $count_key = $network."_count"; my $loops = $anvil->data->{cgi}{$count_key}{value}; @@ -5578,6 +5662,10 @@ sub process_prep_network { $say_network = $anvil->Words->string({key => "striker_0020", variables => { number => $i }}); } + elsif ($network eq "mn") + { + $say_network = $anvil->Words->string({key => "striker_0299", variables => { number => $i }}); + } elsif ($network eq "ifn") { $say_network = $anvil->Words->string({key => "striker_0022", variables => { number => $i }}); @@ -5737,7 +5825,7 @@ sub process_prep_network if ((not $anvil->data->{cgi}{$link1_key}{value}) && (not $anvil->data->{cgi}{$link2_key}{value})) { # If this is network 1, both are required. - if ($i == 1) + if (($host_type eq "node") && ($i == 1)) { # Required. my $error_message = $anvil->Words->string({key => "warning_0021"}); @@ -5896,7 +5984,8 @@ sub process_prep_network bcn_count => $bcn_pair_count, sn_count => $sn_pair_count, ifn_count => $ifn_pair_count, - gateway => $anvil->data->{cgi}{gateway}{value}, + mn_count => $mn_pair_count, + gateway => $anvil->data->{cgi}{gateway}{value}, dns => $anvil->data->{cgi}{dns}{value}, }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'form::body' => $anvil->data->{form}{body} }}); @@ -5921,7 +6010,7 @@ sub process_prep_network my $interface_form = ""; # NOTE: We don't assign IPs at this point, unless the user manually sets one. We'll set all to 'dhcp' # until set during the Anvil! build later. - foreach my $network ("bcn", "sn", "ifn") + foreach my $network ("bcn", "sn", "mn", "ifn") { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { network => $network }}); my $name_key = ""; @@ -5945,7 +6034,13 @@ sub process_prep_network $description_key = "striker_0023"; $count = $ifn_pair_count; } - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + elsif ($network eq "mn") + { + $name_key = "striker_0299"; + $description_key = "striker_0300"; + $count = $mn_pair_count; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { name_key => $name_key, description_key => $description_key, count => $count, @@ -6006,7 +6101,7 @@ sub process_prep_network iface1_select => $this_iface1_form, iface2_select => $this_iface2_form, network_name => $network.$i, - create_bridge => $network eq "sn" ? 0 : 1, + create_bridge => (($network eq "sn") or ($network eq "mn")) ? 0 : 1, }}); } } @@ -6064,6 +6159,7 @@ sub process_prep_network bcn_count => $bcn_pair_count, sn_count => $sn_pair_count, ifn_count => $ifn_pair_count, + mn_count => $mn_pair_count, host_uuid => $anvil->data->{cgi}{host_uuid}{value}, host_name => $host_name, # This is the current host name, used to find the data in the all_status.json }}); @@ -8674,6 +8770,8 @@ x = Network; - SN = 100 + network ie: SN1 = 10.101.y.z SN2 = 10.102.y.z + - MN = 199 + ie: MN1 = 10.199.y.z y = Device Type. Foudation Pack; diff --git a/html/skins/alteeve/anvil.html b/html/skins/alteeve/anvil.html index 74d081a1..e04cfaca 100644 --- a/html/skins/alteeve/anvil.html +++ b/html/skins/alteeve/anvil.html @@ -558,6 +558,7 @@ + @@ -775,8 +776,9 @@ - + + @@ -1104,6 +1106,7 @@ + @@ -2455,6 +2458,7 @@ + @@ -2557,6 +2561,7 @@ + diff --git a/notes b/notes index d6b28bc4..b3c19c85 100644 --- a/notes +++ b/notes @@ -1,13 +1,15 @@ +Add 'lsof' and 'strace' to Required + + When pairing Striker, make sure new config goes to all known nodes! -Immediately set drbdadm to 'secondary' after 'primary --force' + dnf -y update && dnf -y install https://www.alteeve.com/an-repo/m3/anvil-release-latest.noarch.rpm && alteeve-repo-setup -y && dnf -y install anvil-striker --allowerasing dnf -y update && dnf -y install https://www.alteeve.com/an-repo/m3/anvil-release-latest.noarch.rpm && alteeve-repo-setup -y && dnf -y install anvil-node --allowerasing dnf -y update && dnf -y install https://www.alteeve.com/an-repo/m3/anvil-release-latest.noarch.rpm && alteeve-repo-setup -y && dnf -y install anvil-dr --allowerasing - ### Currently set default zone; # Doesn't seem to matter - /etc/firewalld/firewalld.conf:6:DefaultZone=public diff --git a/ocf/alteeve/server b/ocf/alteeve/server index d5a4a225..d4c9defa 100755 --- a/ocf/alteeve/server +++ b/ocf/alteeve/server @@ -3,7 +3,7 @@ # This is the resource agent used to manage servers on the Anvil! Intelligent Availability platform. # # License: GNU General Public License (GPL) v2+ -# (c) 1997-2021 - Alteeve's Niche! Inc. +# (c) 1997-2023 - Alteeve's Niche! Inc. # # WARNING: This is a pretty purpose-specific resource agent. No effort was made to test this on an rgmanager # cluster or on any configuration outside how the Anvil! m3 uses it. If you plan to adapt it to @@ -125,10 +125,18 @@ $anvil->data->{environment}{OCF_ROOT} = defined $ENV{O $anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_source} = defined $ENV{OCF_RESKEY_CRM_meta_migrate_source} ? $ENV{OCF_RESKEY_CRM_meta_migrate_source} : ""; $anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target} = defined $ENV{OCF_RESKEY_CRM_meta_migrate_target} ? $ENV{OCF_RESKEY_CRM_meta_migrate_target} : ""; $anvil->data->{environment}{OCF_RESKEY_CRM_meta_record_pending} = defined $ENV{OCF_RESKEY_CRM_meta_record_pending} ? $ENV{OCF_RESKEY_CRM_meta_record_pending} : ""; + +# When run from 'pcs', environment variables are scrubbed, which can cause us problems. +if (not $ENV{PATH}) +{ + $ENV{PATH} = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin"; +} + # Any variable=value arguments in the resource are set under 'OCF_RESKEY_CRM_meta_' foreach my $key (sort {$a cmp $b} keys %ENV) { - next if $key !~ /^OCF_RESKEY_CRM_meta_/; + #$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { "ENV{".$key."}" => $ENV{$key} }}); + next if $key !~ /^OCF_/; $anvil->data->{environment}{$key} = $ENV{$key}; } @@ -199,8 +207,8 @@ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level foreach my $key (sort {$a cmp $b} keys %{$anvil->data->{environment}}) { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "environment::${key}" => $anvil->data->{environment}{$key}, + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "environment::${key}" => $anvil->data->{environment}{$key}, }}); } foreach my $key (sort {$a cmp $b} keys %{$anvil->data->{switches}}) @@ -218,6 +226,14 @@ if (($anvil->data->{switches}{server}) && (not $anvil->data->{environment}{OCF_R "environment::OCF_RESKEY_name" => $anvil->data->{environment}{OCF_RESKEY_name}, }}); } +elsif ((not $anvil->data->{switches}{server}) && ($anvil->data->{environment}{OCF_RESKEY_name})) +{ + $anvil->data->{switches}{server} =$anvil->data->{environment}{OCF_RESKEY_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "switches::server" => $anvil->data->{switches}{server}, + }}); +} + if (($anvil->data->{switches}{migrate_to}) && (not $anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target})) { $anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target} = $anvil->data->{switches}{migrate_to}; @@ -1059,7 +1075,7 @@ sub stop_server # Read in an parse the server's XML. $anvil->System->check_storage(); - $anvil->Server->get_status({server => $server}); + $anvil->Server->get_status({debug => 2, server => $server}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0313", variables => { server => $server }}); my $success = $anvil->Server->shutdown_virsh({server => $server}); @@ -1281,7 +1297,14 @@ pmsuspended - The domain has been suspended by guest power management, e.g. ente sub migrate_server { my ($anvil) = @_; - + + # Do a sensitive, quick DB connect. + $anvil->Database->connect({ + check_for_resync => 0, + retry => 0, + sensitive => 1, + }); + ### NOTE: For now, we're not going to block if the target is not UpToDate. There are times when a ### user might want to do this (ie: sync will be done soon and the need to evacuate the node ### ASAP is high). Maybe we'll enforce this and require a '--force' switch later? @@ -1594,7 +1617,7 @@ sub validate_all source => $source, target => $target, }}); - + # Log what we're doing. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0581", variables => { server => $server }}); diff --git a/scancore-agents/scan-apc-pdu/scan-apc-pdu b/scancore-agents/scan-apc-pdu/scan-apc-pdu index 0dcfb2d3..5f58f311 100755 --- a/scancore-agents/scan-apc-pdu/scan-apc-pdu +++ b/scancore-agents/scan-apc-pdu/scan-apc-pdu @@ -162,14 +162,6 @@ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => " # Read switches $anvil->Get->switches; -# Too many connections cause the UPS to lag out, so we only run on Strikers. -my $host_type = $anvil->Get->host_type(); -$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }}); -if (($host_type ne "striker") && (not $anvil->data->{switches}{force})) -{ - $anvil->nice_exit({exit_code => 1}); -} - # Handle start-up tasks my $problem = $anvil->ScanCore->agent_startup({agent => $THIS_FILE}); if ($problem) @@ -177,6 +169,17 @@ if ($problem) $anvil->nice_exit({exit_code => 1}); } +# The PDUs don't allow multiple connections at the same time. This causes a lot of false alerts when many +# machines try to scan. As such, only Striker dashboards watch APC PDUs. +my $host_type = $anvil->Get->host_type(); +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }}); +if (($host_type ne "striker") && (not $anvil->data->{switches}{force})) +{ + # Exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_apc_pdu_message_0041", variables => { program => $THIS_FILE }}); + $anvil->nice_exit({exit_code => 0}); +} + if ($anvil->data->{switches}{purge}) { # This can be called when doing bulk-database purges. @@ -1173,15 +1176,26 @@ WHERE # Have any outlets changed? foreach my $scan_apc_pdu_outlet_number (sort {$a cmp $b} keys %{$anvil->data->{pdu}{scan_apc_pdu_uuid}{$scan_apc_pdu_uuid}{scan_apc_pdu_outlets}}) { + my $scan_apc_pdu_serial_number = $anvil->data->{pdu}{scan_apc_pdu_uuid}{$scan_apc_pdu_uuid}{scan_apc_pdu}{scan_apc_pdu_serial_number}; my $new_scan_apc_pdu_outlet_name = $anvil->data->{pdu}{scan_apc_pdu_uuid}{$scan_apc_pdu_uuid}{scan_apc_pdu_outlets}{$scan_apc_pdu_outlet_number}{scan_apc_pdu_outlet_name}; my $new_scan_apc_pdu_outlet_on_phase = $anvil->data->{pdu}{scan_apc_pdu_uuid}{$scan_apc_pdu_uuid}{scan_apc_pdu_outlets}{$scan_apc_pdu_outlet_number}{scan_apc_pdu_outlet_on_phase}; my $new_scan_apc_pdu_outlet_state = $anvil->data->{pdu}{scan_apc_pdu_uuid}{$scan_apc_pdu_uuid}{scan_apc_pdu_outlets}{$scan_apc_pdu_outlet_number}{scan_apc_pdu_outlet_state}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + scan_apc_pdu_serial_number => $scan_apc_pdu_serial_number, new_scan_apc_pdu_outlet_name => $new_scan_apc_pdu_outlet_name, new_scan_apc_pdu_outlet_on_phase => $new_scan_apc_pdu_outlet_on_phase, new_scan_apc_pdu_outlet_state => $new_scan_apc_pdu_outlet_state, }}); + if (($new_scan_apc_pdu_outlet_on_phase ne "!!no_connection!!") or ($new_scan_apc_pdu_outlet_on_phase ne "#!no_value!#")) + { + $anvil->Alert->check_condition_age({clear => 1, name => "scan_apc_pdu::pdu::".$scan_apc_pdu_serial_number."::phase_lost::".$scan_apc_pdu_outlet_number}); + } + if (($new_scan_apc_pdu_outlet_state eq "!!no_connection!!") or ($new_scan_apc_pdu_outlet_state eq "#!no_value!#")) + { + $anvil->Alert->check_condition_age({clear => 1, name => "scan_apc_pdu::pdu::".$scan_apc_pdu_serial_number."::outlet_lost::".$scan_apc_pdu_outlet_number}); + } + # Do I know about this outlet? if ($anvil->data->{sql}{scan_apc_pdu_uuid}{$scan_apc_pdu_uuid}{scan_apc_pdu_outlets}{$scan_apc_pdu_outlet_number}{scan_apc_pdu_outlet_uuid}) { @@ -1241,7 +1255,20 @@ WHERE } if ($new_scan_apc_pdu_outlet_on_phase ne $old_scan_apc_pdu_outlet_on_phase) { - # Phase changed, but why tho? + # Phase changed. If the new phase is '!!no_connection!!', it + # could be contention, so check if this has been the case for + # at least five minutes. + if (($new_scan_apc_pdu_outlet_on_phase eq "!!no_connection!!") or ($new_scan_apc_pdu_outlet_on_phase eq "#!no_value!#")) + { + my $age = $anvil->Alert->check_condition_age({name => "scan_apc_pdu::pdu::".$scan_apc_pdu_serial_number."::phase_lost::".$scan_apc_pdu_outlet_number}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { age => $age }}); + + if ($age < 600) + { + # Ignore it for now. + next; + } + } $changes = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { changes => $changes }}); @@ -1262,6 +1289,19 @@ WHERE } if ($new_scan_apc_pdu_outlet_state ne $old_scan_apc_pdu_outlet_state) { + # If we had a contention and the new value is '!!no_connection!!'. + if (($new_scan_apc_pdu_outlet_state eq "!!no_connection!!") or ($new_scan_apc_pdu_outlet_state eq "#!no_value!#")) + { + my $age = $anvil->Alert->check_condition_age({name => "scan_apc_pdu::pdu::".$scan_apc_pdu_serial_number."::outlet_lost::".$scan_apc_pdu_outlet_number}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { age => $age }}); + + if ($age < 600) + { + # Ignore it for now. + next; + } + } + # This is likely from a fence action, so we make it critical $changes = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { changes => $changes }}); @@ -1658,11 +1698,11 @@ sub clear_phase_low_warning if ($changed) { # Register an alert-cleared event. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0029"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0029", variables => { phase => $scan_apc_pdu_phase_number, name => $pdu_host_name }}); $anvil->Alert->register({ alert_level => "notice", clear => 1, - message => "scan_apc_pdu_message_0029", + message => "scan_apc_pdu_message_0029,!!phase!".$scan_apc_pdu_phase_number."!!,!!name!".$pdu_host_name."!!", set_by => $THIS_FILE, }); } @@ -1683,11 +1723,11 @@ sub set_phase_high_warning if ($changed) { # Register an alert-cleared event. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0027"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0027", variables => { phase => $scan_apc_pdu_phase_number, name => $pdu_host_name }}); $anvil->Alert->register({ alert_level => "notice", clear => 1, - message => "scan_apc_pdu_message_0027", + message => "scan_apc_pdu_message_0027,!!phase!".$scan_apc_pdu_phase_number."!!,!!name!".$pdu_host_name."!!", set_by => $THIS_FILE, }); } @@ -1709,11 +1749,11 @@ sub clear_phase_high_warning if ($changed) { # Register an alert-cleared event. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0028"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0028", variables => { phase => $scan_apc_pdu_phase_number, name => $pdu_host_name }}); $anvil->Alert->register({ alert_level => "notice", clear => 1, - message => "scan_apc_pdu_message_0028", + message => "scan_apc_pdu_message_0028,!!phase!".$scan_apc_pdu_phase_number."!!,!!name!".$pdu_host_name."!!", set_by => $THIS_FILE, }); } @@ -1734,11 +1774,11 @@ sub set_phase_high_critical if ($changed) { # Register an alert-cleared event. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0025"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0025", variables => { phase => $scan_apc_pdu_phase_number, name => $pdu_host_name }}); $anvil->Alert->register({ alert_level => "notice", clear => 1, - message => "scan_apc_pdu_message_0025", + message => "scan_apc_pdu_message_0025,!!phase!".$scan_apc_pdu_phase_number."!!,!!name!".$pdu_host_name."!!", set_by => $THIS_FILE, }); } @@ -1760,11 +1800,11 @@ sub clear_phase_high_critical if ($changed) { # Register an alert-cleared event. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0026"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_apc_pdu_message_0026", variables => { phase => $scan_apc_pdu_phase_number, name => $pdu_host_name }}); $anvil->Alert->register({ alert_level => "notice", clear => 1, - message => "scan_apc_pdu_message_0026", + message => "scan_apc_pdu_message_0026,!!phase!".$scan_apc_pdu_phase_number."!!,!!name!".$pdu_host_name."!!", set_by => $THIS_FILE, }); } diff --git a/scancore-agents/scan-apc-pdu/scan-apc-pdu.xml b/scancore-agents/scan-apc-pdu/scan-apc-pdu.xml index 50f48ac3..320eec55 100644 --- a/scancore-agents/scan-apc-pdu/scan-apc-pdu.xml +++ b/scancore-agents/scan-apc-pdu/scan-apc-pdu.xml @@ -81,6 +81,7 @@ A new PDU: [#!variable!name!#] has been found - Phase: [#!variable!phase!#] current amperage draw: [#!variable!amps!#]. - Outlet: [#!variable!outlet!#], on phase: [#!variable!on_phase!#] is: [#!variable!state!#] (name: [#!variable!name!#]). The PDU model: [#!variable!model!#] at the IP address: [#!variable!ip_address!#] has vanished! Did the network cable come unplugged? + APC PDUs only allow one connection at a time. To avoid contention, only Striker dashboards scan APC PDUs. If you want this to run, you can use '--force'. Exiting. Unknown diff --git a/scancore-agents/scan-cluster/scan-cluster.sql b/scancore-agents/scan-cluster/scan-cluster.sql index 13338531..ed6b1966 100644 --- a/scancore-agents/scan-cluster/scan-cluster.sql +++ b/scancore-agents/scan-cluster/scan-cluster.sql @@ -127,7 +127,7 @@ CREATE TRIGGER trigger_scan_cluster_nodes -- TODO: We may want to track this data in the future. For now, we're not going to bother as we can always -- dig through the historical cib.xml.X files on the nodes. -- --- -- Constraints; Useful for tracking when servers are asked to migate. +-- -- Constraints; Useful for tracking when servers are asked to migrate. -- CREATE TABLE scan_cluster_constraints ( -- scan_cluster_constraint_uuid uuid primary key, -- scan_cluster_constraint_scan_cluster_uuid uuid not null, -- The parent scan_cluster_uuid. diff --git a/scancore-agents/scan-drbd/scan-drbd b/scancore-agents/scan-drbd/scan-drbd index e070e094..8307ba6c 100755 --- a/scancore-agents/scan-drbd/scan-drbd +++ b/scancore-agents/scan-drbd/scan-drbd @@ -154,6 +154,7 @@ sub check_config } my $updated = $anvil->DRBD->update_global_common({ + debug => 2, usage_count => $anvil->data->{sys}{privacy}{strong} ? 0 : 1, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { updated => $updated }}); diff --git a/scancore-agents/scan-network/scan-network b/scancore-agents/scan-network/scan-network index adbce66d..455d79c8 100755 --- a/scancore-agents/scan-network/scan-network +++ b/scancore-agents/scan-network/scan-network @@ -1054,12 +1054,14 @@ sub collect_data 'state' => $state, }}); - if ($state ne "active") + # This could be 'active' or 'activated' + if ($state !~ /activ/) { # Try brinding the interface up. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "warning_0147", variables => { interface => $interface, uptime => $uptime, + 'state' => $state, }}); my $shell_call = $anvil->data->{path}{exe}{ifup}." ".$name; diff --git a/scancore-agents/scan-server/scan-server b/scancore-agents/scan-server/scan-server index d61de45f..390fcc2e 100755 --- a/scancore-agents/scan-server/scan-server +++ b/scancore-agents/scan-server/scan-server @@ -91,6 +91,9 @@ record_migration_times($anvil); # Check if we need to update the websocket stuff. check_vnc($anvil); +# Check that there's a DRBD fence rule for each server. +check_drbd_fence_rules($anvil); + # Shut down. $anvil->ScanCore->agent_shutdown({agent => $THIS_FILE}); @@ -99,6 +102,73 @@ $anvil->ScanCore->agent_shutdown({agent => $THIS_FILE}); # Functions # ############################################################################################################# +# Check that there's a DRBD fence rule for each server. +sub check_drbd_fence_rules +{ + my ($anvil) = @_; + + my $host_type = $anvil->Get->host_type(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }}); + if ($host_type ne "node") + { + return(0); + } + + my $problem = $anvil->Cluster->parse_cib(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }}); + if ($problem) + { + return(0); + } + + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "cib::parsed::local::ready" => $anvil->data->{cib}{parsed}{'local'}{ready}, + }}); + if (not $anvil->data->{cib}{parsed}{'local'}{ready}) + { + return(0); + } + + foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{'scan-server'}{server_name}}) + { + my $drbd_fence_rule_exists = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_rule}{'exists'}; + my $drbd_fence_rule_attribute = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_rule}{attribute}; + my $drbd_fence_rule_operation = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_rule}{operation}; + my $drbd_fence_rule_value = $anvil->data->{cib}{parsed}{data}{server}{$server_name}{drbd_fence_rule}{value}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:server_name' => $server_name, + 's2:drbd_fence_rule_exists' => $drbd_fence_rule_exists, + 's3:drbd_fence_rule_attribute' => $drbd_fence_rule_attribute, + 's4:drbd_fence_rule_operation' => $drbd_fence_rule_operation, + 's5:drbd_fence_rule_value' => $drbd_fence_rule_value, + }}); + + ### TODO: Verify that the other values are correct. + # If it's missing, add it + if (not $drbd_fence_rule_exists) + { + # Create it. + my $variables = { + server => $server_name, + }; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_server_alert_0019", variables => $variables}); + $anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0019", variables => $variables, set_by => $THIS_FILE}); + + # + my $shell_call = $anvil->data->{path}{exe}{pcs}." constraint location ".$server_name." rule score=-INFINITY drbd-fenced_".$server_name." eq 1"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); + + my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); + } + } + + return(0); +} + # sub check_vnc { @@ -755,7 +825,7 @@ DELETED - Marks a server as no longer existing } if ($server_host_uuid ne $old_server_host_uuid) { - # Server migated (to the peer or to a new Anvil!) + # Server migrated (to the peer or to a new Anvil!) my $variables = { server => $server_name, old_host_name => $old_server_host_uuid eq "NULL" ? "NULL" : $anvil->Get->host_name_from_uuid({host_uuid => $old_server_host_uuid}), diff --git a/scancore-agents/scan-server/scan-server.xml b/scancore-agents/scan-server/scan-server.xml index 009fa620..930427d6 100644 --- a/scancore-agents/scan-server/scan-server.xml +++ b/scancore-agents/scan-server/scan-server.xml @@ -110,6 +110,9 @@ The definition for the server: [#!variable!server!#] was changed in the database #!variable!new_difference!# ====================== + +There was no DRBD fence rule for the: [#!variable!server!#] in the pacemaker configuration. Adding it now. + Starting: [#!variable!program!#]. diff --git a/share/anvil.sql b/share/anvil.sql index f832360d..cdcbb03a 100644 --- a/share/anvil.sql +++ b/share/anvil.sql @@ -408,6 +408,59 @@ CREATE TRIGGER trigger_anvils FOR EACH ROW EXECUTE PROCEDURE history_anvils(); +-- This is the new method of tracking DR hosts and while Anvil! node pairs they can back up. This allows DR +-- hosts to protect multiple nodes, and allow multiple DRs to protect one node pair. +CREATE TABLE dr_links ( + dr_link_uuid uuid not null primary key, + dr_link_host_uuid uuid not null, + dr_link_anvil_uuid uuid not null, + dr_link_note text, -- Set to 'DELETE' when no longer used. + modified_date timestamp with time zone not null, + + FOREIGN KEY(dr_link_host_uuid) REFERENCES hosts(host_uuid), + FOREIGN KEY(dr_link_anvil_uuid) REFERENCES anvils(anvil_uuid) +); +ALTER TABLE dr_links OWNER TO admin; + +CREATE TABLE history.dr_links ( + history_id bigserial, + dr_link_uuid uuid, + dr_link_host_uuid uuid, + dr_link_anvil_uuid uuid, + dr_link_note text, + modified_date timestamp with time zone not null +); +ALTER TABLE history.dr_links OWNER TO admin; + +CREATE FUNCTION history_dr_links() RETURNS trigger +AS $$ +DECLARE + history_dr_links RECORD; +BEGIN + SELECT INTO history_dr_links * FROM dr_links WHERE dr_link_uuid = new.dr_link_uuid; + INSERT INTO history.dr_links + (dr_link_uuid, + dr_link_host_uuid, + dr_link_anvil_uuid, + dr_link_note, + modified_date) + VALUES + (history_dr_links.dr_link_uuid, + history_dr_links.dr_link_host_uuid, + history_dr_links.dr_link_anvil_uuid, + history_dr_links.dr_link_note, + history_dr_links.modified_date); + RETURN NULL; +END; +$$ +LANGUAGE plpgsql; +ALTER FUNCTION history_dr_links() OWNER TO admin; + +CREATE TRIGGER trigger_dr_links + AFTER INSERT OR UPDATE ON dr_links + FOR EACH ROW EXECUTE PROCEDURE history_dr_links(); + + -- This stores alerts coming in from various sources CREATE TABLE alerts ( alert_uuid uuid not null primary key, diff --git a/share/words.xml b/share/words.xml index 4be9d6a3..3f26cb44 100644 --- a/share/words.xml +++ b/share/words.xml @@ -16,7 +16,7 @@ Author: Madison Kelly Anvil! Striker ScanCore - Alteeve's Niche! Inc., Toronto, Ontario, Canada]]> + Alteeve's Niche! Inc., Toronto, Ontario, Canada]]> Anvil!]]> Node DR Host @@ -566,7 +566,13 @@ The definition data passed in was: * 2 or "warning" * 3 or "notice" * 4 or "info" - + [ Error ] - The host UUID: [#!variable!uuid!#] was not found. + [ Error ] - The host UUID: [#!variable!uuid!#], with the host name: [#!variable!name!#] is of host type: [#!variable!type!#]. This must be a type 'dr'. + [ Error ] - The Anvil! UUID: [#!variable!uuid!#] was not found. + [ Error ] - The DR link UUID: [#!variable!uuid!#] was not found. + [ Error ] - There was a problem processing the requested network: [#!variable!network!#]. Details should be logged. + [ Error ] - It looks like the new device: [#!variable!resource!#] failed to appear. Unable to proceed. + @@ -956,7 +962,7 @@ resource #!variable!server!# { RAM Used RAM Free Bridges - Storage Group + #!free!# Used Free Anvil! Node @@ -984,7 +990,8 @@ resource #!variable!server!# { Host Type Host UUID Machines - + MN link #!variable!number!# + Configure Network The network configuration will be updated based on the variables stored in the database. Reconnecting to the machine using the new IP address may be required. @@ -1153,7 +1160,7 @@ Failure! The return code: [#!variable!return_code!#] was received ('0' was expec * Please enter a number between 1 and #!variable!max_cores!#. -=] Available cores / threads: [#!variable!cores!# / #!variable!threads!#] - Node #!variable!core!# CPU Model: [#!variable!model!#] - - DR Host CPU: .... [#!variable!model!#], [#!variable!cores!#c]/[#!variable!threads!#t] + #!free!# RAM: ........... [#!variable!ram!#] * Please enter a valid amount up to: [#!variable!ram_total!# / #!variable!ram_available!#]. -=] Available RAM: [#!variable!ram_available!#] @@ -1161,9 +1168,8 @@ Failure! The return code: [#!variable!return_code!#] was received ('0' was expec - Allocated to servers: [#!variable!ram_allocated!#] - Node 1 RAM (total): . [#!variable!ram_node1!#] - Node 2 RAM (total): . [#!variable!ram_node2!#] - - DR Host RAM (total): [#!variable!ram_available!#] - Available on Anvil!: [#!variable!vg_free!#], Total: [#!variable!vg_size!#] - Available on DR: ... [#!variable!dr_free!#], Total: [#!variable!dr_size!#] + #!free!# + Available on Anvil!: [#!variable!vg_free!#], Total: [#!variable!vg_size!#] Storage Group: . [#!variable!storage_group!#] * Please enter a number beside the storage group you want to use. -=] Storage groups @@ -1197,7 +1203,9 @@ It should be provisioned in the next minute or two. The LV(s) behind the resource: [#!variable!resource!#] already existed, and the DRBD resource is not in the disk state 'UpToDate'. As such, we'll keep waiting before provisioning the server. The resource needs to be forced to UpToDate as it is brand now, doing that now. -=] OS Short List - * Please enter an OS key that is closest to your target OS. Run 'osinfo-query os' for a full list. + * Please enter an OS key that is closest to your target OS. +From any machine in the Anvil!, run 'osinfo-query os'. +Use the 'Short ID' that best matches your OS. Optimize for: .. [#!variable!os!#] Ready to provision the server! Please be patient, this could take a moment. The call to create the server will be: ==== @@ -1221,7 +1229,7 @@ It should be provisioned in the next minute or two. The server has been flagged as deleted now. The server delete is complete on this host! It looks like ScanCore has not yet run on one or both nodes in this Anvil! system. Missing resource data, so unable to proceed. - Manually calling 'scan-drbd' to ensure that the new agent is recorded. + Manually calling 'scan-drbd' to ensure that the new resource is recorded. The server name: [#!variable!server_name!#] is already used by another server. Deleting the server's definition file: [#!variable!file!#]... The server: [#!variable!server_name!#] was not found in the cluster configuration. This can happen if a server was partially deleted and we're trying again. @@ -1470,7 +1478,45 @@ Note: This is a permanent action! If you protect this server again later, a full The server: [#!variable!server!#] is still running two minutes after asking it to stop. It might have woken up on the first press and ignored the shutdown request (Hi Windows). Pressing the poewr button again. Copying the Long-throw (drbd proxy) license file: [#!variable!file!#] into place. The fence device: [#!variable!device!#] no longer has a port associated with it, will remove it. - + Calling drbdadm adjust to load the new resource, then waiting for the DRBD device to appear. + The new DRBD resource: [#!variable!resource!#] now exists! + Still waiting for the new DRBD resource: [#!variable!resource!#] to appear... + Waiting for up to a minute to see if the peer connects before provisioning the server. + One or more peer disk state or roles are 'unknown', waiting: [#!variable!waiting!#] seconds longer. + Peer disk state or role still unknown after one minute, proceeding without it. + '.]]> + '.]]> + '. Valid options match 'virt-install --os-variant' (run: 'osinfo-query os' and reference the 'Short ID' column).]]> + + '.]]> + + '.]]> + + '. Valid options are:]]> + + + '. Max is: [#!variable!storage_group_size!#].]]> + '. Max will depend on selected --storage-group.]]> + + '. Valid options are:]]> + + + + + + + + + + + + Starting: [#!variable!program!#]. @@ -2303,7 +2349,9 @@ The file: [#!variable!file!#] needs to be updated. The difference is: The DRBD Proxy license file has expired. None of the MAC sddresses in the The DRBD Proxy license file match any of the MAC addresses on this system. The DRBD Proxy license file: [#!data!path::configs::drbd-proxy.license!#] is missing expected data or is malformed. - + Updating logind to ignore ACPI power button events so that IPMI-based fence requests don't trigger an attempt to gracefully shut down. For more information, see: https://access.redhat.com/solutions/1578823 + Restarting the daemon: [#!variable!daemon!#]. + The host name: [#!variable!target!#] does not resolve to an IP address. The connection to: [#!variable!connection!#] was refused. If you recently booted the target, the network might have started, the ssh daemon might not be running yet. @@ -2634,7 +2682,7 @@ Are you sure that you want to delete the server: [#!variable!server_name!#]? [Ty It appears that another instance of 'anvil-safe-start' is already runing. Please wait for it to complete (or kill it manually if needed). Preparing to rename a server. Preparing to rename stop this node. - This records how long it took to migate a given server. The average of the last five migations is used to guess how long future migrations will take. + This records how long it took to migrate a given server. The average of the last five migations is used to guess how long future migrations will take. One or more servers are migrating. While this is the case, ScanCore post-scan checks are not performed. Preventative live migration has completed. Preventative live migration has been disabled. We're healthier than our peer, but we will take no action. @@ -3137,7 +3185,9 @@ If you are comfortable that the target has changed for a known reason, you can s This indicates that this node or DR host has had base DRBD configured. This indicates that this node or DR host has completed all tasks needed to be a full member of the Anvil!. TCP Port - + Migration Network link #!variable!number!# + This is where you configure the optional network dedicated to RAM-copy during live migrations. + #!variable!number!#/sec s @@ -3298,7 +3348,7 @@ Here we will inject 't_0006', which injects 't_0001' which has a variable: [#!st [ Warning ] - Failed to log into the host. Is the IP or root user's password right? Click here to resolve.]]> [ Warning ] - The host UUID: [#!variable!host_uuid!#] was not found in the #!data!path::json::all_status!# file on the local dashboard. - [ Warning ] - To configure a host as either an #!string!brand_0002!# node or a disaster recovery host, there must be at least 6 network interfaces. This machine only has: [#!variable!interface_count!#] interfaces. + [ Warning ] - To configure a #!string!brand_0002!# sub-node, there must be at least 6 network interfaces. This machine only has: [#!variable!interface_count!#] interfaces. [ Warning ] - No databases are available. Changes to the network interfaces will be cached. [ Warning ] - The subnet mask is not valid [ Warning ] - The IP address was specified, but the subnet mask was not @@ -3497,766 +3547,8 @@ The error was: #!variable!error!# ======== - [ Warning ] - The interface: [#!variable!interface!#] is in a bond, but it is down. The system uptime is: [#!variable!uptime!#], so it might be a problem where the interface didn't start on boot as it should have. So we're going to bring the interface up. + [ Warning ] - The interface: [#!variable!interface!#] appears to be down (state: [#!variable!state!#]). The system uptime is: [#!variable!uptime!#], so it might be a problem where the interface didn't start on boot as it should have. So we're going to bring the interface up. [ Warning ] - The IPMI stonith resource: [#!variable!resource!#] is in the role: [#!variable!role!#] (should be 'Started'). Will check the IPMI config now. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -4283,7 +3575,7 @@ The error was: Anvil! ストライカ スカンコア - Alteeve's Niche! Inc., トロント、オンタリオ、カナダ]]> + Alteeve's Niche! Inc., トロント、オンタリオ、カナダ]]> diff --git a/tools/Makefile.am b/tools/Makefile.am index 5f228021..d2733ca1 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -43,6 +43,7 @@ dist_sbin_SCRIPTS = \ anvil-update-issue \ anvil-update-states \ anvil-update-system \ + anvil-version-changes \ anvil-watch-bonds \ scancore \ striker-auto-initialize-all \ @@ -67,7 +68,8 @@ fencedir = ${FASEXECPREFIX}/sbin dist_fence_SCRIPTS = \ fence_delay \ - fence_pacemaker + fence_pacemaker \ + unfence_pacemaker sharedir = ${datarootdir}/anvil diff --git a/tools/anvil-boot-server b/tools/anvil-boot-server index d1693399..a64e5ce7 100755 --- a/tools/anvil-boot-server +++ b/tools/anvil-boot-server @@ -9,6 +9,7 @@ # # TODO: # - Add support for boot ordering. +# - Check which node we want to put on and set a location constraint to prefer that node before calling pcs. # use strict; diff --git a/tools/anvil-configure-host b/tools/anvil-configure-host index 9a65699e..ed03a1fb 100755 --- a/tools/anvil-configure-host +++ b/tools/anvil-configure-host @@ -213,6 +213,7 @@ sub reconfigure_network my $organization = exists $anvil->data->{variables}{form}{config_step1}{organization}{value} ? $anvil->data->{variables}{form}{config_step1}{organization}{value} : ""; my $bcn_count = exists $anvil->data->{variables}{form}{config_step1}{bcn_count}{value} ? $anvil->data->{variables}{form}{config_step1}{bcn_count}{value} : 1; my $sn_count = exists $anvil->data->{variables}{form}{config_step1}{sn_count}{value} ? $anvil->data->{variables}{form}{config_step1}{sn_count}{value} : 0; + my $mn_count = exists $anvil->data->{variables}{form}{config_step1}{mn_count}{value} ? $anvil->data->{variables}{form}{config_step1}{mn_count}{value} : 0; my $ifn_count = exists $anvil->data->{variables}{form}{config_step1}{ifn_count}{value} ? $anvil->data->{variables}{form}{config_step1}{ifn_count}{value} : 1; my $new_host_name = exists $anvil->data->{variables}{form}{config_step2}{host_name}{value} ? $anvil->data->{variables}{form}{config_step2}{host_name}{value} : ""; my $type = $anvil->Get->host_type(); @@ -222,8 +223,9 @@ sub reconfigure_network domain => $domain, organization => $organization, bcn_count => $bcn_count, - sn_count => $sn_count, - ifn_count => $ifn_count, + sn_count => $sn_count, + mn_count => $mn_count, + ifn_count => $ifn_count, new_host_name => $new_host_name, type => $type, }}); @@ -324,7 +326,7 @@ sub reconfigure_network if (not $gateway_interface) { # IFN first, BCN second, SN last - foreach my $network_type ("ifn", "bcn", "sn") + foreach my $network_type ("ifn", "bcn", "sn", "mn") { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_type => $network_type }}); @@ -332,6 +334,7 @@ sub reconfigure_network if ($network_type eq "bcn") { $count = $bcn_count; } elsif ($network_type eq "sn") { $count = $sn_count; } elsif ($network_type eq "ifn") { $count = $ifn_count; } + elsif ($network_type eq "mn") { $count = $mn_count; } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); next if not $count; @@ -403,6 +406,9 @@ ORDER BY 's2:variable_name' => $variable_name, 's3:variable_value' => $variable_value, }}); + + # An undefined interface will have the MAC address value set to '1', ignore those. + next if $variable_value = 1; if ($variable_name =~ /form::config_step2::(.*?)_mac_to_set::value/) { @@ -442,7 +448,7 @@ ORDER BY # This will be set to '1' if we make a change. my $changes = 0; my $new_interfaces = []; - foreach my $network_type ("bcn", "sn", "ifn") + foreach my $network_type ("bcn", "sn", "mn", "ifn") { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_type => $network_type }}); @@ -450,6 +456,7 @@ ORDER BY if ($network_type eq "bcn") { $count = $bcn_count; } elsif ($network_type eq "sn") { $count = $sn_count; } elsif ($network_type eq "ifn") { $count = $ifn_count; } + elsif ($network_type eq "mn") { $count = $mn_count; } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); next if not $count; @@ -571,6 +578,12 @@ ORDER BY $say_interface = "sn".$network_count; $interface_prefix = "SN"; } + elsif ($network_type eq "mn") + { + $say_network = "Migration Network ".$network_count; + $say_interface = "mn".$network_count; + $interface_prefix = "MN"; + } elsif ($network_type eq "ifn") { $say_network = "Internet-Facing Network ".$network_count; @@ -1140,6 +1153,12 @@ ORDER BY $say_interface = "ifn".$network_count; $interface_prefix = "IFN"; } + elsif ($network_type eq "mn") + { + $say_network = "Migration Network ".$network_count; + $say_interface = "mn".$network_count; + $interface_prefix = "MN"; + } my $say_defroute = $is_gateway ? "yes" : "no"; my $cidr = $anvil->Convert->cidr({subnet_mask => $subnet_mask}); my $new_link1_file = $anvil->data->{path}{directories}{ifcfg}."/ifcfg-".$interface_prefix."_".$network_count."_-_Link_1"; diff --git a/tools/anvil-daemon b/tools/anvil-daemon index c4e5487c..9b20a131 100755 --- a/tools/anvil-daemon +++ b/tools/anvil-daemon @@ -1294,192 +1294,15 @@ sub handle_special_cases { my ($anvil) = @_; - my $host_type = $anvil->Get->host_type(); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }}); - if ($host_type ne "striker") - { - ### TODO: Test that this is fixed. The bug is now ERRATA - # RHBZ #1961562 - https://bugzilla.redhat.com/show_bug.cgi?id=1961562#c16 - # We're a node or DR host. We need to touch this file. - my $work_around_file = "/etc/qemu/firmware/50-edk2-ovmf-cc.json"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { work_around_file => $work_around_file }}); - if (not -e $work_around_file) - { - $anvil->Storage->write_file({ - debug => 2, - file => $work_around_file, - body => "", - overwrite => 0, - backup => 0, - mode => "0644", - user => "root", - group => "root", - }); - } - - # Make sure DRBD compiled after a kernel upgrade. - $anvil->DRBD->_initialize_kmod({debug => 2}); - } - - if ($host_type eq "striker") - { - # This converts the old/broken 'notifications' tables with the more appropriately named 'alert-override' - if (1) - { - foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}}) - { - my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_catalog = 'anvil' AND table_name = 'notifications';"; - $anvil->Log->variables({source => $THIS_FILE, uuid => $uuid, line => __LINE__, level => 2, list => { query => $query }}); - - my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); - - if ($count) - { - my $queries = []; - push @{$queries}, "DROP FUNCTION history_notifications() CASCADE;"; - push @{$queries}, "DROP TABLE history.notifications;"; - push @{$queries}, "DROP TABLE public.notifications;"; - push @{$queries}, q|CREATE TABLE alert_overrides ( - alert_override_uuid uuid not null primary key, - alert_override_recipient_uuid uuid not null, -- The recipient we're linking. - alert_override_host_uuid uuid not null, -- This host_uuid of the referenced machine - alert_override_alert_level integer not null, -- This is the alert level (at or above) that this user wants alerts from. If set to '-1', the record is deleted. - modified_date timestamp with time zone not null, - - FOREIGN KEY(alert_override_host_uuid) REFERENCES hosts(host_uuid), - FOREIGN KEY(alert_override_recipient_uuid) REFERENCES recipients(recipient_uuid) -); -ALTER TABLE alert_overrides OWNER TO admin; - -CREATE TABLE history.alert_overrides ( - history_id bigserial, - alert_override_uuid uuid, - alert_override_recipient_uuid uuid, - alert_override_host_uuid uuid, - alert_override_alert_level integer, - modified_date timestamp with time zone not null -); -ALTER TABLE history.alert_overrides OWNER TO admin; - -CREATE FUNCTION history_alert_overrides() RETURNS trigger -AS $$ -DECLARE - history_alert_overrides RECORD; -BEGIN - SELECT INTO history_alert_overrides * FROM alert_overrides WHERE alert_override_uuid = new.alert_override_uuid; - INSERT INTO history.alert_overrides - (alert_override_uuid, - alert_override_recipient_uuid, - alert_override_host_uuid, - alert_override_alert_level, - modified_date) - VALUES - (history_alert_overrides.alert_override_uuid, - history_alert_overrides.alert_override_recipient_uuid, - history_alert_overrides.alert_override_host_uuid, - history_alert_overrides.alert_override_alert_level, - history_alert_overrides.modified_date); - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; -ALTER FUNCTION history_alert_overrides() OWNER TO admin; - -CREATE TRIGGER trigger_alert_overrides - AFTER INSERT OR UPDATE ON alert_overrides - FOR EACH ROW EXECUTE PROCEDURE history_alert_overrides(); -|; - foreach my $query (@{$queries}) - { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); - } - $anvil->Database->write({debug => 2, uuid => $uuid, query => $queries, source => $THIS_FILE, line => __LINE__}); - } - } - } - - # This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release) - if (0) - { - foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}}) - { - my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_catalog = 'anvil' AND table_name = 'audits';"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); - - my $count = $anvil->Database->query({query => $query, uuid => $uuid, source => $THIS_FILE, line => __LINE__})->[0]->[0]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); - - if (not $count) - { - # Add the table. - my $query = q| -CREATE TABLE audits ( - audit_uuid uuid primary key, - audit_user_uuid uuid not null, -- This is the users -> user_uuid the audit is tracking - audit_details text not null, -- This is the information explaining the action being audited. - modified_date timestamp with time zone not null, - - FOREIGN KEY(audit_user_uuid) REFERENCES users(user_uuid) -); -ALTER TABLE audits OWNER TO admin; - -CREATE TABLE history.audits ( - history_id bigserial, - audit_uuid uuid, - audit_user_uuid uuid, - audit_details text, - modified_date timestamp with time zone not null -); -ALTER TABLE history.audits OWNER TO admin; - -CREATE FUNCTION history_audits() RETURNS trigger -AS $$ -DECLARE - history_audits RECORD; -BEGIN - SELECT INTO history_audits * FROM audits WHERE audit_uuid = new.audit_uuid; - INSERT INTO history.audits - (audit_uuid, - audit_user_uuid, - audit_details, - modified_date) - VALUES - (history_audit.audit_uuid, - history_audit.audit_user_uuid, - history_audit.audit_details, - history_audit.modified_date); - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; -ALTER FUNCTION history_audits() OWNER TO admin; - -CREATE TRIGGER trigger_audits - AFTER INSERT OR UPDATE ON audits - FOR EACH ROW EXECUTE PROCEDURE history_audits(); -|; - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); - $anvil->Database->write({debug => 2, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); - } - } - } - } + # Thsi is now handled by 'anvil-version-changes' + my $shell_call = $anvil->data->{path}{exe}{'anvil-version-changes'}.$anvil->Log->switches; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); - ### TODO: Remove these later. This is here to clean up how we used to handle db_in_use and lock_request flags. - if (1) - { - # Broadly clear all states that are '0' now. - my $queries = []; - push @{$queries}, "DELETE FROM states WHERE state_name LIKE 'db_in_use::%' AND state_note != '1';"; - push @{$queries}, "DELETE FROM history.variables WHERE variable_name = 'lock_request';"; - push @{$queries}, "DELETE FROM variables WHERE variable_name = 'lock_request';"; - foreach my $query (@{$queries}) - { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); - } - $anvil->Database->write({debug => 2, query => $queries, source => $THIS_FILE, line => __LINE__}); - } + my ($states_output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call, source => $THIS_FILE, line => __LINE__}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + states_output => $states_output, + return_code => $return_code, + }}); return(0); } diff --git a/tools/anvil-join-anvil b/tools/anvil-join-anvil index d553c24b..9dfe8b5c 100755 --- a/tools/anvil-join-anvil +++ b/tools/anvil-join-anvil @@ -1068,7 +1068,11 @@ sub configure_pacemaker } } } - + + # Make sure logind is update to handle fencing properly + # see - https://access.redhat.com/solutions/1578823 + $anvil->Cluster->configure_logind({debug => 2}); + # Enable fencing and set the retry to INFINITY, if needed. $anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'} = "" if not defined $anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { diff --git a/tools/anvil-manage-files b/tools/anvil-manage-files index 03ab133d..4a0257c1 100755 --- a/tools/anvil-manage-files +++ b/tools/anvil-manage-files @@ -30,6 +30,8 @@ # TODO: # - If two Strikers have the same file name, but different sizes, we get into a yo-yo of updating the two # sides. If this happens, we need to rsync the larger one over the smaller one. +# +# - Create a ".done." when the upload completes so that we know it's time to add it to the database. # # NOTE: # - remove unsyncs, add syncs. diff --git a/tools/anvil-manage-server b/tools/anvil-manage-server index f6d26a64..173e359b 100755 --- a/tools/anvil-manage-server +++ b/tools/anvil-manage-server @@ -123,6 +123,17 @@ sub process_interactive $anvil->data->{old_config}{ram}{'bytes'} = ""; # Did they specify an Anvil! system? + if (not $anvil->data->{switches}{anvil}) + { + # Is this machine in an Anvil!? + $anvil->Database->get_hosts(); + my $host_uuid = $anvil->Get->host_uuid(); + my $anvil->data->{switches}{anvil} = $anvil->data->{hosts}{host_uuid}{$host_uuid}{anvil_uuid} + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + host_uuid => $host_uuid, + "switches::anvil" => $anvil->data->{switches}{anvil}, + }}); + } if ($anvil->data->{switches}{anvil}) { $anvil->Get->anvil_from_switch({anvil => $anvil->data->{switches}{anvil}}); @@ -142,8 +153,14 @@ sub process_interactive }}); } - if (not $anvil->data->{target_server}{anvil_uuid}) + # If we don't habe a server, show the list of servers. + if (not $anvil->data->{switches}{server}) { + # If we've got an Anvil!, show the VMs on it. Otherwise, show all VMs. + if ($anvil->data->{switches}{anvil_name}) + { + + } } return(0); diff --git a/tools/anvil-provision-server b/tools/anvil-provision-server index 7015e81e..7b8608e5 100755 --- a/tools/anvil-provision-server +++ b/tools/anvil-provision-server @@ -34,7 +34,25 @@ my $anvil = Anvil::Tools->new(); # Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is # passed directly, it will be used. Otherwise, the password will be read from the database. -$anvil->Get->switches({list => ["anvil", "anvil-name", "anvil-uuid", "ci-test", "driver-disc", "cpu", "install-media", "machine", "name", "options", "os", "pre-test", "uuid", "ram", "storage-group", "storage-size", "use-image"], man => $THIS_FILE}); +$anvil->Get->switches({list => [ + "anvil", + "anvil-name", + "anvil-uuid", + "ci-test", + "driver-disc", + "cpu", + "install-media", + "machine", + "name", + "network", + "options", + "os", + "pre-test", + "uuid", + "ram", + "storage-group", + "storage-size", + "use-image"], man => $THIS_FILE}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }}); @@ -122,8 +140,8 @@ sub run_jobs if (not $waiting_reported) { $anvil->Job->update_progress({ - progress => 5, - message => "job_0275", + progress => 5, + message => "job_0275", }); $waiting_reported = 1; } @@ -138,8 +156,8 @@ sub run_jobs # We're ready! $waiting = 0; $anvil->Job->update_progress({ - progress => 8, - message => "job_0276", + progress => 8, + message => "job_0276", }); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0276"}); } @@ -147,8 +165,8 @@ sub run_jobs { # Cluster is coming up, but it's not up yet. $anvil->Job->update_progress({ - progress => 6, - message => "job_0278", + progress => 6, + message => "job_0278", }); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0278"}); } @@ -457,9 +475,7 @@ sub provision_server say_memory => $say_memory, }}); - ### TODO: Support user-selected IFN. For now, we hard-code it to 'ifn_bridge1' Also allow a MAC to be - ### set with '--network ifn1_bridge1[:mac],ifn2_bridge1[:mac]...'. - ### Support disk images (ie: sysprep) via '--import'. The device used for booting is the first + ### TODO: Support disk images (ie: sysprep) via '--import'. The device used for booting is the first ### device specified via "--disk" ### Consider support for TPM, RNG and watchdog devices my $server_uuid = $anvil->data->{job}{server_uuid} ? $anvil->data->{job}{server_uuid} : $anvil->Get->uuid(); @@ -467,17 +483,36 @@ sub provision_server "job::os" => $anvil->data->{job}{os}, server_uuid => $server_uuid, }}); - my $disk_bus = ",target.bus=virtio"; - my $nic_model = ",model.type=virtio"; + my $disk_bus = ",target.bus=virtio"; if ($anvil->data->{job}{os} eq "win7") { - $disk_bus = ""; - $nic_model = ",model.type=e1000e"; + $disk_bus = ""; } - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - disk_bus => $disk_bus, - nic_model => $nic_model, - }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { disk_bus => $disk_bus }}); + + # Setup the network line + my $nic = "bridge="; + if ($anvil->data->{job}{network}{bridge}) + { + $nic .= $anvil->data->{job}{network}{bridge}; + } + else + { + $nic .= "ifn1_bridge1"; + } + if ($anvil->data->{job}{network}{model}) + { + $nic .= ",model.type=".$anvil->data->{job}{network}{model}; + } + elsif ($anvil->data->{job}{os} eq "win7") + { + $nic .= ",model.type=e1000e"; + } + if ($anvil->data->{job}{network}{mac}) + { + $nic .= ",mac.address=".$anvil->data->{job}{network}{mac}; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nic => $nic }}); my $shell_call = $anvil->data->{path}{exe}{'virt-install'}." --connect qemu:///system \\\n"; $shell_call .= "--name ".$server." \\\n"; @@ -486,7 +521,7 @@ sub provision_server $shell_call .= " --events on_poweroff=destroy,on_reboot=restart \\\n"; $shell_call .= " --vcpus ".$anvil->data->{job}{cpu_cores}.",sockets=1,cores=".$anvil->data->{job}{cpu_cores}." \\\n"; $shell_call .= " --cpu host \\\n"; - $shell_call .= " --network bridge=ifn1_bridge1".$nic_model." \\\n"; + $shell_call .= " --network ".$nic." \\\n"; $shell_call .= " --graphics vnc \\\n"; $shell_call .= " --sound ich9 \\\n"; $shell_call .= " --clock offset=".$clock_offset.",rtc_tickpolicy=catchup \\\n"; @@ -670,7 +705,17 @@ sub provision_server output => $output, return_code => $return_code, }}); - + + # Call an adjust in case we forced the resource to disable fencing. + $shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$anvil->data->{job}{server_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); + + ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); + return(0); } @@ -723,8 +768,8 @@ sub startup_resource task => "up", }); - # If both sides are Inconsistent, for node 1 to primary - + # If both sides are Inconsistent, for node 1 to primary. If we're primary and the peer is + # Unknown, force to primary. my $waiting = 1; while($waiting) { @@ -889,8 +934,93 @@ sub startup_resource $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_ready => $all_ready }}); if ($all_ready) { + # If we're here, we're mostly ready, however we're in a bit of a spot if our + # peer hasn't connected. We're going to wait up to one minute for the peer to + # connect. If it doesn't, we're going to force back to primary. + $anvil->Job->update_progress({ + progress => 59, + message => "job_0434", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0434"}); + + my $wait_until = time + 60; + my $resource = $anvil->data->{job}{server_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:time' => time, + 's2:wait_until' => $wait_until, + 's3:resource' => $resource, + }}); + while ($waiting) + { + $anvil->DRBD->gather_data({debug => 3}); + my $any_unknown = 0; + foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{volume}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }}); + foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { peer => $peer }}); + if (not defined $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{local_disk_state}) + { + $any_unknown = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }}); + next; + } + + my $local_disk_state = $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{local_disk_state}; + my $peer_disk_state = $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{peer_disk_state}; + my $local_role = $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{local_role}; + my $peer_role = $anvil->data->{new}{resource}{$resource}{volume}{$volume}{peer}{$peer}{peer_role}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:local_disk_state' => $local_disk_state, + 's2:peer_disk_state' => $peer_disk_state, + 's3:local_role' => $local_role, + 's4:peer_role' => $peer_role, + }}); + + if (($peer_disk_state =~ /unknown/i) or ($peer_role =~ /unknown/i)) + { + $any_unknown = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }}); + } + last if $any_unknown; + } + last if $any_unknown; + } + + if ($any_unknown) + { + if (time > $wait_until) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0436"}); + + my $shell_call = $anvil->data->{path}{exe}{drbdsetup}." net-options ".$anvil->data->{job}{server_name}." ".$anvil->data->{job}{drbd_peer_node_id}." --set-defaults --_name=".$anvil->data->{job}{peer_short_name}." --protocol=C --fencing=dont-care"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); + + my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); + + $waiting = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); + } + else + { + my $waiting = $wait_until - time; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0435", variables => { waiting => $waiting }}); + sleep 5; + } + } + else + { + $waiting = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); + } + } + $waiting = 0; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); } if ($waiting) @@ -918,13 +1048,13 @@ sub create_md # Create the DRBD metadata my $shell_call = $anvil->data->{path}{exe}{drbdadm}." --force create-md --max-peers=3 ".$anvil->data->{job}{server_name}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); - + my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code, }}); - + ### Return codes # 0 == Success # 1 == ? @@ -934,17 +1064,17 @@ sub create_md # Metadata creation failed. $anvil->Job->update_progress({ progress => 100, - message => "error_0204,!!return_code!".$return_code."!!,!!output!".$output."!!", - job_status => "failed", + message => "error_0204,!!return_code!".$return_code."!!,!!output!".$output."!!", + job_status => "failed", }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0204", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0204", variables => { return_code => $return_code, - output => $output, + output => $output, }}); $anvil->nice_exit({exit_code => 1}); } $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0579", variables => { resource => $anvil->data->{job}{server_name} }}); - + $anvil->Job->update_progress({ progress => 50, message => "job_0191,!!resource!".$anvil->data->{job}{server_name}."!!", @@ -1339,6 +1469,27 @@ sub parse_job_data $anvil->data->{job}{os} =~ s/\s+?//; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::os' => $anvil->data->{job}{os} }}); } + if ($line =~ /network=(.*)$/) + { + $anvil->data->{job}{network}{raw} = $1; + $anvil->data->{job}{network}{raw} =~ s/^\s+//; + $anvil->data->{job}{network}{raw} =~ s/\s+?//; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::network::raw' => $anvil->data->{job}{network}{raw} }}); + + # Now break up the data, if needed. + my $problem = process_network($anvil, $anvil->data->{job}{network}{raw}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); + if ($problem) + { + $anvil->Job->update_progress({ + progress => 100, + message => "error_0398,!!network!".$anvil->data->{job}{network}{raw}."!!", + job_status => "failed", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0398", variables => { job_uuid => $anvil->data->{job}{network}{raw} }}); + $anvil->nice_exit({exit_code => 1}); + } + } } # We need a server name and storage group UUID regardless of which mode we're in. @@ -1414,7 +1565,7 @@ sub parse_job_data $anvil->nice_exit({exit_code => 1}); } } - + if (not $anvil->data->{job}{server_name}) { # No server name given @@ -1515,6 +1666,7 @@ sub parse_job_data }}); $anvil->nice_exit({exit_code => 1}); } + # Driver disc is optional. $anvil->data->{new_server}{driver_iso_path} = ""; if (($anvil->data->{job}{driver_iso_uuid}) && ($anvil->data->{job}{driver_iso_uuid} ne "none")) @@ -1561,6 +1713,90 @@ sub parse_job_data return(0); } +sub process_network +{ + my ($anvil, $string) = @_; + + my $problem = 0; + my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid}; + my $anvil_name = $anvil->data->{new_server}{anvil_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + anvil_uuid => $anvil_uuid, + anvil_name => $anvil_name, + }}); + + $anvil->data->{job}{network}{bridge} = ""; + $anvil->data->{job}{network}{mac} = ""; + $anvil->data->{job}{network}{model} = ""; + if (($string =~ /bridge=(.*?),/) or ($string =~ /bridge=(.*?)$/)) + { + my $bridge_name = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bridge_name => $bridge_name }}); + + # Make sure the bridge is valid. + if ((exists $anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}) && ($anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}{on_nodes})) + { + $anvil->data->{job}{network}{bridge} = $bridge_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::network::bridge' => $anvil->data->{job}{network}{bridge} }}); + } + else + { + print "The requested bridge: [".$bridge_name."] is not valid.\n"; + print "Valid bridges on the Anvil! node: [".$anvil_name."] are:'\n"; + foreach my $bridge_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{bridges}}) + { + next if $anvil->data->{anvil_resources}{$anvil_uuid}{bridges}{$bridge_name}{on_nodes}; + print "- ".$bridge_name."\n"; + } + $problem = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); + } + } + if (($string =~ /mac=(.*?),/) or ($string =~ /mac=(.*?)$/)) + { + my $mac_address = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mac_address => $mac_address }}); + + # Validate it + if (not $anvil->Validate->mac({mac => $mac_address})) + { + print "The requested MAC address: [".$mac_address."] does not appear to be valid.\n"; + $problem = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); + } + else + { + $anvil->data->{job}{network}{mac} = $mac_address; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::network::mac' => $anvil->data->{job}{network}{mac} }}); + } + } + if (($string =~ /model=(.*?),/) or ($string =~ /model=(.*?)$/)) + { + my $model = $1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { model => $model }}); + + $anvil->Get->virsh_list_net({debug => 2}); + if (exists $anvil->data->{osinfo}{net}{$model}) + { + $anvil->data->{job}{network}{model} = $model; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::network::model' => $anvil->data->{job}{network}{model} }}); + } + else + { + print "The requested network model: [".$model."] does not appear to be valid.\n"; + print "Valid network models are:\n"; + foreach my $model (sort {$a cmp $b} keys %{$anvil->data->{osinfo}{net}}) + { + print "- ".$model." (".$anvil->data->{osinfo}{net}{$model}{vendor}." - ".$anvil->data->{osinfo}{net}{$model}{product}.")\n"; + } + $problem = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); + } + } + + return($problem); +} + sub check_anvil { my ($anvil) = @_; @@ -1920,20 +2156,10 @@ sub interactive_ask_server_cpu }})."\n"; my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - my $dr1_host_uuid = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : ""; print $anvil->Words->string({key => "job_0163", variables => { core => 1, model => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{cpu}{model} }})."\n"; print $anvil->Words->string({key => "job_0163", variables => { core => 2, model => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node2_host_uuid}{cpu}{model} }})."\n"; - if ($anvil->data->{anvil_resources}{$anvil_uuid}{has_dr}) - { - print $anvil->Words->string({key => "job_0164", variables => { - model => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{cpu}{model}, - cores => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$dr1_host_uuid}{cpu}{cores}, - threads => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$dr1_host_uuid}{cpu}{threads}, - }})."\n"; - } - print $terminal->Tgoto('cm', 0, 4)."? "; my $answer = ; chomp $answer; @@ -2015,12 +2241,6 @@ sub interactive_ask_server_ram ram_node1 => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{ram}{hardware}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node1_host_uuid}{ram}{hardware}, unit => "M"}).")", ram_node2 => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node2_host_uuid}{ram}{hardware}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$node2_host_uuid}{ram}{hardware}, unit => "M"}).")", }})."\n"; - if ($anvil->data->{anvil_resources}{$anvil_uuid}{has_dr}) - { - print $anvil->Words->string({key => "job_0168", variables => { - ram_available => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$dr1_host_uuid}{ram}{hardware}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{host_uuid}{$dr1_host_uuid}{ram}{hardware}, unit => "M"}).")", - }})."\n"; - } print $terminal->Tgoto('cm', 0, 5)."? "; my $answer = ; @@ -2089,15 +2309,11 @@ sub interactive_ask_server_storage_group my $storage_group_uuid = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid}; my $vg_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size}; my $vg_free = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}; - my $dr_size = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr} : 0; - my $dr_free = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr} : 0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:storage_group_name' => $storage_group_name, 's2:storage_group_uuid' => $storage_group_uuid, 's3:vg_size' => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")", 's4:vg_free' => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")", - 's5:dr_size' => $dr_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_size}).")", - 's6:dr_free' => $dr_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_free}).")", }}); if ($anvil->data->{switches}{'storage-group'}) @@ -2113,8 +2329,6 @@ sub interactive_ask_server_storage_group $show_list .= $anvil->Words->string({key => "job_0169", variables => { vg_free => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}), vg_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}), - dr_free => $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->Convert->bytes_to_human_readable({'bytes' => $dr_free}) : "--", - dr_size => $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->Convert->bytes_to_human_readable({'bytes' => $dr_size}) : "--", }})."\n"; } @@ -2138,7 +2352,6 @@ sub interactive_ask_server_storage_group } my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - my $dr1_host_uuid = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : ""; print $anvil->Words->string({key => "job_0171"})."\n"; print $show_list."\n"; @@ -2182,16 +2395,12 @@ sub interactive_ask_server_storage_size my $storage_group_name = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name}; my $vg_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size}; my $vg_free = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}; - my $dr_size = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr} : 0; - my $dr_free = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr} : 0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:anvil_uuid' => $anvil_uuid, 's2:storage_group_name' => $storage_group_name, 's3:storage_group_uuid' => $storage_group_uuid, 's4:vg_size' => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")", 's5:vg_free' => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")", - 's6:dr_size' => $dr_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_size}).")", - 's7:dr_free' => $dr_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_free}).")", }}); $anvil->Database->get_anvils(); @@ -2200,13 +2409,9 @@ sub interactive_ask_server_storage_size $vg_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size}; $vg_free = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}; - $dr_size = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr} : 0; - $dr_free = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr} : 0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:vg_size' => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")", 's2:vg_free' => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")", - 's3:dr_size' => $dr_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_size}).")", - 's4:dr_free' => $dr_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $dr_free}).")", }}); # I need a list of Storage groups, @@ -2249,7 +2454,6 @@ sub interactive_ask_server_storage_size } my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - my $dr1_host_uuid = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : ""; print $anvil->Words->string({key => "job_0175", variables => { storage_group => $say_storage_group, @@ -2259,10 +2463,12 @@ sub interactive_ask_server_storage_size print $terminal->Tgoto('cm', 0, 7)."? "; my $answer = ; chomp $answer; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }}); - if ($answer eq "") + if (($answer eq "") or ($answer eq "100%")) { $answer = $default_storage_size; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }}); } if ($answer) { @@ -2273,6 +2479,28 @@ sub interactive_ask_server_storage_size }); # Make sure they've asked for at least 10 MiB $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer_bytes => $answer_bytes }}); + + # If the answer is within 1GiB set it to the available space. + if ($answer_bytes =~ /^\d+$/) + { + my $difference = $answer_bytes - $vg_free; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }}); + + $difference =~ s/^-//; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + difference => $difference." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $difference}).")", + }}); + + if ($difference < (2**30)) + { + # Close enough. + $answer_bytes = $vg_free; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + answer_bytes => $answer_bytes." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $answer_bytes}).")", + }}); + } + } + if (($answer_bytes eq "!!error!!") or (not $answer_bytes) or ($answer_bytes < (10*(2**20))) or @@ -2366,7 +2594,6 @@ sub interactive_ask_server_install_media } my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - my $dr1_host_uuid = $anvil->data->{anvil_resources}{$anvil_uuid}{has_dr} ? $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid} : ""; print $anvil->Words->string({key => "job_0178"})."\n"; print $iso_list."\n"; @@ -2502,23 +2729,33 @@ sub interactive_ask_server_os my $language = $anvil->Words->language; my $os_list = ""; my $default_os = $anvil->data->{switches}{os} ? $anvil->data->{switches}{os} : ""; + my $longest_os = 0; foreach my $os_code (split/,/, $anvil->data->{sys}{servers}{os_short_list}) { - $os_code =~ s/ //g; - my $os_key = "os_list_".$os_code; - my $os_name = $anvil->Words->string({key => $os_key}); - if ($os_name =~ /#!not_found/) + $os_code =~ s/ //g; + + next if not exists $anvil->data->{osinfo}{'os-list'}{$os_code}; + if (length($os_code) > $longest_os) { - # Skip it. + $longest_os = length($os_code); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { longest_os => $longest_os }}); } + } + + $anvil->Get->virsh_list_os({debug => 2}); + foreach my $os_code (split/,/, $anvil->data->{sys}{servers}{os_short_list}) + { + $os_code =~ s/ //g; + + next if not exists $anvil->data->{osinfo}{'os-list'}{$os_code}; + my $os_name = $anvil->data->{osinfo}{'os-list'}{$os_code}{name}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:os_code' => $os_code, - 's2:os_key' => $os_key, 's2:os_name' => $os_name, }}); # Still here? - $os_list .= " - ".sprintf("%-10s", $os_code)." - ".$os_name."\n"; + $os_list .= " - ".sprintf("%-${longest_os}s", $os_code)." - ".$os_name."\n"; } my $retry = 0; @@ -2567,11 +2804,14 @@ sub interactive_ask_server_os if ($answer) { # Is this valid? - my $os_key = "os_list_".$answer; - $os_name = $anvil->Words->string({key => $os_key}); + if (exists $anvil->data->{osinfo}{'os-list'}{$answer}) + { + $os_name = $anvil->data->{osinfo}{'os-list'}{$answer}{name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { os_name => $os_name }}); + } } - if ((not $answer) or ($os_name =~ /#!not_found/)) + if ((not $answer) or ($os_name eq "")) { # invalid. $retry = 1; @@ -2615,7 +2855,7 @@ sub interactive_ask_server_confirm if (not $anvil->data->{new_server}{anvil_uuid}) { # Instantly fatal - print "Missing '--anvil '\n"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0437"}); $anvil->nice_exit({exit_code => 1}); } @@ -2636,12 +2876,14 @@ sub interactive_ask_server_confirm my $problem = 0; if (not $anvil->data->{switches}{name}) { - print "Missing '--name '\n" if not $anvil->data->{switches}{options}; + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0438"}); $problem = 1; } if (not $anvil->data->{switches}{os}) { - print "Missing '--os ', valid options match 'virt-install --os-variant' (run: 'osinfo-query os' and reference the 'Short ID' column).\n" if not $anvil->data->{switches}{options}; + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0439"}); $problem = 1; } else @@ -2655,18 +2897,28 @@ sub interactive_ask_server_confirm }}); if ($os_name =~ /#!not_found/) { - print "The OS: [".$anvil->data->{switches}{os}."] was not found. If you're sure the OS is valid, please run 'striker-parse-os-list --xml --new' and add the output to 'words.xml'.\n" if not $anvil->data->{switches}{options}; + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0440", variables => { + os => $anvil->data->{switches}{os}, + }}); $problem = 1; } } if (not $anvil->data->{switches}{cpu}) { - print "Missing '--cpu <1 ~ ".$anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}.">'\n" if not $anvil->data->{switches}{options}; + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0441", variables => { + threads => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}, + }}); $problem = 1; } elsif (($anvil->data->{switches}{cpu} =~ /\D/) or ($anvil->data->{switches}{cpu} > $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}) or ($anvil->data->{switches}{cpu} < 1)) { - print "The number of CPU cores: [".$anvil->data->{switches}{cpu}."] is invalid. Must be between 1 and ".$anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}.".\n" if not $anvil->data->{switches}{options}; + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0442", variables => { + cores => $anvil->data->{switches}{cpu}, + threads => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}, + }}); $problem = 1; } @@ -2683,7 +2935,10 @@ sub interactive_ask_server_confirm }}); if (not $anvil->data->{switches}{ram}) { - print "Missing '--ram '\n" if not $anvil->data->{switches}{options}; + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0443", variables => { + max_ram => $say_max_ram, + }}); $problem = 1; } elsif (($requested_ram eq "!!error!!") or @@ -2692,7 +2947,11 @@ sub interactive_ask_server_confirm ($requested_ram > $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available})) { # Invalid - print "The requested RAM: [".$anvil->data->{switches}{ram}."] is not valid. Must be between: [64KiB] and: [".$say_max_ram."]\n" if not $anvil->data->{switches}{options}; + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0444", variables => { + ram => $anvil->data->{switches}{ram}, + max_ram => $say_max_ram, + }}); $problem = 1; } @@ -2700,10 +2959,14 @@ sub interactive_ask_server_confirm my $storage_group_uuid = ""; if (not $anvil->data->{switches}{'storage-group'}) { - print "Missing '--storage-group . Valid options are:'\n" if not $anvil->data->{switches}{options}; + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0445"}); foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}}) { - print "- Name: [".$storage_group_name."], UUID: [".$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid}."]\n" if not $anvil->data->{switches}{options}; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0446", variables => { + name => $storage_group_name, + uuid => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid}, + }}); } $problem = 1; } @@ -2732,10 +2995,16 @@ sub interactive_ask_server_confirm else { # Invalid - print "- The requested storage group: [".$storage_group."] does not appear to be valid. Valid options are;\n" if not $anvil->data->{switches}{options}; + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0447", variables => { + storage_group => $storage_group, + }}); foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}}) { - print "- Name: [".$storage_group_name."], UUID: [".$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid}."]\n" if not $anvil->data->{switches}{options}; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0446", variables => { + name => $storage_group_name, + uuid => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid}, + }}); } $problem = 1; } @@ -2743,13 +3012,16 @@ sub interactive_ask_server_confirm my $say_max_storage_group_size = $max_storage_group_size ? $anvil->Convert->bytes_to_human_readable({"bytes" => $max_storage_group_size}) : ""; if (not $anvil->data->{switches}{'storage-size'}) { + my $print = $anvil->data->{switches}{options} ? 0 : 1; if ($max_storage_group_size) { - print "Missing '--storage-size '\n" if not $anvil->data->{switches}{options}; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0448", variables => { + storage_group_size => $say_max_storage_group_size, + }}); } else { - print "Missing '--storage-size . Max will depend on selected --storage-group.'\n" if not $anvil->data->{switches}{options}; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0449"}); } $problem = 1; } @@ -2765,7 +3037,11 @@ sub interactive_ask_server_confirm ($requested_disk > $max_storage_group_size)) { # Invalid - print "The requested disk size: [".$anvil->data->{switches}{'storage-size'}."] is not valid. Must be between: [10MiB] and: [".$say_max_storage_group_size."]\n" if not $anvil->data->{switches}{options}; + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0450", variables => { + storage_size => $anvil->data->{switches}{'storage-size'}, + max_size => $say_max_storage_group_size, + }}); $problem = 1; } } @@ -2778,7 +3054,18 @@ sub interactive_ask_server_confirm } if (not $anvil->data->{switches}{'install-media'}) { - print "Missing '--install-media '\n" if not $anvil->data->{switches}{options}; + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0451"}); + foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}}) + { + my $file_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}; + my $file_type = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type}; + next if $file_type ne "iso"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0452", variables => { + name => $file_name, + uuid => $file_uuid, + }}); + } $problem = 1; } else @@ -2790,7 +3077,8 @@ sub interactive_ask_server_confirm my $file_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}; my $file_type = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type}; my $file_directory = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory}; - if ($file_name eq $anvil->data->{switches}{'install-media'}) + if (($file_name eq $anvil->data->{switches}{'install-media'}) or + ($file_uuid eq $anvil->data->{switches}{'install-media'})) { # Found it. $found = 1; @@ -2805,7 +3093,10 @@ sub interactive_ask_server_confirm else { # Not an ISO. - print "The install file: [".$anvil->data->{switches}{'install-media'}."] is not an ISO, so it can't be used to install.\n" if not $anvil->data->{switches}{options}; + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0453", variables => { + file => $anvil->data->{switches}{'install-media'}, + }}); $problem = 1; } } @@ -2813,8 +3104,24 @@ sub interactive_ask_server_confirm } if (not $found) { - print "The install file: [".$anvil->data->{switches}{'install-media'}."] was not found on this Anvil!.\n" if not $anvil->data->{switches}{options}; - print "Is it in '/mnt/shared/files/' and are the daemons running?\n" if not $anvil->data->{switches}{options}; + if (not $anvil->data->{switches}{options}) + { + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0454", variables => { + file => $anvil->data->{switches}{'install-media'}, + }}); + foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}}) + { + my $file_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}; + my $file_type = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type}; + next if $file_type ne "iso"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0452", variables => { + name => $file_name, + uuid => $file_uuid, + }}); + } + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0455"}); + } $problem = 1; } } @@ -2827,7 +3134,8 @@ sub interactive_ask_server_confirm my $file_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}; my $file_type = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type}; my $file_directory = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory}; - if ($file_name eq $anvil->data->{switches}{'driver-disc'}) + if (($file_name eq $anvil->data->{switches}{'driver-disc'}) or + ($file_uuid eq $anvil->data->{switches}{'driver-disc'})) { # Found it. $found = 1; @@ -2842,7 +3150,10 @@ sub interactive_ask_server_confirm else { # Not an ISO. - print "The driver file: [".$anvil->data->{switches}{'driver-disc'}."] is not an ISO, so it can't be used as an optical disc.\n" if not $anvil->data->{switches}{options}; + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0456", variables => { + file => $anvil->data->{switches}{'driver-disc'}, + }}); $problem = 1; } } @@ -2850,8 +3161,24 @@ sub interactive_ask_server_confirm } if (not $found) { - print "The driver file: [".$anvil->data->{switches}{'driver-disc'}."] was not found on this Anvil!.\n" if not $anvil->data->{switches}{options}; - print "Is it in '/mnt/shared/files/' and are the daemons running?\n" if not $anvil->data->{switches}{options}; + if (not $anvil->data->{switches}{options}) + { + my $print = $anvil->data->{switches}{options} ? 0 : 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0457", variables => { + file => $anvil->data->{switches}{'driver-disc'}, + }}); + foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}}) + { + my $file_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}; + my $file_type = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type}; + next if $file_type ne "iso"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0452", variables => { + name => $file_name, + uuid => $file_uuid, + }}); + } + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 0, key => "job_0455"}); + } $problem = 1; } } @@ -2887,21 +3214,22 @@ sub interactive_ask_server_confirm else { # Show valid options to build a VM in a machine-parsable way. - print "Available options - --name - alphanumeric, 1~16 characters long. - --os - Valid options; run: 'osinfo-query os' and reference the 'Short ID' column. - --cpu - 1 ~ ".$anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}." - --ram - bytes or human readable, min is 64KiB, max is ".$say_max_ram." - --storage-group - name or uuid. Valid options are:\n"; + print $anvil->Words->string({key => "job_0458", variables => { + threads => $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads}, + max_ram => $say_max_ram, + }})."\n"; foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}}) { my $storage_group_uuid = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid}; my $max_storage_group_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}; my $say_max_storage_group_size = $anvil->Convert->bytes_to_human_readable({"bytes" => $max_storage_group_size}); - print " - Name: [".$storage_group_name."], UUID: [".$anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid}."], free space: [".$say_max_storage_group_size."]\n"; + print $anvil->Words->string({key => "job_0459", variables => { + name => $storage_group_name, + uuid => $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid}, + free_space => $say_max_storage_group_size, + }})."\n"; } - print " --storage-size - Disk size, see above for space available - --install-media - file name or file UUID. Available discs are:\n"; + print $anvil->Words->string({key => "job_0460"})."\n"; foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}}) { my $file_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}; @@ -2909,12 +3237,25 @@ sub interactive_ask_server_confirm my $file_size = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size}; next if $file_type ne "iso"; my $say_size = $anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}); - print " - File name: [".$file_name."], file UUID: [".$file_uuid."], size: [".$say_size."]\n"; + print $anvil->Words->string({key => "job_0461", variables => { + name => $file_name, + uuid => $file_uuid, + size => $say_max_storage_group_size, + }})."\n"; } - print " --driver-disc - (optional) A driver disc to be added as a second optical drive. Valid options are above.\n"; + print $anvil->Words->string({key => "job_0462"})."\n"; } $anvil->nice_exit({exit_code => 0}); } + + ### TODO: Better sanity check this + # If we were passed network info, validate it. + if ($anvil->data->{switches}{network}) + { + $problem = process_network($anvil, $anvil->data->{switches}{network}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); + } + if ($problem) { $anvil->nice_exit({exit_code => 1}); diff --git a/tools/anvil-report-usage b/tools/anvil-report-usage index e62a2896..da21c6e5 100755 --- a/tools/anvil-report-usage +++ b/tools/anvil-report-usage @@ -73,12 +73,10 @@ sub collect_anvil_data $anvil->data->{anvil_data}{$anvil_name}{description} = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description}; $anvil->data->{anvil_data}{$anvil_name}{node1_host_uuid} = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_node1_host_uuid}; $anvil->data->{anvil_data}{$anvil_name}{node2_host_uuid} = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_node2_host_uuid}; - $anvil->data->{anvil_data}{$anvil_name}{dr1_host_uuid} = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_dr1_host_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "s1:anvil_data::${anvil_name}::anvil_description" => $anvil->data->{anvil_data}{$anvil_name}{description}, "s2:anvil_data::${anvil_name}::node1_host_uuid" => $anvil->data->{anvil_data}{$anvil_name}{node1_host_uuid}, "s3:anvil_data::${anvil_name}::node2_host_uuid" => $anvil->data->{anvil_data}{$anvil_name}{node2_host_uuid}, - "s4:anvil_data::${anvil_name}::dr1_host_uuid" => $anvil->data->{anvil_data}{$anvil_name}{dr1_host_uuid}, }}); if (length($anvil_name) > $anvil->data->{longest}{anvil_name}) @@ -99,34 +97,22 @@ sub collect_anvil_data my $node1_host_uuid = $anvil->data->{anvil_data}{$anvil_name}{node1_host_uuid}; my $node2_host_uuid = $anvil->data->{anvil_data}{$anvil_name}{node2_host_uuid}; - my $dr1_host_uuid = $anvil->data->{anvil_data}{$anvil_name}{dr1_host_uuid}; if ($anvil->data->{switches}{detailed}) { $anvil->data->{anvil_data}{$anvil_name}{node1_host_name} = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name}; $anvil->data->{anvil_data}{$anvil_name}{node2_host_name} = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name}; - $anvil->data->{anvil_data}{$anvil_name}{dr1_host_name} = ""; - if ($dr1_host_uuid) - { - $anvil->data->{anvil_data}{$anvil_name}{dr1_host_name} = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name}; - } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "s1:anvil_data::${anvil_name}::node1_host_name" => $anvil->data->{anvil_data}{$anvil_name}{node1_host_name}, "s2:anvil_data::${anvil_name}::node2_host_name" => $anvil->data->{anvil_data}{$anvil_name}{node2_host_name}, - "s3:anvil_data::${anvil_name}::dr1_host_name" => $anvil->data->{anvil_data}{$anvil_name}{dr1_host_name}, }}); } else { $anvil->data->{anvil_data}{$anvil_name}{node1_host_name} = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name}; $anvil->data->{anvil_data}{$anvil_name}{node2_host_name} = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name}; - if ($dr1_host_uuid) - { - $anvil->data->{anvil_data}{$anvil_name}{dr1_host_name} = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{short_host_name}; - } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "s1:anvil_data::${anvil_name}::node1_host_name" => $anvil->data->{anvil_data}{$anvil_name}{node1_host_name}, "s2:anvil_data::${anvil_name}::node2_host_name" => $anvil->data->{anvil_data}{$anvil_name}{node2_host_name}, - "s3:anvil_data::${anvil_name}::dr1_host_name" => $anvil->data->{anvil_data}{$anvil_name}{dr1_host_name}, }}); } @@ -202,18 +188,12 @@ sub collect_anvil_data my $vg_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size}; my $free_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size}; my $sg_used = $vg_size - $free_size; - my $vg_size_on_dr = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size_on_dr}; - my $free_size_on_dr = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{available_on_dr}; - my $sg_used_on_dr = $vg_size_on_dr - $free_size_on_dr; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:storage_group_name' => $storage_group_name, 's2:storage_group_uuid' => $storage_group_uuid, 's3:vg_size' => $anvil->Convert->add_commas({number => $vg_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")", 's4:free_size' => $anvil->Convert->add_commas({number => $free_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $free_size}).")", 's5:sg_used' => $anvil->Convert->add_commas({number => $sg_used})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $sg_used}).")", - 's6:vg_size_on_dr' => $anvil->Convert->add_commas({number => $vg_size_on_dr})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size_on_dr}).")", - 's7:free_size_on_dr' => $anvil->Convert->add_commas({number => $free_size_on_dr})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $free_size_on_dr}).")", - 's8:sg_used_on_dr' => $anvil->Convert->add_commas({number => $sg_used_on_dr})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $sg_used_on_dr}).")", }}); $anvil->data->{anvil_data}{$anvil_name}{storage_group}{$storage_group_name}{say_used_size} = $anvil->Convert->bytes_to_human_readable({'bytes' => $sg_used}); @@ -266,7 +246,7 @@ sub show_anvils my $longest_ram_free = length($ram_free_header) > $anvil->data->{longest}{ram_free} ? length($ram_free_header) : $anvil->data->{longest}{ram_free}; my $bridge_header = $anvil->Words->string({key => "header_0077"}); my $longest_bridge_string = length($bridge_header) > $anvil->data->{longest}{bridge_string} ? length($bridge_header) : $anvil->data->{longest}{bridge_string}; - my $storage_group_header = $anvil->Words->string({key => "header_0078"}); + my $storage_group_header = $anvil->Words->string({key => "header_0070"}); my $longest_storage_group = length($storage_group_header) > $anvil->data->{longest}{storage_group} ? length($storage_group_header) : $anvil->data->{longest}{storage_group}; my $sg_used_header = $anvil->Words->string({key => "header_0079"}); my $longest_sg_used = length($sg_used_header) > $anvil->data->{longest}{sg_used} ? length($sg_used_header) : $anvil->data->{longest}{sg_used}; @@ -287,37 +267,37 @@ sub show_anvils # Anvil! my $break_line = "+-".sprintf("%0${longest_anvil_name}d", 0); my $header_line = "| ".sprintf("%-${longest_anvil_name}s", $anvil_header)." "; - my $blank_lead = "| ".sprintf("%-${longest_anvil_name}s", $anvil_header)." "; + my $blank_lead = "| ".sprintf("%-${longest_anvil_name}s", "")." "; if ($anvil->data->{switches}{detailed}) { # Description $break_line .= "-+-".sprintf("%0${longest_description}d", 0); $header_line .= "| ".sprintf("%-${longest_description}s", $description_header)." "; - $blank_lead .= " ".sprintf("%-${longest_description}s", $description_header)." "; + $blank_lead .= "| ".sprintf("%-${longest_description}s", "")." "; } # CPU String $break_line .= "-+-".sprintf("%0${longest_cpu_string}d", 0); $header_line .= "| ".sprintf("%-${longest_cpu_string}s", $cpu_header)." "; - $blank_lead .= " ".sprintf("%-${longest_cpu_string}s", $cpu_header)." "; + $blank_lead .= "| ".sprintf("%-${longest_cpu_string}s", "")." "; if ($anvil->data->{switches}{detailed}) { # RAM used $break_line .= "-+-".sprintf("%0${longest_ram_used}d", 0); $header_line .= "| ".sprintf("%-${longest_ram_used}s", $ram_used_header)." "; - $blank_lead .= " ".sprintf("%-${longest_ram_used}s", $ram_used_header)." "; + $blank_lead .= "| ".sprintf("%-${longest_ram_used}s", "")." "; } # RAM Free $break_line .= "-+-".sprintf("%0${longest_ram_free}d", 0); $header_line .= "| ".sprintf("%-${longest_ram_free}s", $ram_free_header)." "; - $blank_lead .= " ".sprintf("%-${longest_ram_free}s", $ram_free_header)." "; + $blank_lead .= "| ".sprintf("%-${longest_ram_free}s", "")." "; # Bridges $break_line .= "-+-".sprintf("%0${longest_bridge_string}d", 0); $header_line .= "| ".sprintf("%-${longest_bridge_string}s", $bridge_header)." "; - $blank_lead .= " ".sprintf("%-${longest_bridge_string}s", $bridge_header)." "; + $blank_lead .= "| ".sprintf("%-${longest_bridge_string}s", ""); # Storage Group $break_line .= "-+-".sprintf("%0${longest_storage_group}d", 0); @@ -379,7 +359,8 @@ sub show_anvils $storage_line .= " | ".sprintf("%-${longest_sg_used}s", $say_used_size); } $storage_line .= " | ".sprintf("%-${longest_sg_free}s", $say_free_size)." |"; - + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { storage_line => $storage_line }}); + push @{$storage_groups}, $storage_line; } @@ -741,27 +722,17 @@ sub collect_server_data # have a matching node name. my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; - my $dr1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}; # Get names. my $node1_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name}; my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name}; my $node2_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name}; my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name}; - my $dr1_host_name = ""; - my $dr1_short_host_name = ""; - if (($dr1_host_uuid) && (exists $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid})) - { - $dr1_host_name = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{host_name}; - $dr1_short_host_name = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{short_host_name}; - } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:node1_host_name' => $node1_host_name, 's2:node1_short_host_name' => $node1_short_host_name, 's3:node2_host_name' => $node2_host_name, 's4:node2_short_host_name' => $node2_short_host_name, - 's5:dr1_host_name' => $dr1_host_name, - 's6:dr1_short_host_name' => $dr1_short_host_name, }}); # Storage info. @@ -806,9 +777,7 @@ sub collect_server_data if (($drbd_node eq $node1_host_name) or ($drbd_node eq $node1_short_host_name) or ($drbd_node eq $node2_host_name) or - ($drbd_node eq $node2_short_host_name) or - (($dr1_host_name) && ($drbd_node eq $dr1_host_name)) or - (($dr1_short_host_name) && ($drbd_node eq $dr1_short_host_name))) + ($drbd_node eq $node2_short_host_name)) { $anvil->data->{server_data}{$server_name}{server_uuid}{$server_uuid}{disk}{$resource}{$volume}{node}{$drbd_node}{drbd_path} = $anvil->data->{drbd}{drbd_node}{$drbd_node}{config}{resource}{$resource}{volume}{$volume}{drbd_path}; $anvil->data->{server_data}{$server_name}{server_uuid}{$server_uuid}{disk}{$resource}{$volume}{node}{$drbd_node}{drbd_path_by_res} = $anvil->data->{drbd}{drbd_node}{$drbd_node}{config}{resource}{$resource}{volume}{$volume}{drbd_path_by_res}; @@ -835,11 +804,6 @@ sub collect_server_data $node_host_uuid = $node2_host_uuid; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node_host_uuid => $node_host_uuid }}); } - elsif (($drbd_node eq $dr1_host_name) or ($drbd_node eq $dr1_short_host_name)) - { - $node_host_uuid = $dr1_host_uuid; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node_host_uuid => $node_host_uuid }}); - } # How big is this LV? my $backing_lv = $anvil->data->{server_data}{$server_name}{server_uuid}{$server_uuid}{disk}{$resource}{$volume}{node}{$drbd_node}{backing_lv}; diff --git a/tools/anvil-version-changes b/tools/anvil-version-changes new file mode 100755 index 00000000..24525abd --- /dev/null +++ b/tools/anvil-version-changes @@ -0,0 +1,411 @@ +#!/usr/bin/perl +# +# This does checks for changes that are needed because of version changes. Over time, checks here can be +# removed. + +use strict; +use warnings; +use Anvil::Tools; +use Data::Dumper; +use Text::Diff; + +$| = 1; + +my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; +my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; +if (($running_directory =~ /^\./) && ($ENV{PWD})) +{ + $running_directory =~ s/^\./$ENV{PWD}/; +} + +my $anvil = Anvil::Tools->new(); + +# Get a list of all interfaces with IP addresses. +$anvil->Get->switches({list => [ +]}); +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); + +$anvil->Database->connect({sensitive => 1}); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132" }); +if (not $anvil->data->{sys}{database}{connections}) +{ + # No databases, exit. + $anvil->Log->entry({ source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003" }); + $anvil->nice_exit({ exit_code => 1 }); +} + +my $host_type = $anvil->Get->host_type(); +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }}); + +if ($host_type eq "striker") +{ + striker_checks($anvil); +} +elsif ($host_type eq "node") +{ + node_checks($anvil); +} +elsif ($host_type eq "dr") +{ + dr_checks($anvil); +} + + +$anvil->nice_exit({exit_code => 0}); + + +############################################################################################################# +# Functions # +############################################################################################################# + +# Check for things that need to happen on Striker dashboards. +sub striker_checks +{ + my ($anvil) = @_; + + # This converts the old/broken 'notifications' tables with the more appropriately named 'alert-override' + update_notifications($anvil); + + ### NOTE: Disabled until review complete + # This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release) + #update_audits($anvil); + + ### NOTE: Disabled until review complete + # This checks to make sure that the new dr_links table exists, and that existing anvil_dr1_host_uuid + # entries are copied. + update_dr_links($anvil); + + ### TODO: Remove these later. This is here to clean up how we used to handle db_in_use and lock_request flags. + if (1) + { + # Broadly clear all states that are '0' now. + my $queries = []; + push @{$queries}, "DELETE FROM states WHERE state_name LIKE 'db_in_use::%' AND state_note != '1';"; + push @{$queries}, "DELETE FROM history.variables WHERE variable_name = 'lock_request';"; + push @{$queries}, "DELETE FROM variables WHERE variable_name = 'lock_request';"; + foreach my $query (@{$queries}) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); + } + $anvil->Database->write({debug => 2, query => $queries, source => $THIS_FILE, line => __LINE__}); + } + + return(0); +} + +# Check for things that need to happen on Anvil! Subnodes. +sub node_checks +{ + my ($anvil) = @_; + + # RHBZ #1961562 - https://bugzilla.redhat.com/show_bug.cgi?id=1961562#c16 + handle_bz1961562($anvil); + + # Make sure DRBD compiled after a kernel upgrade. + $anvil->DRBD->_initialize_kmod({debug => 2}); + + # Make sure logind is update to handle fencing properly + # see - https://access.redhat.com/solutions/1578823 + $anvil->Cluster->configure_logind({debug => 2}); + + return(0); +} + +# Check for things that need to happen on DR hosts. +sub dr_checks +{ + my ($anvil) = @_; + + # RHBZ #1961562 - https://bugzilla.redhat.com/show_bug.cgi?id=1961562#c16 + handle_bz1961562($anvil); + + # Make sure DRBD compiled after a kernel upgrade. + $anvil->DRBD->_initialize_kmod({debug => 2}); + + return(0); +} + +# +sub update_dr_links +{ + my ($anvil) = @_; + + foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}}) + { + my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_catalog = 'anvil' AND table_name = 'dr_links';"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + + my $count = $anvil->Database->query({query => $query, uuid => $uuid, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); + + if (not $count) + { + # Add the table. + my $query = q| +CREATE TABLE dr_links ( + dr_link_uuid uuid not null primary key, + dr_link_host_uuid uuid not null, + dr_link_anvil_uuid uuid not null, + dr_link_note text, -- Set to 'DELETE' when no longer used. + modified_date timestamp with time zone not null, + + FOREIGN KEY(dr_link_host_uuid) REFERENCES hosts(host_uuid), + FOREIGN KEY(dr_link_anvil_uuid) REFERENCES anvils(anvil_uuid) +); +ALTER TABLE dr_links OWNER TO admin; + +CREATE TABLE history.dr_links ( + history_id bigserial, + dr_link_uuid uuid, + dr_link_host_uuid uuid, + dr_link_anvil_uuid uuid, + dr_link_note text, + modified_date timestamp with time zone not null +); +ALTER TABLE history.dr_links OWNER TO admin; + +CREATE FUNCTION history_dr_links() RETURNS trigger +AS $$ +DECLARE + history_dr_links RECORD; +BEGIN + SELECT INTO history_dr_links * FROM dr_links WHERE dr_link_uuid = new.dr_link_uuid; + INSERT INTO history.dr_links + (dr_link_uuid, + dr_link_host_uuid, + dr_link_anvil_uuid, + dr_link_note, + modified_date) + VALUES + (history_dr_links.dr_link_uuid, + history_dr_links.dr_link_host_uuid, + history_dr_links.dr_link_anvil_uuid, + history_dr_links.dr_link_note, + history_dr_links.modified_date); + RETURN NULL; +END; +$$ +LANGUAGE plpgsql; +ALTER FUNCTION history_dr_links() OWNER TO admin; + +CREATE TRIGGER trigger_dr_links + AFTER INSERT OR UPDATE ON dr_links + FOR EACH ROW EXECUTE PROCEDURE history_dr_links(); +|; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); + $anvil->Database->write({debug => 2, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); + } + } + + # Now make sure that existing DR entries are copied here. + $anvil->Database->get_hosts({deubg => 2}); + $anvil->Database->get_dr_links({debug => 2}); + + foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}}) + { + my $anvil_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid}; + my $anvil_dr1_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_dr1_host_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "s1:anvil_name" => $anvil_name, + "s2:anvil_uuid" => $anvil_uuid, + "s3:anvil_dr1_host_uuid" => $anvil_dr1_host_uuid, + }}); + if ($anvil_dr1_host_uuid) + { + my $dr1_host_name = $anvil->data->{hosts}{host_uuid}{$anvil_dr1_host_uuid}{short_host_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr1_host_name => $dr1_host_name }}); + + if ((not exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}) or + (not exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$anvil_dr1_host_uuid}) or + (not exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$anvil_dr1_host_uuid}{dr_link_uuid})) + { + # Add it. + my $dr_link_uuid = $anvil->Database->insert_or_update_dr_links({ + debug => 2, + dr_link_anvil_uuid => $anvil_uuid, + dr_link_host_uuid => $anvil_dr1_host_uuid, + dr_link_note => "auto_generated", + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr1_host_name => $dr1_host_name }}); + } + } + } + + return(0); +} + +# This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release) +sub update_audits +{ + my ($anvil) = @_; + + foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}}) + { + my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_catalog = 'anvil' AND table_name = 'audits';"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + + my $count = $anvil->Database->query({query => $query, uuid => $uuid, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); + + if (not $count) + { + # Add the table. + my $query = q| +CREATE TABLE audits ( + audit_uuid uuid primary key, + audit_user_uuid uuid not null, -- This is the users -> user_uuid the audit is tracking + audit_details text not null, -- This is the information explaining the action being audited. + modified_date timestamp with time zone not null, + + FOREIGN KEY(audit_user_uuid) REFERENCES users(user_uuid) +); +ALTER TABLE audits OWNER TO admin; + +CREATE TABLE history.audits ( + history_id bigserial, + audit_uuid uuid, + audit_user_uuid uuid, + audit_details text, + modified_date timestamp with time zone not null +); +ALTER TABLE history.audits OWNER TO admin; + +CREATE FUNCTION history_audits() RETURNS trigger +AS $$ +DECLARE + history_audits RECORD; +BEGIN + SELECT INTO history_audits * FROM audits WHERE audit_uuid = new.audit_uuid; + INSERT INTO history.audits + (audit_uuid, + audit_user_uuid, + audit_details, + modified_date) + VALUES + (history_audit.audit_uuid, + history_audit.audit_user_uuid, + history_audit.audit_details, + history_audit.modified_date); + RETURN NULL; +END; +$$ +LANGUAGE plpgsql; +ALTER FUNCTION history_audits() OWNER TO admin; + +CREATE TRIGGER trigger_audits + AFTER INSERT OR UPDATE ON audits + FOR EACH ROW EXECUTE PROCEDURE history_audits(); +|; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); + $anvil->Database->write({debug => 2, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); + } + } + + return(0); +} + +# This converts the old/broken 'notifications' tables with the more appropriately named 'alert-override' +sub update_notifications +{ + my ($anvil) = @_; + + foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}}) + { + my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_catalog = 'anvil' AND table_name = 'notifications';"; + $anvil->Log->variables({source => $THIS_FILE, uuid => $uuid, line => __LINE__, level => 2, list => { query => $query }}); + + my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); + + if ($count) + { + my $queries = []; + push @{$queries}, "DROP FUNCTION history_notifications() CASCADE;"; + push @{$queries}, "DROP TABLE history.notifications;"; + push @{$queries}, "DROP TABLE public.notifications;"; + push @{$queries}, q|CREATE TABLE alert_overrides ( + alert_override_uuid uuid not null primary key, + alert_override_recipient_uuid uuid not null, -- The recipient we're linking. + alert_override_host_uuid uuid not null, -- This host_uuid of the referenced machine + alert_override_alert_level integer not null, -- This is the alert level (at or above) that this user wants alerts from. If set to '-1', the record is deleted. + modified_date timestamp with time zone not null, + + FOREIGN KEY(alert_override_host_uuid) REFERENCES hosts(host_uuid), + FOREIGN KEY(alert_override_recipient_uuid) REFERENCES recipients(recipient_uuid) +); +ALTER TABLE alert_overrides OWNER TO admin; + +CREATE TABLE history.alert_overrides ( + history_id bigserial, + alert_override_uuid uuid, + alert_override_recipient_uuid uuid, + alert_override_host_uuid uuid, + alert_override_alert_level integer, + modified_date timestamp with time zone not null +); +ALTER TABLE history.alert_overrides OWNER TO admin; + +CREATE FUNCTION history_alert_overrides() RETURNS trigger +AS $$ +DECLARE + history_alert_overrides RECORD; +BEGIN + SELECT INTO history_alert_overrides * FROM alert_overrides WHERE alert_override_uuid = new.alert_override_uuid; + INSERT INTO history.alert_overrides + (alert_override_uuid, + alert_override_recipient_uuid, + alert_override_host_uuid, + alert_override_alert_level, + modified_date) + VALUES + (history_alert_overrides.alert_override_uuid, + history_alert_overrides.alert_override_recipient_uuid, + history_alert_overrides.alert_override_host_uuid, + history_alert_overrides.alert_override_alert_level, + history_alert_overrides.modified_date); + RETURN NULL; +END; +$$ +LANGUAGE plpgsql; +ALTER FUNCTION history_alert_overrides() OWNER TO admin; + +CREATE TRIGGER trigger_alert_overrides + AFTER INSERT OR UPDATE ON alert_overrides + FOR EACH ROW EXECUTE PROCEDURE history_alert_overrides(); +|; + foreach my $query (@{$queries}) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); + } + $anvil->Database->write({debug => 2, uuid => $uuid, query => $queries, source => $THIS_FILE, line => __LINE__}); + } + } + + return(0); +} + +sub handle_bz1961562 +{ + my ($anvil) = @_; + + ### TODO: Test that this is fixed. The bug is now ERRATA + # RHBZ #1961562 - https://bugzilla.redhat.com/show_bug.cgi?id=1961562#c16 + # We're a node or DR host. We need to touch this file. + my $work_around_file = "/etc/qemu/firmware/50-edk2-ovmf-cc.json"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { work_around_file => $work_around_file }}); + if (not -e $work_around_file) + { + $anvil->Storage->write_file({ + debug => 2, + file => $work_around_file, + body => "", + overwrite => 0, + backup => 0, + mode => "0644", + user => "root", + group => "root", + }); + } + + return(0); +} diff --git a/tools/fence_pacemaker b/tools/fence_pacemaker index c8bc9660..576d90b6 100755 --- a/tools/fence_pacemaker +++ b/tools/fence_pacemaker @@ -75,17 +75,20 @@ my $conf = { path => { exe => { cibadmin => "/usr/sbin/cibadmin", + crm_attribute => "/usr/sbin/crm_attribute", crm_error => "/usr/sbin/crm_error", drbdadm => "/usr/sbin/drbdadm", echo => "/usr/bin/echo", getent => "/usr/bin/getent", logger => "/usr/bin/logger", + pcs => "/usr/sbin/pcs", stonith_admin => "/usr/sbin/stonith_admin", }, }, # The script will set this. cluster => { target_node => "", + fence_method => "stonith", # Will change to 'constraint' if we're using a DRBD resource is passed. }, # These are the environment variables set by DRBD. See 'man drbd.conf' # -> 'handlers'. @@ -112,7 +115,7 @@ my $conf = { find_executables($conf); # Something for the logs -to_log($conf, {message => "DRBD pacemaker's stonith handler invoked.", 'line' => __LINE__}); +to_log($conf, {message => "The Anvil! DRBD fence handler invoked.", 'line' => __LINE__}); # These are the full host names of the nodes given their IDs. foreach my $i (0..31) @@ -122,21 +125,22 @@ foreach my $i (0..31) { $conf->{environment}{$key} = $ENV{$key}; my $level = $conf->{environment}{$key} eq "" ? 3 : 2; - to_log($conf, {message => "DRBD Environment variable: [$key] -> [".$conf->{environment}{$key}."]", 'line' => __LINE__, level => $level}); + to_log($conf, {message => "DRBD Environment variable: [".$key."] -> [".$conf->{environment}{$key}."]", 'line' => __LINE__, level => $level}); } } # Record the environment variables foreach my $key (sort {$a cmp $b} keys %{$conf->{environment}}) { + # $conf->{environment}{DRBD_RESOURCE} -> [srv51-Workstation3] my $level = $conf->{environment}{$key} eq "" ? 3 : 2; - to_log($conf, {message => "DRBD Environment variable: [$key] -> [".$conf->{environment}{$key}."]", 'line' => __LINE__, level => $level}); + to_log($conf, {message => "DRBD Environment variable: [".$key."] -> [".$conf->{environment}{$key}."]", 'line' => __LINE__, level => $level}); } foreach my $key (sort {$a cmp $b} keys %ENV) { next if exists $conf->{environment}{$key}; my $level = $ENV{$key} eq "" ? 3 : 2; - to_log($conf, {message => "System Environment variable: [$key] -> [".$ENV{$key}."]", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "System Environment variable: [".$key."] -> [".$ENV{$key}."]", 'line' => __LINE__, level => 3}); } # Make sure we at least have the target's IP. @@ -152,10 +156,21 @@ identify_peer($conf); # If we're still alive, we now need to check the DRBD resource disk state locally. get_drbd_status($conf); -to_log($conf, {message => "Ready to fence: [".$conf->{cluster}{target_node}."]", 'line' => __LINE__, level => 1}); +# Is there a specific resource? +if ($conf->{environment}{DRBD_RESOURCE}) +{ + # Prevent the resource from running on the peer. + to_log($conf, {message => "We're being asked to fence the specific resource: [".$conf->{environment}{DRBD_RESOURCE}."]. Switching to location constraint fencing to prevent the resource from running on: [".$conf->{cluster}{target_node}."].", 'line' => __LINE__}); + $conf->{cluster}{fence_method} = "constraint"; + to_log($conf, {message => "cluster::fence_method: [".$conf->{cluster}{fence_method}."].", 'line' => __LINE__, level => 2}); +} +else +{ + to_log($conf, {message => "We were not given a specific resource to fence. Requesting pacemaker to stonith. Node: [".$conf->{cluster}{target_node}."] should power power off..", 'line' => __LINE__}); +} -# Do the deed -kill_target($conf); +# No, do the deed +perform_fence($conf); # If we hit here, something very wrong happened. exit(1); @@ -165,6 +180,159 @@ exit(1); # Functions # ############################################################################################################# +# This creates a location constraint that prevents the resource / server from running on the peer node. +sub create_constraint +{ + my ($conf) = @_; + + my $target_server = $conf->{environment}{DRBD_RESOURCE}; + my $target_node = $conf->{cluster}{target_node}; + to_log($conf, {message => "Will now create a location constraint against: [".$target_server."] preventing it from running on: [".$target_node."].", 'line' => __LINE__, level => 1}); + + # Make sure there's a rule to apply the node attribute against. + my $rule_found = 0; + my $rule_name = "drbd-fenced_".$target_server; + my $shell_call = $conf->{path}{exe}{pcs}." constraint location config show ".$target_server; + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 2}); + open (my $file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; + while(<$file_handle>) + { + # This should not generate output. + chomp; + my $line = $_; + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 2}); + + if ($line =~ /Expression: $rule_name /) + { + $rule_found = 1; + to_log($conf, {message => "rule_found: [".$rule_found."]", 'line' => __LINE__, level => 2}); + last; + } + } + close $file_handle; + + if (not $rule_found) + { + # We can't fence. + to_log($conf, {message => "The fence rule: [".$rule_name."] was now found in the cluster, unable to fence by constraint.", 'line' => __LINE__, level => 0, priority => "err"}); + exit(1); + } + + # Set the node attribute + my $rule_set = 0; + $shell_call = $conf->{path}{exe}{crm_attribute}." --type nodes --node ".$target_node." --name ".$rule_name." --update 1"; + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 2}); + open ($file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; + while(<$file_handle>) + { + # This should not generate output. + chomp; + my $line = $_; + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 2}); + } + close $file_handle; + + # Check that the rule was set. + $rule_set = 0; + my $rule_output = ""; + $shell_call = $conf->{path}{exe}{crm_attribute}." --type nodes --node ".$target_node." --name ".$rule_name." --query"; + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 2}); + open ($file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; + while(<$file_handle>) + { + # This should not generate output. + chomp; + my $line = $_; + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 2}); + + if (($line =~ /name=$rule_name/) && ($line =~ /value=1/)) + { + $rule_set = 1; + to_log($conf, {message => "rule_set: [".$rule_set."]", 'line' => __LINE__, level => 2}); + last; + } + else + { + $rule_output .= $line."\n"; + } + } + close $file_handle; + + if (not $rule_set) + { + # We can't fence. + $rule_output =~ s/\n$//gs; + to_log($conf, {message => "The node attribute triggering the fence rule: [".$rule_name."] against the node: [".$target_node."] appears to have not been set. Expected a string with 'name=".$rule_name." value=1' but got: [".$rule_output."].", 'line' => __LINE__, level => 0, priority => "err"}); + exit(1); + } + + # Place the constraint. and wait up to a minute for the target's DRBD resource to not be primary (in + # case the resource is running on the peer and needs to migrate here, which should not happen but + # best to check and be safe). + my $stop_waiting = time + 60; + my $peer_rolee = ""; + my $waiting = 1; + while ($waiting) + { + # Check the peer's disk state + my $shell_call = $conf->{path}{exe}{drbdadm}." status ".$target_server; + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 2}); + open (my $file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; + while(<$file_handle>) + { + # This should not generate output. + chomp; + my $line = $_; + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 3}); + + + if ($line =~ /$target_node role:(.*)$/) + { + $peer_rolee = $1; + to_log($conf, {message => "peer_rolee: [".$peer_rolee."]", 'line' => __LINE__, level => 2}); + last; + } + } + close $file_handle; + + if (lc($peer_rolee) ne "primary") + { + # We're good, fence is complete. + to_log($conf, {message => "Resource: [".$target_server."] has been fenced via location constraint successfully!", 'line' => __LINE__, level => 1}); + + exit(7); + } + + if (time > $stop_waiting) + { + # Done waiting, failed. + to_log($conf, {message => "The resource: [".$target_server."] on the fence target: [".$target_node."] is still in the 'Primary' role after a minute. Is the server running on the target? Is something else holding it open? Giving up on the location constraint fence attempt.", 'line' => __LINE__, level => 0, priority => "err"}); + exit(1); + } + + # Check again in a couple seconds. + sleep 2; + } + + return(0); +} + +sub perform_fence +{ + my ($conf) = @_; + + if ($conf->{cluster}{fence_method} eq "constraint") + { + create_constraint($conf); + } + else + { + kill_target($conf); + } + + return(0); +} + # This reads the status of all resources. If we're not all UpToDate, check if the peer is. If the peer is, # abort. If not, proceed (someone is gouig to have a bad day, but maybe some servers will live) sub get_drbd_status @@ -176,32 +344,32 @@ sub get_drbd_status my $local_all_uptodate = 1; my $peer_all_uptodate = 1; my $shell_call = $conf->{path}{exe}{drbdadm}." status all"; - to_log($conf, {message => "Calling: [$shell_call]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 2}); open (my $file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; while(<$file_handle>) { # This should not generate output. chomp; my $line = $_; - to_log($conf, {message => "Output: [$line]", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 3}); if (not $line) { $resource = ""; $peer = ""; - to_log($conf, {message => "resource: [$resource], peer: [$peer]", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "resource: [".$resource."], peer: [".$peer."]", 'line' => __LINE__, level => 3}); next; } if ($line =~ /^(\S+)\s+role/) { $resource = $1; - to_log($conf, {message => "resource: [$resource]", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "resource: [".$resource."]", 'line' => __LINE__, level => 3}); next; } if ($line =~ /^\s+(.*?) role:/) { $peer = $1; - to_log($conf, {message => "peer: [$peer]", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "peer: [".$peer."]", 'line' => __LINE__, level => 3}); next; } if ($resource) @@ -210,11 +378,11 @@ sub get_drbd_status { my $local_dstate = $1; $local_dstate =~ s/\s.*$//; - to_log($conf, {message => "local_dstate: [$local_dstate]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "local_dstate: [".$local_dstate."]", 'line' => __LINE__, level => 2}); if (lc($local_dstate) ne "uptodate") { $local_all_uptodate = 0; - to_log($conf, {message => "local_all_uptodate: [$local_all_uptodate]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "local_all_uptodate: [".$local_all_uptodate."]", 'line' => __LINE__, level => 2}); } next; } @@ -222,11 +390,11 @@ sub get_drbd_status { my $peer_dstate = $1; $peer_dstate =~ s/\s.*$//; - to_log($conf, {message => "peer: [$peer], peer_dstate: [$peer_dstate]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "peer: [".$peer."], peer_dstate: [".$peer_dstate."]", 'line' => __LINE__, level => 2}); if (lc($peer_dstate) ne "uptodate") { $peer_all_uptodate = 0; - to_log($conf, {message => "peer_all_uptodate: [$peer_all_uptodate]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "peer_all_uptodate: [".$peer_all_uptodate."]", 'line' => __LINE__, level => 2}); } next; } @@ -234,11 +402,9 @@ sub get_drbd_status } close $file_handle; - my $return_code = $?; - to_log($conf, {message => "Return code: [$return_code]", 'line' => __LINE__, level => 2}); # If we're not all UpToDate, but the peer is, abort - to_log($conf, {message => "local_all_uptodate: [$local_all_uptodate], peer_all_uptodate: [$peer_all_uptodate]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "local_all_uptodate: [".$local_all_uptodate."], peer_all_uptodate: [".$peer_all_uptodate."]", 'line' => __LINE__, level => 2}); if ((not $local_all_uptodate) && ($peer_all_uptodate)) { # We're not good @@ -260,30 +426,28 @@ sub identify_peer # First, can we translate the IP to a host name? my $shell_call = $conf->{path}{exe}{getent}." hosts ".$target_ip; - to_log($conf, {message => "Calling: [$shell_call]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 2}); open (my $file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; while(<$file_handle>) { # This should not generate output. chomp; my $line = $_; - to_log($conf, {message => "Output: [$line]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 2}); if ($line =~ /^$target_ip\s+(.*)$/) { # This could be multiple names. $target_host = $1; - to_log($conf, {message => "target_host: [$target_host]", 'line' => __LINE__, level => 2}); - #to_log($conf, {message => ">> target_host: [$target_host]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "target_host: [".$target_host."]", 'line' => __LINE__, level => 2}); + #to_log($conf, {message => ">> target_host: [".$target_host."]", 'line' => __LINE__, level => 2}); # Strip off any suffix, we only want the short name. #$target_host =~ s/\..*//; - #to_log($conf, {message => "<< target_host: [$target_host]", 'line' => __LINE__, level => 2}); + #to_log($conf, {message => "<< target_host: [".$target_host."]", 'line' => __LINE__, level => 2}); #last; } } close $file_handle; - my $return_code = $?; - to_log($conf, {message => "Return code: [$return_code]", 'line' => __LINE__, level => 2}); # If I got the host name, try to match it to a pacemaker node name. if ($target_host) @@ -296,18 +460,18 @@ sub identify_peer my $host_name = $ENV{HOSTNAME}; my $short_host_name = $ENV{HOSTNAME}; $short_host_name =~ s/\..*$//; - to_log($conf, {message => "host_name: [$host_name], short_host_name: [".$short_host_name."]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "host_name: [".$host_name."], short_host_name: [".$short_host_name."]", 'line' => __LINE__, level => 2}); foreach my $hash_ref (sort {$a cmp $b} @{$body->{configuration}{nodes}{node}}) { my $node = $hash_ref->{uname}; my $id = $hash_ref->{id}; - to_log($conf, {message => "node: [$node], id: [$id]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "node: [".$node."], id: [".$id."]", 'line' => __LINE__, level => 2}); foreach my $target_name (split/ /, $target_host) { - to_log($conf, {message => ">> target_name: [$target_name]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => ">> target_name: [".$target_name."]", 'line' => __LINE__, level => 2}); $target_name =~ s/\..*//; - to_log($conf, {message => "<< target_name: [$target_name]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "<< target_name: [".$target_name."]", 'line' => __LINE__, level => 2}); if ($node =~ /^$target_name/) { $conf->{cluster}{target_node} = $node; @@ -321,7 +485,7 @@ sub identify_peer # We've got some data... my $name = defined $hash_ref->{instance_attributes}{nvpair}{name} ? $hash_ref->{instance_attributes}{nvpair}{name} : ""; my $value = defined $hash_ref->{instance_attributes}{nvpair}{value} ? $hash_ref->{instance_attributes}{nvpair}{value} : ""; - to_log($conf, {message => "node: [$node] instance attribyte name: [$name], value: [$value]", 'line' => __LINE__, level => 1}); + to_log($conf, {message => "node: [".$node."] instance attribyte name: [".$name."], value: [".$value."]", 'line' => __LINE__, level => 1}); if (($name eq "maintenance") and ($value eq "on")) { # We're in maintenance mode, abort. @@ -334,7 +498,7 @@ sub identify_peer } my $quorate = $body->{'have-quorum'}; - to_log($conf, {message => "quorate: [$quorate]", 'line' => __LINE__, level => 1}); + to_log($conf, {message => "quorate: [".$quorate."]", 'line' => __LINE__, level => 1}); if (not $quorate) { to_log($conf, {message => "This not is not quorate. Refusing to fence the peer!", 'line' => __LINE__, level => 0, priority => "err"}); @@ -347,19 +511,19 @@ sub identify_peer my $node = $hash_ref->{uname}; my $join = $hash_ref->{'join'}; my $expected = $hash_ref->{expected}; - to_log($conf, {message => "node: [$node] join: [$join], expected: [$expected]", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "node: [".$node."] join: [".$join."], expected: [".$expected."]", 'line' => __LINE__, level => 3}); if ($node eq $conf->{cluster}{target_node}) { - to_log($conf, {message => "Checking the status of target node: [$node].", 'line' => __LINE__, level => 1}); + to_log($conf, {message => "Checking the status of target node: [".$node."].", 'line' => __LINE__, level => 1}); if (($join eq "down") && ($expected eq "down")) { # The node is out. - to_log($conf, {message => "The node: [$node] is already down. No actual fence needed.", 'line' => __LINE__, level => 1}); + to_log($conf, {message => "The node: [".$node."] is already down. No actual fence needed.", 'line' => __LINE__, level => 1}); exit(7); } else { - to_log($conf, {message => "The node: [$node] is: [$join/$expected] (join/expected). Proceeding with the fence action.", 'line' => __LINE__, level => 1}); + to_log($conf, {message => "The node: [".$node."] is: [".$join."/".$expected."] (join/expected). Proceeding with the fence action.", 'line' => __LINE__, level => 1}); } } } @@ -385,14 +549,14 @@ sub read_cib my $body = ""; my $cib = ''; my $shell_call = $conf->{path}{exe}{cibadmin}." --local --query"; - to_log($conf, {message => "Calling: [$shell_call]", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 3}); open (my $file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; while(<$file_handle>) { # This should not generate output. chomp; my $line = $_; - to_log($conf, {message => "Output: [$line]", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 3}); $cib .= "\n".$line; if ($line =~ /Signon to CIB failed/i) @@ -404,21 +568,19 @@ sub read_cib if ($line =~ /^$/) { $xml_opened = 1; - to_log($conf, {message => "xml_opened: [$xml_opened].", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "xml_opened: [".$xml_opened."].", 'line' => __LINE__, level => 3}); } if ($line =~ /^<\/cib>$/) { $xml_closed = 1; - to_log($conf, {message => "xml_closed: [$xml_closed].", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "xml_closed: [".$xml_closed."].", 'line' => __LINE__, level => 3}); } } close $file_handle; - my $return_code = $?; - to_log($conf, {message => "Return code: [$return_code]", 'line' => __LINE__, level => 3}); - to_log($conf, {message => "cib: ==========\n$cib\n==========", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "cib: ==========\n".$cib."\n==========", 'line' => __LINE__, level => 3}); # Now parse the CIB XML if I read it OK. - to_log($conf, {message => "xml_opened: [$xml_opened], xml_closed: [$xml_closed].", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "xml_opened: [".$xml_opened."], xml_closed: [".$xml_closed."].", 'line' => __LINE__, level => 3}); if (($xml_opened) && ($xml_closed)) { # We're good @@ -428,7 +590,7 @@ sub read_cib if (not $test) { chomp $@; - my $error = "[ Error ] - The was a problem parsing: [$cib]. The error was:\n"; + my $error = "[ Error ] - The was a problem parsing: [".$cib."]. The error was:\n"; $error .= "===========================================================\n"; $error .= $@."\n"; $error .= "===========================================================\n"; @@ -455,28 +617,34 @@ sub find_executables my $check = ""; my $bad = 0; - # Log entries can only happen if I've found 'logger', so an extra check will be made on 'to_log' + # If PATH isn't set, set it (could have been scrubbed by a caller). + if (not $ENV{PATH}) + { + $ENV{PATH} = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin"; + } + + # Log entries can only happen if I've found 'logger', so an extra check will be made on 'to_log' # calls. my @dirs = split/:/, $ENV{PATH}; foreach my $exe (sort {$b cmp $a} keys %{$conf->{path}{exe}}) { if ( not -e $conf->{path}{exe}{$exe} ) { - to_log($conf, {message => "The program: [$exe] is not at: [".$conf->{path}{exe}{$exe}."]. Looking for it now...", 'line' => __LINE__, level => 1}); + to_log($conf, {message => "The program: [".$exe."] is not at: [".$conf->{path}{exe}{$exe}."]. Looking for it now...", 'line' => __LINE__, level => 1}); foreach my $path (@dirs) { $check = "$path/$exe"; $check =~ s/\/\//\//g; - to_log($conf, {message => "Checking: [$check]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "Checking: [".$check."]", 'line' => __LINE__, level => 2}); if ( -e $check ) { if (-e $conf->{path}{exe}{logger}) { - to_log($conf, {message => "Found it! Changed path for: [$exe] from: [".$conf->{path}{exe}{$exe}."] to: [$check]", 'line' => __LINE__, level => 1}); + to_log($conf, {message => "Found it! Changed path for: [".$exe."] from: [".$conf->{path}{exe}{$exe}."] to: [".$check."]", 'line' => __LINE__, level => 1}); } else { - warn "DEBUG: Found it! Changed path for: [$exe] from: [".$conf->{path}{exe}{$exe}."] to: [$check]\n"; + warn "DEBUG: Found it! Changed path for: [".$exe."] from: [".$conf->{path}{exe}{$exe}."] to: [".$check."]\n"; } $conf->{path}{exe}{$exe} = $check; } @@ -493,17 +661,17 @@ sub find_executables } # Make sure it exists now. - to_log($conf, {message => "Checking again if: [$exe] is at: [".$conf->{path}{exe}{$exe}."].", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "Checking again if: [".$exe."] is at: [".$conf->{path}{exe}{$exe}."].", 'line' => __LINE__, level => 3}); if (not -e $conf->{path}{exe}{$exe}) { $bad = 1; if (-e $conf->{path}{exe}{logger}) { - to_log($conf, {message => "Failed to find executable: [$exe]. Unable to proceed.", 'line' => __LINE__, level => 0}); + to_log($conf, {message => "Failed to find executable: [".$exe."]. Unable to proceed.", 'line' => __LINE__, level => 0}); } else { - warn "Failed to find executable: [$exe]. Unable to proceed.\n"; + warn "Failed to find executable: [".$exe."]. Unable to proceed.\n"; } } } @@ -527,27 +695,27 @@ sub check_peer_is_fenced my $node = $hash_ref->{uname}; my $join = $hash_ref->{'join'}; my $expected = $hash_ref->{expected}; - to_log($conf, {message => "node: [$node] join: [$join], expected: [$expected]", 'line' => __LINE__, level => 3}); + to_log($conf, {message => "node: [".$node."] join: [".$join."], expected: [".$expected."]", 'line' => __LINE__, level => 3}); if ($node eq $conf->{cluster}{target_node}) { - to_log($conf, {message => "Checking the status of target node: [$node].", 'line' => __LINE__, level => 1}); + to_log($conf, {message => "Checking the status of target node: [".$node."].", 'line' => __LINE__, level => 1}); if (($join eq "down") && ($expected eq "down")) { # The node is out. - to_log($conf, {message => "The node: [$node] has been fenced successfully.", 'line' => __LINE__, level => 1}); + to_log($conf, {message => "The node: [".$node."] has been fenced successfully.", 'line' => __LINE__, level => 1}); # Call 'drbdadm adjust all' as it seems like drbd's in-memory can change # causing 'incompatible ' on return of the peer. to_log($conf, {message => "Reloading DRBD config from disk to ensure in-memory and on-disk configs match.", 'line' => __LINE__, level => 1}); my $shell_call = $conf->{path}{exe}{drbdadm}." adjust all"; - to_log($conf, {message => "Calling: [$shell_call]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 2}); open (my $file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; while(<$file_handle>) { # This should not generate output. chomp; my $line = $_; - to_log($conf, {message => "Output: [$line]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 2}); } close $file_handle; to_log($conf, {message => "Fence completed successfully!", 'line' => __LINE__, level => 1}); @@ -556,7 +724,7 @@ sub check_peer_is_fenced } else { - to_log($conf, {message => "The node: [$node] is: [$join/$expected] (join/expected). It has not yet been fenced.", 'line' => __LINE__, level => 1}); + to_log($conf, {message => "The node: [".$node."] is: [".$join."/".$expected."] (join/expected). It has not yet been fenced.", 'line' => __LINE__, level => 1}); } } } @@ -570,15 +738,15 @@ sub kill_target my ($conf) = @_; # Variables - my $shell_call = $conf->{path}{exe}{stonith_admin}." --fence ".$conf->{cluster}{target_node}." --verbose; RC=\$?; ".$conf->{path}{exe}{crm_error}." \$RC; ".$conf->{path}{exe}{echo}." rc:\$RC"; - to_log($conf, {message => "Calling: [$shell_call]", 'line' => __LINE__, level => 2}); + my $shell_call = $conf->{path}{exe}{stonith_admin}." --fence ".$conf->{cluster}{target_node}." --reboot --verbose; RC=\$?; ".$conf->{path}{exe}{crm_error}." \$RC; ".$conf->{path}{exe}{echo}." rc:\$RC"; + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 2}); open (my $file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; while(<$file_handle>) { # This should not generate output. chomp; my $line = $_; - to_log($conf, {message => "Output: [$line]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 2}); } close $file_handle; @@ -587,7 +755,7 @@ sub kill_target my $start_time = time; my $end_time = $start_time + 300; my $fenced = 0; - to_log($conf, {message => "start_time: [$start_time], end_time: [$end_time]", 'line' => __LINE__, level => 2}); + to_log($conf, {message => "start_time: [".$start_time."], end_time: [".$end_time."]", 'line' => __LINE__, level => 2}); until ($fenced) { # This will exit @@ -595,7 +763,7 @@ sub kill_target if (time > $end_time) { # Done waiting, failed. - to_log($conf, {message => "The node has not been fenced after five minutes. Giving up..", 'line' => __LINE__, level => 0, priority => "err"}); + to_log($conf, {message => "The node has not been fenced after five minutes. Giving up.", 'line' => __LINE__, level => 0, priority => "err"}); exit(1); } else diff --git a/tools/striker-get-peer-data b/tools/striker-get-peer-data index cb6119d2..cbdab6f6 100755 --- a/tools/striker-get-peer-data +++ b/tools/striker-get-peer-data @@ -355,7 +355,7 @@ sub get_password else { # We have the password. Delete the entry now. - my $query = "DELETE FROM states WHERE state_uuid=".$anvil->Database->quote($anvil->data->{switches}{'state-uuid'}).";"; + my $query = "DELETE FROM states WHERE state_uuid = ".$anvil->Database->quote($anvil->data->{switches}{'state-uuid'}).";"; $anvil->Database->write({uuid => $anvil->data->{sys}{host_uuid}, debug => 3, query => $query, source => $THIS_FILE, line => __LINE__}); } diff --git a/tools/unfence_pacemaker b/tools/unfence_pacemaker new file mode 100755 index 00000000..12c4b307 --- /dev/null +++ b/tools/unfence_pacemaker @@ -0,0 +1,652 @@ +#!/usr/bin/perl +# +# Author: Madison Kelly (mkelly@alteeve.ca) +# Alteeve's Niche! Inc. - https://alteeve.com/w/ +# Version: 0.0.1 +# License: GPL v2+ +# +# This program ties LINBIT's DRBD fencing into pacemaker's stonith. It provides a power-fence alternative to +# the default 'crm-{un,}fence-peer.sh' {un,}fence-handler. +# +# WARNING: This unfence handler is probably not safe to use outside of an Anvil! IA platform. It makes a lot +# of operational assumptions about the system and desired goals. +# +# TODO: +# - + +### NOTE: This doesn't use Anvil::Tools on purpose. We want to be quick and depend on as few things as +### possible. + +use strict; +use warnings; +use XML::Simple; +use Data::Dumper; + +# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete. +$| = 1; + +my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; +my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; +if (($running_directory =~ /^\./) && ($ENV{PWD})) +{ + $running_directory =~ s/^\./$ENV{PWD}/; +} + +my $conf = { + 'log' => { + facility => "local0", + level => 2, + line_numbers => 1, + tag => $THIS_FILE, + }, + # If a program isn't at the defined path, $ENV{PATH} will be searched. + path => { + exe => { + cibadmin => "/usr/sbin/cibadmin", + crm_attribute => "/usr/sbin/crm_attribute", + crm_error => "/usr/sbin/crm_error", + drbdadm => "/usr/sbin/drbdadm", + echo => "/usr/bin/echo", + getent => "/usr/bin/getent", + logger => "/usr/bin/logger", + pcs => "/usr/sbin/pcs", + stonith_admin => "/usr/sbin/stonith_admin", + }, + }, + # The script will set this. + cluster => { + target_node => "", + }, + # These are the environment variables set by DRBD. See 'man drbd.conf' + # -> 'handlers'. + environment => { + # The resource triggering the fence. + 'DRBD_RESOURCE' => defined $ENV{DRBD_RESOURCE} ? $ENV{DRBD_RESOURCE} : "", + # The resource minor number, or, in the case of volumes, numbers. + 'DRBD_MINOR' => defined $ENV{DRBD_MINOR} ? $ENV{DRBD_MINOR} : "", + # This is the address format (ipv4, ipv6, etc) + 'DRBD_PEER_AF' => defined $ENV{DRBD_PEER_AF} ? $ENV{DRBD_PEER_AF} : "", + # This is the IP address of the target node. + 'DRBD_PEER_ADDRESS' => defined $ENV{DRBD_PEER_ADDRESS} ? $ENV{DRBD_PEER_ADDRESS} : "", + # This isn't set + 'DRBD_PEERS' => defined $ENV{DRBD_PEERS} ? $ENV{DRBD_PEERS} : "", + ### NOTE: Below here are undocumented variables. Don't expect them to always be useful. + # My node ID + 'DRBD_MY_NODE_ID' => defined $ENV{DRBD_MY_NODE_ID} ? $ENV{DRBD_MY_NODE_ID} : "", + # The target's ID + 'DRBD_PEER_NODE_ID' => defined $ENV{DRBD_PEER_NODE_ID} ? $ENV{DRBD_PEER_NODE_ID} : "", + }, +}; + +# Find executables. +find_executables($conf); + +# Something for the logs +to_log($conf, {message => "The Anvil! DRBD unfence handler has been invoked.", 'line' => __LINE__}); + +# These are the full host names of the nodes given their IDs. +foreach my $i (0..31) +{ + my $key = "DRBD_NODE_ID_".$i; + if ((exists $ENV{$key}) && (defined $ENV{$key})) + { + $conf->{environment}{$key} = $ENV{$key}; + my $level = $conf->{environment}{$key} eq "" ? 3 : 2; + to_log($conf, {message => "DRBD Environment variable: [$key] -> [".$conf->{environment}{$key}."]", 'line' => __LINE__, level => $level}); + } +} + +# Record the environment variables +foreach my $key (sort {$a cmp $b} keys %{$conf->{environment}}) +{ + # $conf->{environment}{DRBD_RESOURCE} -> [srv51-Workstation3] + my $level = $conf->{environment}{$key} eq "" ? 3 : 2; + to_log($conf, {message => "DRBD Environment variable: [$key] -> [".$conf->{environment}{$key}."]", 'line' => __LINE__, level => $level}); +} +foreach my $key (sort {$a cmp $b} keys %ENV) +{ + next if exists $conf->{environment}{$key}; + my $level = $ENV{$key} eq "" ? 3 : 2; + to_log($conf, {message => "System Environment variable: [$key] -> [".$ENV{$key}."]", 'line' => __LINE__, level => 3}); +} + +# Make sure we at least have the target's IP. +if (not $conf->{environment}{DRBD_PEER_ADDRESS}) +{ + to_log($conf, {message => "Called without target's IP. Nothing to do, exiting. Were we called by 'pcs stonith list'?", 'line' => __LINE__, level => 1, priority => "alert"}); + exit(1); +} + +# This also checks that we're quorate and not in maintenance mode. +identify_peer($conf); + +# If we're still alive, we now need to check the DRBD resource disk state locally. +get_drbd_status($conf); + +# Is there a specific resource? +if ($conf->{environment}{DRBD_RESOURCE}) +{ + # Prevent the resource from running on the peer. + to_log($conf, {message => "We're being asked to unfence the specific resource: [".$conf->{environment}{DRBD_RESOURCE}."]. Will remove the node attribute blocking this server from running on: [".$conf->{cluster}{target_node}."].", 'line' => __LINE__}); + remove_constraint($conf); +} +else +{ + to_log($conf, {message => "We were not given a specific resource to unfence. Nothing to do.", 'line' => __LINE__}); + exit(0); +} + +# If we hit here, something very wrong happened. +exit(1); + + +############################################################################################################# +# Functions # +############################################################################################################# + +# This removes a location constraint that prevents the resource / server from running on the peer node. +sub remove_constraint +{ + my ($conf) = @_; + + my $target_server = $conf->{environment}{DRBD_RESOURCE}; + my $target_node = $conf->{cluster}{target_node}; + to_log($conf, {message => "Will now create a location constraint against: [".$target_server."] preventing it from running on: [".$target_node."].", 'line' => __LINE__, level => 1}); + + # Check that the rule was set. + my $rule_name = "drbd-fenced_".$target_server; + my $rule_set = 1; + my $rule_found = 0; + my $rule_output = ""; + my $shell_call = $conf->{path}{exe}{crm_attribute}." --type nodes --node ".$target_node." --name ".$rule_name." --query"; + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 2}); + open (my $file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; + while(<$file_handle>) + { + # This should not generate output. + chomp; + my $line = $_; + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 2}); + + if (($line =~ /name=$rule_name/) && ($line =~ /value=0/)) + { + $rule_set = 0; + to_log($conf, {message => "rule_set: [".$rule_set."]", 'line' => __LINE__, level => 2}); + last; + } + else + { + $rule_output .= $line."\n"; + } + } + close $file_handle; + + if (not $rule_set) + { + # No need to unfence. + to_log($conf, {message => "The node attribute rule: [".$rule_name."] against the node: [".$target_node."] was not found. No need to unfence.", 'line' => __LINE__, level => 0, priority => "err"}); + exit(0); + } + + # Clear the node attribute + $shell_call = $conf->{path}{exe}{crm_attribute}." --type nodes --node ".$target_node." --name ".$rule_name." --update 0"; + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 2}); + open ($file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; + while(<$file_handle>) + { + # This should not generate output. + chomp; + my $line = $_; + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 3}); + } + close $file_handle; + + # Check that the rule was set. + $rule_output = ""; + $shell_call = $conf->{path}{exe}{crm_attribute}." --type nodes --node ".$target_node." --name ".$rule_name." --query"; + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 2}); + open ($file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; + while(<$file_handle>) + { + # This should not generate output. + chomp; + my $line = $_; + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 2}); + + if (($line =~ /name=$rule_name/) && ($line =~ /value=0/)) + { + $rule_set = 0; + to_log($conf, {message => "rule_set: [".$rule_set."]", 'line' => __LINE__, level => 2}); + last; + } + else + { + $rule_output .= $line."\n"; + } + } + close $file_handle; + + if (not $rule_set) + { + # Success! + to_log($conf, {message => "The node attribute rule: [".$rule_name."] against the node: [".$target_node."] has been cleared successfully.", 'line' => __LINE__, level => 0, priority => "err"}); + exit(0); + } + else + { + # Failed. + $rule_output =~ s/\n$//gs; + to_log($conf, {message => "The node attribute triggering the fence rule: [".$rule_name."] against the node: [".$target_node."] appears to have not been cleared. Expected a string with 'name=".$rule_name." value=0' but got: [".$rule_output."].", 'line' => __LINE__, level => 0, priority => "err"}); + exit(1); + } + + return(0); +} + +# This reads the status of all resources. If we're not all UpToDate, check if the peer is. If the peer is, +# abort. If not, proceed (someone is gouig to have a bad day, but maybe some servers will live) +sub get_drbd_status +{ + my ($conf) = @_; + + my $resource = ""; + my $peer = ""; + my $local_all_uptodate = 1; + my $peer_all_uptodate = 1; + my $shell_call = $conf->{path}{exe}{drbdadm}." status all"; + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 2}); + open (my $file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; + while(<$file_handle>) + { + # This should not generate output. + chomp; + my $line = $_; + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 3}); + + if (not $line) + { + $resource = ""; + $peer = ""; + to_log($conf, {message => "resource: [".$resource."], peer: [".$peer."]", 'line' => __LINE__, level => 3}); + next; + } + if ($line =~ /^(\S+)\s+role/) + { + $resource = $1; + to_log($conf, {message => "resource: [".$resource."]", 'line' => __LINE__, level => 3}); + next; + } + if ($line =~ /^\s+(.*?) role:/) + { + $peer = $1; + to_log($conf, {message => "peer: [".$peer."]", 'line' => __LINE__, level => 3}); + next; + } + if ($resource) + { + if ($line =~ /disk:(.*)$/) + { + my $local_dstate = $1; + $local_dstate =~ s/\s.*$//; + to_log($conf, {message => "local_dstate: [".$local_dstate."]", 'line' => __LINE__, level => 2}); + if (lc($local_dstate) ne "uptodate") + { + $local_all_uptodate = 0; + to_log($conf, {message => "local_all_uptodate: [".$local_all_uptodate."]", 'line' => __LINE__, level => 2}); + } + next; + } + if ($line =~ /peer-disk:(.*)$/) + { + my $peer_dstate = $1; + $peer_dstate =~ s/\s.*$//; + to_log($conf, {message => "peer: [".$peer."], peer_dstate: [".$peer_dstate."]", 'line' => __LINE__, level => 2}); + if (lc($peer_dstate) ne "uptodate") + { + $peer_all_uptodate = 0; + to_log($conf, {message => "peer_all_uptodate: [".$peer_all_uptodate."]", 'line' => __LINE__, level => 2}); + } + next; + } + } + + } + close $file_handle; + + # If we're not all UpToDate, but the peer is, abort + to_log($conf, {message => "local_all_uptodate: [".$local_all_uptodate."], peer_all_uptodate: [".$peer_all_uptodate."]", 'line' => __LINE__, level => 2}); + if ((not $local_all_uptodate) && ($peer_all_uptodate)) + { + # We're not good + to_log($conf, {message => "This node has DRBD resources that are not UpToDate, but the peer is fully UpToDate. Aborting.", 'line' => __LINE__, level => 0, priority => "err"}); + exit(1); + } + + return(0); +} + +# This identifies the pacemaker name of the target node. If it can't find the peer, it exits with '1'. +sub identify_peer +{ + my ($conf) = @_; + + # I know the target's (SN) IP, map it to a node. + my $target_host = ""; + my $target_ip = $conf->{environment}{DRBD_PEER_ADDRESS}; + + # First, can we translate the IP to a host name? + my $shell_call = $conf->{path}{exe}{getent}." hosts ".$target_ip; + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 2}); + open (my $file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; + while(<$file_handle>) + { + # This should not generate output. + chomp; + my $line = $_; + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 2}); + if ($line =~ /^$target_ip\s+(.*)$/) + { + # This could be multiple names. + $target_host = $1; + to_log($conf, {message => "target_host: [".$target_host."]", 'line' => __LINE__, level => 2}); + #to_log($conf, {message => ">> target_host: [".$target_host."]", 'line' => __LINE__, level => 2}); + + # Strip off any suffix, we only want the short name. + #$target_host =~ s/\..*//; + #to_log($conf, {message => "<< target_host: [".$target_host."]", 'line' => __LINE__, level => 2}); + #last; + } + } + close $file_handle; + + # If I got the host name, try to match it to a pacemaker node name. + if ($target_host) + { + # Get the current CIB (in an XML::Simple hash). This will exit if it fails to read the XML + # and convert it to an XML::Simple hash. + my $body = read_cib($conf); + + # Parse the XML. + my $host_name = $ENV{HOSTNAME}; + my $short_host_name = $ENV{HOSTNAME}; + $short_host_name =~ s/\..*$//; + to_log($conf, {message => "host_name: [".$host_name."], short_host_name: [".$short_host_name."]", 'line' => __LINE__, level => 2}); + + foreach my $hash_ref (sort {$a cmp $b} @{$body->{configuration}{nodes}{node}}) + { + my $node = $hash_ref->{uname}; + my $id = $hash_ref->{id}; + to_log($conf, {message => "node: [".$node."], id: [".$id."]", 'line' => __LINE__, level => 2}); + foreach my $target_name (split/ /, $target_host) + { + to_log($conf, {message => ">> target_name: [".$target_name."]", 'line' => __LINE__, level => 2}); + $target_name =~ s/\..*//; + to_log($conf, {message => "<< target_name: [".$target_name."]", 'line' => __LINE__, level => 2}); + if ($node =~ /^$target_name/) + { + $conf->{cluster}{target_node} = $node; + to_log($conf, {message => "Found the pacemaker name of the target node: [".$conf->{cluster}{target_node}."]", 'line' => __LINE__, level => 1}); + } + elsif ($node =~ /^$short_host_name/) + { + # This is me. Am I in maintenance mode? + if (exists $hash_ref->{instance_attributes}) + { + # We've got some data.. + my $name = defined $hash_ref->{instance_attributes}{nvpair}{name} ? $hash_ref->{instance_attributes}{nvpair}{name} : ""; + my $value = defined $hash_ref->{instance_attributes}{nvpair}{value} ? $hash_ref->{instance_attributes}{nvpair}{value} : ""; + to_log($conf, {message => "node: [".$node."] instance attribyte name: [".$name."], value: [".$value."]", 'line' => __LINE__, level => 1}); + if (($name eq "maintenance") and ($value eq "on")) + { + # We're in maintenance mode, abort. + to_log($conf, {message => "This node is in maintenance mode. Not able to fence!", 'line' => __LINE__, level => 0, priority => "err"}); + exit(1); + } + } + } + } + } + + my $quorate = $body->{'have-quorum'}; + to_log($conf, {message => "quorate: [".$quorate."]", 'line' => __LINE__, level => 1}); + if (not $quorate) + { + to_log($conf, {message => "This not is not quorate. Refusing to fence the peer!", 'line' => __LINE__, level => 0, priority => "err"}); + exit(1); + } + + # See if the target node is already out of the cluster. + foreach my $hash_ref (@{$body->{status}{node_state}}) + { + my $node = $hash_ref->{uname}; + my $join = $hash_ref->{'join'}; + my $expected = $hash_ref->{expected}; + to_log($conf, {message => "node: [".$node."] join: [".$join."], expected: [".$expected."]", 'line' => __LINE__, level => 3}); + if ($node eq $conf->{cluster}{target_node}) + { + to_log($conf, {message => "Checking the status of target node: [".$node."].", 'line' => __LINE__, level => 1}); + if (($join eq "down") && ($expected eq "down")) + { + # The node is out. + to_log($conf, {message => "The node: [".$node."] is already down. No actual fence needed.", 'line' => __LINE__, level => 1}); + exit(7); + } + else + { + to_log($conf, {message => "The node: [".$node."] is: [".$join."/".$expected."] (join/expected). Proceeding with the fence action.", 'line' => __LINE__, level => 1}); + } + } + } + } + + # Did I find the target? + if (not $conf->{cluster}{target_node}) + { + to_log($conf, {message => "Failed to find the pacemaker name of the target node. Unable to proceed!", 'line' => __LINE__, level => 0, priority => "err"}); + exit(1); + } + + return(0); +} + +# This reads in the CIB XML and returns it as a single multi-line string. +sub read_cib +{ + my ($conf) = @_; + + my $xml_opened = 0; + my $xml_closed = 0; + my $body = ""; + my $cib = ''; + my $shell_call = $conf->{path}{exe}{cibadmin}." --local --query"; + to_log($conf, {message => "Calling: [".$shell_call."]", 'line' => __LINE__, level => 3}); + open (my $file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; + while(<$file_handle>) + { + # This should not generate output. + chomp; + my $line = $_; + to_log($conf, {message => "Output: [".$line."]", 'line' => __LINE__, level => 3}); + + $cib .= "\n".$line; + if ($line =~ /Signon to CIB failed/i) + { + # Failed to connect, we're probably not in the cluster. + to_log($conf, {message => "This node does not appear to be in the cluster. Unable to get the CIB status.", 'line' => __LINE__, level => 0, priority => "err"}); + exit(1); + } + if ($line =~ /^$/) + { + $xml_opened = 1; + to_log($conf, {message => "xml_opened: [".$xml_opened."].", 'line' => __LINE__, level => 3}); + } + if ($line =~ /^<\/cib>$/) + { + $xml_closed = 1; + to_log($conf, {message => "xml_closed: [".$xml_closed."].", 'line' => __LINE__, level => 3}); + } + } + close $file_handle; + to_log($conf, {message => "cib: ==========\n".$cib."\n==========", 'line' => __LINE__, level => 3}); + + # Now parse the CIB XML if I read it OK. + to_log($conf, {message => "xml_opened: [".$xml_opened."], xml_closed: [".$xml_closed."].", 'line' => __LINE__, level => 3}); + if (($xml_opened) && ($xml_closed)) + { + # We're good + local $@; + my $xml = XML::Simple->new(); + my $test = eval { $body = $xml->XMLin($cib, KeyAttr => { language => 'name', key => 'name' }, ForceArray => [ 'id' ]) }; + if (not $test) + { + chomp $@; + my $error = "[ Error ] - The was a problem parsing: [".$cib."]. The error was:\n"; + $error .= "===========================================================\n"; + $error .= $@."\n"; + $error .= "===========================================================\n"; + to_log($conf, {message => $error, 'line' => __LINE__, level => 0, priority => "err"}); + exit(1); + } + } + else + { + # Failed to read the CIB XML. + to_log($conf, {message => "This node does not appear to be in the cluster. Unable to read the CIB XML properly.", 'line' => __LINE__, level => 2, priority => "err"}); + exit(1); + } + + return($body); +} + +# This checks the given paths and, if something isn't found, it searches PATH trying to find it. +sub find_executables +{ + my ($conf) = @_; + + # Variables. + my $check = ""; + my $bad = 0; + + # If PATH isn't set, set it (could have been scrubbed by a caller). + if (not $ENV{PATH}) + { + $ENV{PATH} = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin"; + } + + # Log entries can only happen if I've found 'logger', so an extra check will be made on 'to_log' + # calls. + my @dirs = split/:/, $ENV{PATH}; + foreach my $exe (sort {$b cmp $a} keys %{$conf->{path}{exe}}) + { + if ( not -e $conf->{path}{exe}{$exe} ) + { + to_log($conf, {message => "The program: [".$exe."] is not at: [".$conf->{path}{exe}{$exe}."]. Looking for it now...", 'line' => __LINE__, level => 1}); + foreach my $path (@dirs) + { + $check = "$path/$exe"; + $check =~ s/\/\//\//g; + to_log($conf, {message => "Checking: [".$check."]", 'line' => __LINE__, level => 2}); + if ( -e $check ) + { + if (-e $conf->{path}{exe}{logger}) + { + to_log($conf, {message => "Found it! Changed path for: [".$exe."] from: [".$conf->{path}{exe}{$exe}."] to: [".$check."]", 'line' => __LINE__, level => 1}); + } + else + { + warn "DEBUG: Found it! Changed path for: [".$exe."] from: [".$conf->{path}{exe}{$exe}."] to: [".$check."]\n"; + } + $conf->{path}{exe}{$exe} = $check; + } + else + { + to_log($conf, {message => "Not found.", 'line' => __LINE__, level => 2}); + } + } + } + else + { + to_log($conf, {message => "Found!", 'line' => __LINE__, level => 3}); + next; + } + + # Make sure it exists now. + to_log($conf, {message => "Checking again if: [".$exe."] is at: [".$conf->{path}{exe}{$exe}."].", 'line' => __LINE__, level => 3}); + if (not -e $conf->{path}{exe}{$exe}) + { + $bad = 1; + if (-e $conf->{path}{exe}{logger}) + { + to_log($conf, {message => "Failed to find executable: [".$exe."]. Unable to proceed.", 'line' => __LINE__, level => 0}); + } + else + { + warn "Failed to find executable: [".$exe."]. Unable to proceed.\n"; + } + } + } + if ($bad) + { + exit(1); + } + + return(0); +} + +# Log file entries +sub to_log +{ + my ($conf, $parameters) = @_; + + my $facility = defined $parameters->{facility} ? $parameters->{facility} : $conf->{'log'}{facility}; + my $level = defined $parameters->{level} ? $parameters->{level} : 1; + my $line = defined $parameters->{'line'} ? $parameters->{'line'} : 0; + my $message = defined $parameters->{message} ? $parameters->{message} : ""; + my $priority = defined $parameters->{priority} ? $parameters->{priority} : ""; + + # Leave if we don't care about this message + return if $level > $conf->{'log'}{level}; + return if not $message; + + # Build the message. We log the line + if (($conf->{'log'}{line_numbers}) && ($line)) + { + $message = $line."; ".$message; + } + + my $priority_string = $facility; + if ($priority) + { + $priority_string .= ".".$priority; + } + elsif ($level eq "0") + { + $priority_string .= ".notice"; + } + elsif (($level eq "1") or ($level eq "2")) + { + $priority_string .= ".info"; + } + else + { + $priority_string .= ".debug"; + } + + # Clean up the string for bash + $message =~ s/"/\\\"/gs; + #$message =~ s/\(/\\\(/gs; + + my $shell_call = $conf->{path}{exe}{logger}." --priority ".$priority_string." --tag ".$conf->{'log'}{tag}." -- \"".$message."\""; + open (my $file_handle, $shell_call." 2>&1 |") or die "Failed to call: [".$shell_call."]. The error was: $!\n"; + while(<$file_handle>) + { + # This should not generate output. + chomp; + my $line = $_; + print "Unexpected logging output: [".$line."]\n"; + } + close $file_handle; + + return(0); +} +