You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
3339 lines
141 KiB
3339 lines
141 KiB
#!/usr/bin/perl |
|
# |
|
# This adds a node (or gets a node to join to a new/rebuilt peer or added DR) to an Anvil!. |
|
# |
|
# Exit codes; |
|
# 0 = Normal exit. |
|
# 1 = Failed to connect to any database. |
|
# 2 = Failed to load/parse the manifest. |
|
# 3 = Failed to change the host name. |
|
# 4 = Failed to reconnect to any database after the network was reconfigured |
|
# 5 = Problem parsing job data or loading manifest or anvil data using job data. |
|
# 6 = A pcs call failed. |
|
# 7 = No job was found to run. |
|
# |
|
# TODO: |
|
# - Check to see if this is a cluster node and/or running VMs, and if so, refuse to run. |
|
# |
|
|
|
use strict; |
|
use warnings; |
|
use Anvil::Tools; |
|
use Data::Dumper; |
|
use String::ShellQuote; |
|
use Text::Diff; |
|
use NetAddr::IP; |
|
|
|
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; |
|
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; |
|
if (($running_directory =~ /^\./) && ($ENV{PWD})) |
|
{ |
|
$running_directory =~ s/^\./$ENV{PWD}/; |
|
} |
|
|
|
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete. |
|
$| = 1; |
|
|
|
my $anvil = Anvil::Tools->new(); |
|
|
|
# Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is |
|
# passed directly, it will be used. Otherwise, the password will be read from the database. |
|
$anvil->Get->switches({list => [ |
|
"as-machine", |
|
"join", |
|
"manifest", |
|
"rejoin", |
|
], man => $THIS_FILE}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }}); |
|
|
|
$anvil->Database->connect(); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"}); |
|
if (not $anvil->data->{sys}{database}{connections}) |
|
{ |
|
# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try |
|
# again after we exit. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0129"}); |
|
sleep 2; |
|
$anvil->nice_exit({exit_code => 1}); |
|
} |
|
|
|
# If there isn't a job UUID, see if the user is asking to rejoin this subnode to a cluster. |
|
if ($anvil->data->{switches}{rejoin}) |
|
{ |
|
process_rejoin($anvil); |
|
$anvil->nice_exit({exit_code => 0}); |
|
} |
|
|
|
# Get the job details |
|
load_job($anvil); |
|
|
|
# Hold until both subnodes are marked as configured and not in maintenance mode. |
|
wait_for_subnodes($anvil); |
|
|
|
# Update the user passwords |
|
update_passwords($anvil); |
|
|
|
# Check if we need to change any IPs or our hostname. |
|
check_local_network($anvil); |
|
|
|
# Make sure the hosts file has entries for all nets for both subnodes |
|
wait_for_etc_hosts($anvil); |
|
|
|
# Wait until we can ping our peer on all networks. |
|
wait_for_access($anvil); |
|
|
|
### TODO: Change corosync.conf to use IPs, including MN if it exists. |
|
# (wait for out peer and) Configure pacemaker |
|
configure_pacemaker($anvil); |
|
|
|
# Configure DRBD |
|
configure_drbd($anvil); |
|
|
|
# Enable anvil-safe-start |
|
if (1) |
|
{ |
|
my ($return_code) = $anvil->System->enable_daemon({daemon => "anvil-safe-start.service"}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }}); |
|
update_progress($anvil, 99, "job_0094,!!daemon!anvil-safe-start.service!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "anvil-safe-start.service" }}); |
|
} |
|
|
|
update_progress($anvil, 100, "job_0129"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0129"}); |
|
|
|
# Record that we're done configuring pacemaker. |
|
$anvil->Database->insert_or_update_variables({ |
|
variable_name => "system::all::configured", |
|
variable_value => 1, |
|
variable_default => "", |
|
variable_description => "striker_0297", |
|
variable_section => "system", |
|
variable_source_uuid => $anvil->data->{sys}{host_uuid}, |
|
variable_source_table => "hosts", |
|
}); |
|
|
|
$anvil->nice_exit({exit_code => 0}); |
|
|
|
|
|
############################################################################################################# |
|
# Functions # |
|
############################################################################################################# |
|
|
|
# Rejoin this host to an existing subnode. |
|
sub process_rejoin |
|
{ |
|
my ($anvil) = @_; |
|
|
|
print "-=] Rejoin this host: [".$anvil->Get->short_host_name."] as a subnode in an existing Anvil! node.\n"; |
|
my $host_uuid = $anvil->Get->host_uuid(); |
|
my $as_machine = $anvil->data->{switches}{'as-machines'}; |
|
my $manifest = $anvil->data->{switches}{manifest}; |
|
my $manifest_name = ""; |
|
my $manifest_uuid = ""; |
|
my $anvil_uuid = ""; |
|
my $rebuild = 0; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:host_uuid' => $host_uuid, |
|
's2:as_machine' => $as_machine, |
|
's3:manifest' => $manifest, |
|
}}); |
|
|
|
# Load data |
|
$anvil->Database->get_hosts(); |
|
$anvil->Database->get_anvils(); |
|
$anvil->Database->get_manifests(); |
|
|
|
# In case we're being re-installed, see if we can find our own data. |
|
my $old_manifest_uuid = ""; |
|
if (exists $anvil->data->{anvils}{host_uuid}{$host_uuid}) |
|
{ |
|
# Found it. |
|
my $anvil_name = $anvil->data->{anvils}{host_uuid}{$host_uuid}{anvil_name}; |
|
$anvil_uuid = $anvil->data->{anvils}{host_uuid}{$host_uuid}{anvil_uuid}; |
|
my $old_role = $anvil->data->{anvils}{host_uuid}{$host_uuid}{role}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:anvil_name' => $anvil_name, |
|
's2:anvil_uuid' => $anvil_uuid, |
|
}}); |
|
print "This host used to be in the Anvil! [".$anvil_name."] as: [".$old_role."]\n"; |
|
if (exists $anvil->data->{manifests}{manifest_name}{$anvil_name}) |
|
{ |
|
# Found the manifest. |
|
$old_manifest_uuid = $anvil->data->{manifests}{manifest_name}{$anvil_name}{manifest_uuid}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_manifest_uuid => $old_manifest_uuid }}); |
|
print "The manifest was found with the UUID: [".$old_manifest_uuid."]\n"; |
|
} |
|
else |
|
{ |
|
# Didn't find the manifest, something is wrong. |
|
print "The manifest was NOT found (was it deleted?).\n"; |
|
print "- The manifes must be created for this host to rejoin the Anvil! node.\n"; |
|
print "- Alternatively, this host must be removed for the Anvil!\n"; |
|
print "[ Error ] - Unable to proceed at this time.\n"; |
|
$anvil->nice_exit({exit_code => 1}); |
|
} |
|
|
|
# Does it match? |
|
if (($manifest) && (($manifest ne $anvil_name) && ($manifest ne $old_manifest_uuid))) |
|
{ |
|
# They asked for this machine to be joined to a different Anvil!. |
|
print "[ Error ] - You asked to join the Anvil! node: [".$manifest."].\n"; |
|
print "[ Error ] - This host must be removed from: [".$anvil_name."] first.\n"; |
|
print "[ Error ] - Unable to proceed at this time.\n"; |
|
$anvil->nice_exit({exit_code => 1}); |
|
} |
|
|
|
if (($as_machine) && ($as_machine ne $old_role)) |
|
{ |
|
print "[ Error ] - You asked to join the Anvil! node: [".$manifest."] as: [".$as_machine."]\n"; |
|
print "[ Error ] - This host was previously: [".$old_role."], so it needs to be that role again.\n"; |
|
print "[ Error ] - Alternatively, remove this host from that Anvil! and try again.\n"; |
|
print "[ Error ] - Unable to proceed at this time.\n"; |
|
$anvil->nice_exit({exit_code => 1}); |
|
} |
|
|
|
$manifest = $anvil_name; |
|
$as_machine = $old_role; |
|
$rebuild = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:manifest' => $manifest, |
|
's2:as_machine' => $as_machine, |
|
's3:rebuild' => $rebuild, |
|
}}); |
|
} |
|
|
|
if ($manifest) |
|
{ |
|
# Did we get a valid manifest? |
|
if (exists $anvil->data->{manifests}{manifest_name}{$manifest}) |
|
{ |
|
$manifest_name = $manifest; |
|
$manifest_uuid = $anvil->data->{manifests}{manifest_name}{$manifest}{manifest_uuid}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:manifest_name' => $manifest_name, |
|
's2:manifest_uuid' => $manifest_uuid, |
|
}}); |
|
} |
|
elsif (exists $anvil->data->{manifests}{manifest_uuid}{$manifest}) |
|
{ |
|
$manifest_name = $anvil->data->{manifests}{manifest_uuid}{$manifest}{manifest_name}; |
|
$manifest_uuid = $manifest; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:manifest_name' => $manifest_name, |
|
's2:manifest_uuid' => $manifest_uuid, |
|
}}); |
|
} |
|
else |
|
{ |
|
print "[ Error ] - The manifest: [".$manifest."] was not found in the database.\n"; |
|
print "[ Error ] - Try again without this switch to see the available manifests.\n"; |
|
$anvil->nice_exit({exit_code => 1}); |
|
} |
|
} |
|
else |
|
{ |
|
# Show the existing manifests. |
|
my $count = keys %{$anvil->data->{manifests}{manifest_name}}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); |
|
foreach my $this_manifest_name (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_name}}) |
|
{ |
|
my $this_manifest_uuid = $anvil->data->{manifests}{manifest_name}{$this_manifest_name}{manifest_uuid}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:this_manifest_name' => $this_manifest_name, |
|
's2:this_manifest_uuid' => $this_manifest_uuid, |
|
}}); |
|
if ($count == 1) |
|
{ |
|
$manifest_name = $this_manifest_name; |
|
$manifest_uuid = $this_manifest_uuid; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:manifest_name' => $manifest_name, |
|
's2:manifest_uuid' => $manifest_uuid, |
|
}}); |
|
} |
|
else |
|
{ |
|
print "- Manifest: [".$manifest_name."] (uuid: [".$manifest_uuid."])\n"; |
|
} |
|
} |
|
print "- Which manifest do you want to join this host to?\n"; |
|
my $answer = <STDIN>; |
|
chomp($answer); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }}); |
|
|
|
# Did we get a valid manifest? |
|
if (exists $anvil->data->{manifests}{manifest_name}{$answer}) |
|
{ |
|
$manifest_name = $answer; |
|
$manifest_uuid = $anvil->data->{manifests}{manifest_name}{$answer}{manifest_uuid}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:manifest_name' => $manifest_name, |
|
's2:manifest_uuid' => $manifest_uuid, |
|
}}); |
|
} |
|
elsif (exists $anvil->data->{manifests}{manifest_uuid}{$answer}) |
|
{ |
|
$manifest_name = $anvil->data->{manifests}{manifest_uuid}{$answer}{manifest_name}; |
|
$manifest_uuid = $answer; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:manifest_name' => $manifest_name, |
|
's2:manifest_uuid' => $manifest_uuid, |
|
}}); |
|
} |
|
else |
|
{ |
|
if ($answer) |
|
{ |
|
print "[ Error ] - Your answer: [".$answer."] doesn't match a valid manifest.\n"; |
|
} |
|
print "[ Error ] - Please try again (you can use --manifest <name or UUID> to avoind this prompt).\n"; |
|
$anvil->nice_exit({exit_code => 1}); |
|
} |
|
} |
|
|
|
if (not $as_machine) |
|
{ |
|
print "- Will this node be 'node1' or 'node2'?\n"; |
|
my $answer = <STDIN>; |
|
chomp($answer); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }}); |
|
if ($answer eq "node1") |
|
{ |
|
$as_machine = "node1"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { as_machine => $as_machine }}); |
|
} |
|
elsif ($answer eq "node2") |
|
{ |
|
$as_machine = "node2"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { as_machine => $as_machine }}); |
|
} |
|
else |
|
{ |
|
print "[ Error ] - Please enter 'node1' or 'node2'.\n"; |
|
print "[ Error ] - Please try again (you can use '--as-manchine node{1,2}' to avoind this prompt).\n"; |
|
$anvil->nice_exit({exit_code => 1}); |
|
} |
|
} |
|
|
|
if (not $anvil_uuid) |
|
{ |
|
$anvil_uuid = $anvil->data->{anvils}{anvil_name}{$manifest_name}{anvil_uuid}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); |
|
|
|
if (not $anvil_uuid) |
|
{ |
|
print "[ Error ] - Failed to find an Anvil! UUID for the node: [".$manifest_name."].\n"; |
|
print "[ Error ] - Has the Anvil! been deleted from the database?\n"; |
|
print "[ Error ] - Unable to proceed.\n"; |
|
$anvil->nice_exit({exit_code => 1}); |
|
} |
|
} |
|
|
|
print "\n"; |
|
print "-=] Joining: [".$manifest_name."] as: [".$as_machine."] (Anvil! UUID: [".$anvil_uuid."]\n"; |
|
if ($rebuild) |
|
{ |
|
print "[ Note ] - This is a rebuild, the previous data recorded by this host will be\n"; |
|
print " preserved.\n"; |
|
} |
|
else |
|
{ |
|
print "[ Warning ] - This will replace the old subnode, and all the previous data\n"; |
|
print " associated with it!\n"; |
|
print "[ Warning ] - Be certain the old host will NOT come back! If it does, it can\n"; |
|
print " cause confusion with the Anvil! node!\n"; |
|
} |
|
if (($anvil->data->{switches}{y}) or ($anvil->data->{switches}{yes})) |
|
{ |
|
print "[ Note ] - Confirmed by switch, proceeding.\b"; |
|
} |
|
else |
|
{ |
|
print $anvil->Words->string({key => "message_0021"})." "; |
|
my $answer = <STDIN>; |
|
chomp($answer); |
|
if ($answer !~ /^y/i) |
|
{ |
|
print $anvil->Words->string({key => "message_0022"})."\n"; |
|
$anvil->nice_exit({exit_code => 0}); |
|
} |
|
} |
|
|
|
# If this isn't a rebuild, purge the old host. |
|
if (not $rebuild) |
|
{ |
|
my $node_key = $as_machine eq "node1" ? "anvil_node1_host_uuid" : "anvil_node2_host_uuid"; |
|
my $old_host_uuid = $anvil->data->{anvils}{anvil_name}{$manifest_name}{$node_key} ? $anvil->data->{anvils}{anvil_name}{$manifest_name}{$node_key} : ""; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:node_key' => $node_key, |
|
's2:old_host_uuid' => $old_host_uuid, |
|
}}); |
|
|
|
if ($old_host_uuid) |
|
{ |
|
print "[ Note ] - Purging the old host: [".$old_host_uuid."] from the database.\n"; |
|
print "[ Note ] - Please be patient!\n"; |
|
my $shell_call = $anvil->data->{path}{exe}{'striker-purge-target'}." ".$anvil->Log->switches; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
} |
|
} |
|
|
|
# Register a job. |
|
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ |
|
job_host_uuid => $host_uuid, |
|
job_command => $anvil->data->{path}{exe}{'anvil-join-anvil'}.$anvil->Log->switches, |
|
job_data => "as_machine=".$as_machine.",manifest_uuid=".$manifest_uuid.",anvil_uuid=".$anvil_uuid, |
|
job_name => "join_anvil::".$as_machine, |
|
job_title => "job_0072", |
|
job_description => "job_0073", |
|
job_progress => 0, |
|
}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); |
|
|
|
print "Registering a job. This host should (re)join the Anvil! node shortly.\n"; |
|
|
|
return(0); |
|
} |
|
|
|
# Make sure the hosts file has entries for all nets for both subnodes |
|
sub wait_for_etc_hosts |
|
{ |
|
my ($anvil) = @_; |
|
|
|
my $anvil_uuid = $anvil->data->{sys}{anvil_uuid}; |
|
my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; |
|
my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; |
|
my $manifest_uuid = $anvil->data->{sys}{manifest_uuid}; |
|
my $i_am = $anvil->data->{sys}{machine}; |
|
my $peer_is = $i_am eq "node1" ? "node2" : "node1"; |
|
my $peer_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{"anvil_".$peer_is."_host_uuid"}; |
|
my $peer_short_host_name = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{short_host_name}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:anvil_uuid' => $anvil_uuid, |
|
's2:manifest_uuid' => $manifest_uuid, |
|
's3:node1_host_uuid' => $node1_host_uuid, |
|
's4:node2_host_uuid' => $node2_host_uuid, |
|
's5:i_am' => $i_am, |
|
's6:peer_is' => $peer_is, |
|
's7:peer_host_uuid' => $peer_host_uuid, |
|
's8:peer_short_host_name' => $peer_short_host_name, |
|
}}); |
|
|
|
my $problem = $anvil->Striker->load_manifest({debug => 2, manifest_uuid => $manifest_uuid}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
manifest_uuid => $anvil->data->{sys}{manifest_uuid}, |
|
problem => $problem, |
|
}}); |
|
if ($problem) |
|
{ |
|
# Something went wrong, fatally. Abort the job. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0076", variables => { uuid => $anvil->data->{sys}{manifest_uuid} }}); |
|
update_progress($anvil, 100, "job_0076,!!uuid!".$anvil->data->{sys}{manifest_uuid}."!!"); |
|
$anvil->nice_exit({exit_code => 2}); |
|
} |
|
|
|
# Create the list of host names we need to see in /etc/hosts and the hostnames to use in corosync. |
|
foreach my $machine (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}}) |
|
{ |
|
my $this_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{"anvil_".$machine."_host_uuid"}; |
|
my $this_short_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name}; |
|
my $bcn_name = ""; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { machine => $machine }}); |
|
foreach my $network_name (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}}) |
|
{ |
|
my $host_name = $this_short_host_name.".".$network_name; |
|
my $ip_address = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}{$network_name}{ip}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:network_name' => $network_name, |
|
's2:host_name' => $host_name, |
|
's3:ip_address' => $ip_address, |
|
}}); |
|
|
|
$anvil->data->{networks}{$host_name}{ip_address} = $ip_address; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
"s1:networks::${host_name}::ip_address" => $anvil->data->{networks}{$host_name}{ip_address}, |
|
}}); |
|
|
|
# If this is the first BCN, record it's name (in case BCN1 isn't used here) and IP |
|
# address, and add the short and full host names. |
|
if (($network_name =~ /^bcn/) && (not $bcn_name)) |
|
{ |
|
$bcn_name = $network_name; |
|
my $this_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{host_name}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bcn_name => $bcn_name }}); |
|
|
|
$anvil->data->{networks}{$this_short_host_name}{ip_address} = $ip_address; |
|
$anvil->data->{networks}{$this_host_name}{ip_address} = $ip_address; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
"s1:networks::${this_short_host_name}::ip_address" => $anvil->data->{networks}{$this_short_host_name}{ip_address}, |
|
"s2:networks::${this_host_name}::ip_address" => $anvil->data->{networks}{$this_host_name}{ip_address}, |
|
}}); |
|
} |
|
} |
|
} |
|
|
|
my $waiting = 1; |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0477"); |
|
while($waiting) |
|
{ |
|
# Update the /etc/hosts file. Note that this doesn't add hosts that are not yet trusted, so |
|
# anything not found will be inserted here. |
|
$anvil->System->update_hosts({debug => 2}); |
|
|
|
# Now lets see if all expected hosts names are in the /etc/hosts file yet. |
|
my $ready = 1; |
|
my $hosts_file = $anvil->Storage->read_file({ |
|
file => $anvil->data->{path}{configs}{hosts}, |
|
force_read => 1, |
|
cache => 0, |
|
}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { hosts_file => $hosts_file }}); |
|
|
|
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{networks}}) |
|
{ |
|
my $found = 0; |
|
my $ip_address = $anvil->data->{networks}{$host_name}{ip_address}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:host_name' => $host_name, |
|
's2:ip_address' => $ip_address, |
|
}}); |
|
|
|
foreach my $line (split/\n/, $hosts_file) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); |
|
|
|
$line =~ s/#.*$//; |
|
if ($line =~ /^(\d.*?)\s+(.*)$/) |
|
{ |
|
my $this_ip = $1; |
|
my $hosts = $2; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
this_ip => $this_ip, |
|
hosts => $hosts, |
|
}}); |
|
|
|
if ($anvil->Validate->ip({ip => $this_ip})) |
|
{ |
|
foreach my $this_host (split/\s+/, $hosts) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { this_host => $this_host }}); |
|
|
|
if ($host_name eq $this_host) |
|
{ |
|
my $variables = { |
|
host_name => $host_name, |
|
ip_address => $ip_address, |
|
found_ip => $this_ip, |
|
}; |
|
if ($ip_address eq $this_ip) |
|
{ |
|
# Found it |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0830", variables => $variables }); |
|
$found = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }}); |
|
} |
|
else |
|
{ |
|
# Found the host, but the IP doesn't match. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0831", variables => $variables }); |
|
} |
|
} |
|
last if $found; |
|
} |
|
} |
|
} |
|
last if $found; |
|
} |
|
|
|
if (not $found) |
|
{ |
|
$ready = 0; |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0832", variables => { host_name => $host_name }}); |
|
|
|
# Add the IP to be added to /etc/hosts. |
|
$anvil->data->{to_add}{$host_name} = $ip_address; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "to_add::${host_name}" => $anvil->data->{to_add}{$host_name} }}); |
|
} |
|
} |
|
|
|
if ($ready) |
|
{ |
|
# Ready! |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0833"}); |
|
|
|
$waiting = 0; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); |
|
} |
|
else |
|
{ |
|
# Not ready, wait a bit. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0834"}); |
|
sleep 5; |
|
|
|
# Try to update the /etc/hosts file |
|
$anvil->System->update_hosts({debug => 2}); |
|
sleep 1; |
|
} |
|
} |
|
|
|
return(0); |
|
} |
|
|
|
sub wait_for_access |
|
{ |
|
my ($anvil) = @_; |
|
|
|
# NOTE: This logic is a copy of anvil-safe-start. |
|
$anvil->Database->get_hosts(); |
|
$anvil->Database->get_anvils(); |
|
my $anvil_uuid = $anvil->Cluster->get_anvil_uuid(); |
|
my $host_uuid = $anvil->Get->host_uuid(); |
|
my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; |
|
my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; |
|
my $peer_host_uuid = $host_uuid eq $node1_host_uuid ? $node2_host_uuid : $node1_host_uuid; |
|
my $short_host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name} // ""; |
|
my $peer_short_host_name = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{short_host_name} // ""; |
|
my $peer_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password} // ""; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:anvil_uuid' => $anvil_uuid, |
|
's2:host_uuid' => $host_uuid, |
|
's3:node1_host_uuid' => $node1_host_uuid, |
|
's4:node2_host_uuid' => $node2_host_uuid, |
|
's5:short_host_name' => $short_host_name, |
|
's6:peer_host_uuid' => $peer_host_uuid, |
|
's7:peer_short_host_name' => $peer_short_host_name, |
|
's8:peer_password' => $anvil->Log->is_secure($peer_password), |
|
}}); |
|
|
|
if (not $peer_short_host_name) |
|
{ |
|
die "Peer not found!\n"; |
|
} |
|
|
|
my $waiting = 1; |
|
while ($waiting) |
|
{ |
|
# This will get set back to '1' if |
|
$waiting = 0; |
|
|
|
# Load IPs (again, to catch changes that might be delaying startup) |
|
$anvil->Network->load_ips({ |
|
clear => 1, |
|
host => $short_host_name, |
|
host_uuid => $host_uuid, |
|
|
|
}); |
|
$anvil->Network->load_ips({ |
|
clear => 1, |
|
host => $peer_short_host_name, |
|
host_uuid => $peer_host_uuid, |
|
|
|
}); |
|
|
|
# Loop through our interfaces and then loop our peers. Test access over them and set |
|
# 'waiting' back to '1' if the connection fails. |
|
foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$short_host_name}{interface}}) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
interface => $interface, |
|
waiting => $waiting, |
|
}}); |
|
|
|
# Only care about our networks. |
|
next if $waiting; |
|
if (not $anvil->Network->is_our_interface({interface => $interface})) |
|
{ |
|
# Not an interface we care about |
|
next; |
|
} |
|
|
|
my $this_network = ($interface =~ /^(.*?)_/)[0]; |
|
my $ip_address = $anvil->data->{network}{$short_host_name}{interface}{$interface}{ip}; |
|
my $subnet_mask = $anvil->data->{network}{$short_host_name}{interface}{$interface}{subnet_mask}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:this_network' => $this_network, |
|
's2:ip_address' => $ip_address, |
|
's3:subnet_mask' => $subnet_mask, |
|
}}); |
|
|
|
### NOTE: I know I could match interface names, but that's not certain enough. It's |
|
### possible (if unlikely) that the network name+numbre differs on our peer. So |
|
### this is safer. |
|
# Loop through my peer's interfaces and see if we're sharing this one. |
|
my $local_network = NetAddr::IP->new($ip_address."/".$subnet_mask); |
|
my $peer_match_found = 0; |
|
foreach my $peer_interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$peer_short_host_name}{interface}}) |
|
{ |
|
last if $peer_match_found; |
|
my $peer_ip_address = $anvil->data->{network}{$peer_short_host_name}{interface}{$peer_interface}{ip}; |
|
my $peer_subnet_mask = $anvil->data->{network}{$peer_short_host_name}{interface}{$peer_interface}{subnet_mask}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
peer_interface => $peer_interface, |
|
peer_ip_address => $peer_ip_address, |
|
peer_subnet_mask => $peer_subnet_mask, |
|
}}); |
|
|
|
# This the matching network? |
|
next if $subnet_mask ne $peer_subnet_mask; |
|
|
|
my $peer_network = NetAddr::IP->new($peer_ip_address."/".$peer_subnet_mask); |
|
if ($peer_network->within($local_network)) |
|
{ |
|
# Match, test access. |
|
$peer_match_found = 1; |
|
my $access = $anvil->Remote->test_access({ |
|
target => $peer_ip_address, |
|
password => $peer_password, |
|
}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }}); |
|
if ($access) |
|
{ |
|
# This network is good. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0604", variables => { |
|
peer => $peer_short_host_name, |
|
network => $this_network, |
|
peer_ip => $peer_ip_address, |
|
}}); |
|
|
|
$anvil->data->{sys}{peer_target_ip} = $peer_ip_address; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
"sys::peer_target_ip" => $anvil->data->{sys}{peer_target_ip}, |
|
}}); |
|
} |
|
else |
|
{ |
|
# No access, wait and try it again. |
|
$waiting = 1; |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "log_0605", variables => { |
|
peer => $peer_short_host_name, |
|
network => $this_network, |
|
peer_ip => $peer_ip_address, |
|
}}); |
|
} |
|
} |
|
|
|
} |
|
} |
|
|
|
if ($waiting) |
|
{ |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "log_0606", variables => { peer => $peer_short_host_name }}); |
|
sleep 5; |
|
} |
|
} |
|
|
|
# All networks are up. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "log_0607", variables => { peer => $peer_short_host_name }}); |
|
|
|
return(0); |
|
} |
|
|
|
# Configure DRBD |
|
sub configure_drbd |
|
{ |
|
my ($anvil) = @_; |
|
|
|
### TODO: See if there is a hardware RAID controller and, if so, auto-enable |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0126"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0126"}); |
|
my $updated = $anvil->DRBD->update_global_common({ |
|
usage_count => $anvil->data->{sys}{privacy}{strong} ? 0 : 1, |
|
}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { updated => $updated }}); |
|
if ($updated) |
|
{ |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0127"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0127"}); |
|
} |
|
else |
|
{ |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0128"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0128"}); |
|
} |
|
|
|
# Record that we're done configuring pacemaker. |
|
$anvil->Database->insert_or_update_variables({ |
|
variable_name => "system::drbd::configured", |
|
variable_value => 1, |
|
variable_default => "", |
|
variable_description => "striker_0296", |
|
variable_section => "system", |
|
variable_source_uuid => $anvil->data->{sys}{host_uuid}, |
|
variable_source_table => "hosts", |
|
}); |
|
|
|
return(0); |
|
} |
|
|
|
# Update the passwords for user accounts. |
|
sub update_passwords |
|
{ |
|
my ($anvil) = @_; |
|
|
|
my $machine = $anvil->data->{sys}{machine}; |
|
my $manifest_uuid = $anvil->data->{sys}{manifest_uuid}; |
|
my $anvil_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{name}; |
|
my $anvil_uuid = $anvil->data->{sys}{anvil_uuid}; |
|
my $new_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password}; |
|
my $users = $machine =~ /dr\d+/ ? ["root", "admin"] : ["root", "admin", "hacluster"]; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { |
|
machine => $machine, |
|
manifest_uuid => $manifest_uuid, |
|
anvil_name => $anvil_name, |
|
anvil_uuid => $anvil_uuid, |
|
new_password => $anvil->Log->is_secure($new_password), |
|
}}); |
|
|
|
foreach my $user (@{$users}) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { user => $user }}); |
|
|
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0093,!!user!".$user."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0093", variables => { user => $user }}); |
|
my ($return_code) = $anvil->System->change_shell_user_password({ |
|
debug => 3, |
|
user => $user, |
|
new_password => $new_password, |
|
}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }}); |
|
} |
|
|
|
return(0); |
|
} |
|
|
|
|
|
# (wait for our peer and) Configure pacemaker. If this is a DR host, this is skipped. |
|
sub configure_pacemaker |
|
{ |
|
my ($anvil) = @_; |
|
|
|
my $machine = $anvil->data->{sys}{machine}; |
|
my $manifest_uuid = $anvil->data->{sys}{manifest_uuid}; |
|
|
|
### TODO: Move these to variables in the 'sys' hash |
|
my $anvil_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{name}; |
|
my $anvil_uuid = $anvil->data->{sys}{anvil_uuid}; |
|
my $host_name = $anvil->Get->host_name; |
|
my $new_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password}; |
|
my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; |
|
my $node1_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name}; |
|
$node1_host_name =~ s/\..*$//; |
|
my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; |
|
my $node2_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name}; |
|
$node2_host_name =~ s/\..*$//; |
|
my $peer_host_name = $anvil->Get->host_uuid() eq $node1_host_uuid ? $node2_host_name : $node1_host_name; |
|
my $peer_host_uuid = $anvil->Get->host_uuid() eq $node1_host_uuid ? $node2_host_uuid : $node1_host_uuid; |
|
my $escaped_password = shell_quote($new_password); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
machine => $machine, |
|
anvil_uuid => $anvil_uuid, |
|
anvil_name => $anvil_name, |
|
host_name => $host_name, |
|
manifest_uuid => $manifest_uuid, |
|
node1_host_uuid => $node1_host_uuid, |
|
node1_host_name => $node1_host_name, |
|
node2_host_uuid => $node2_host_uuid, |
|
node2_host_name => $node2_host_name, |
|
peer_host_name => $peer_host_name, |
|
peer_host_uuid => $peer_host_uuid, |
|
new_password => $anvil->Log->is_secure($new_password), |
|
escaped_password => $anvil->Log->is_secure($escaped_password), |
|
}}); |
|
|
|
# If this is a DR box, we don't use pacemaker. |
|
if ($machine =~ /dr\d+/) |
|
{ |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 10), "job_0096"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0096"}); |
|
return(0); |
|
} |
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { 'path::configs::corosync-authkey' => $anvil->data->{path}{configs}{'corosync-authkey'} }}); |
|
if (not -f $anvil->data->{path}{configs}{'corosync-authkey'}) |
|
{ |
|
# In case we're rebuilding, see if the peer already has the '/etc/corosync/authkey' file. |
|
my $corosync_authkey = $anvil->Storage->read_file({ |
|
file => $anvil->data->{path}{configs}{'corosync-authkey'}, |
|
target => $peer_host_name, |
|
}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { corosync_authkey => $corosync_authkey }}); |
|
if ($corosync_authkey ne "!!error!!") |
|
{ |
|
# Write the file out. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0485"}); |
|
$anvil->Storage->write_file({ |
|
debug => 2, |
|
body => $corosync_authkey, |
|
file => $anvil->data->{path}{configs}{'corosync-authkey'}, |
|
user => "root", |
|
group => "root", |
|
mode => "0400", |
|
}); |
|
} |
|
} |
|
|
|
### Run on both nodes. |
|
# Enable pcsd and start the pcsd daemon. |
|
my ($return_code) = $anvil->System->enable_daemon({daemon => "pcsd.service"}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }}); |
|
$return_code = undef; |
|
($return_code) = $anvil->System->start_daemon({daemon => "pcsd.service"}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }}); |
|
$return_code = undef; |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0094,!!daemon!pcsd.service!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "pcsd.service" }}); |
|
|
|
# Enable libvirtd and start the libvirtd daemon. |
|
($return_code) = $anvil->System->enable_daemon({daemon => "libvirtd.service"}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }}); |
|
$return_code = undef; |
|
($return_code) = $anvil->System->start_daemon({daemon => "libvirtd.service"}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }}); |
|
$return_code = undef; |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0094,!!daemon!libvirtd.service!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "libvirtd.service" }}); |
|
|
|
# Disabled and stop the drbd daemon. |
|
($return_code) = $anvil->System->disable_daemon({daemon => "drbd.service"}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }}); |
|
$return_code = undef; |
|
($return_code) = $anvil->System->stop_daemon({daemon => "drbd.service"}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }}); |
|
$return_code = undef; |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!drbd.service!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "drbd.service" }}); |
|
|
|
# Disabled and stop the ksm and ksmtuned daemon. |
|
($return_code) = $anvil->System->disable_daemon({daemon => "ksm.service"}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }}); |
|
$return_code = undef; |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "ksm.service" }}); |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!ksm.service!!"); |
|
|
|
($return_code) = $anvil->System->stop_daemon({daemon => "ksmtuned.service"}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }}); |
|
$return_code = undef; |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "ksmtuned.service" }}); |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!ksmtuned.service!!"); |
|
|
|
# If there is no corosync.conf, see if the peer has it. If so, copy it. If not, we'll initialize the |
|
# cluster shortly. |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { 'path::configs::corosync.conf' => $anvil->data->{path}{configs}{'corosync.conf'} }}); |
|
if (not -e $anvil->data->{path}{configs}{'corosync.conf'}) |
|
{ |
|
my $corosync_conf = $anvil->Storage->read_file({ |
|
file => $anvil->data->{path}{configs}{'corosync.conf'}, |
|
target => $peer_host_name, |
|
}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { corosync_conf => $corosync_conf }}); |
|
if ($corosync_conf ne "!!error!!") |
|
{ |
|
# Write the file out. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0100"}); |
|
$anvil->Storage->write_file({ |
|
debug => 2, |
|
body => $corosync_conf, |
|
file => $anvil->data->{path}{configs}{'corosync.conf'}, |
|
user => "root", |
|
group => "root", |
|
mode => "0644", |
|
}); |
|
} |
|
|
|
# Restart corosync |
|
my ($return_code) = $anvil->System->restart_daemon({daemon => "corosync.service"}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }}); |
|
$return_code = undef; |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "corosync.service" }}); |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!corosync.service!!"); |
|
} |
|
|
|
# Node 1 initializes, node 2 waits. |
|
if ($machine eq "node2") |
|
{ |
|
my $start_time = 0; |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0103"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0103"}); |
|
|
|
# We loop until the peer finishes or the peer's job hit's 100. |
|
my $tried_starting = 0; |
|
my $both_online = 0; |
|
my $try_starting = time + 30; |
|
my $delay = 5; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
delay => $delay, |
|
try_starting => $try_starting, |
|
}}); |
|
until($both_online) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
'path::configs::corosync.conf' => $anvil->data->{path}{configs}{'corosync.conf'}, |
|
}}); |
|
if (-e $anvil->data->{path}{configs}{'corosync.conf'}) |
|
{ |
|
if (not $start_time) |
|
{ |
|
# Corosync is configured, we'll wait up to two minutes and then try |
|
# joining the cluster ourselves. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0108"}); |
|
$start_time = time + 120; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { start_time => $start_time }}); |
|
} |
|
elsif ((time > $start_time) && (not $tried_starting)) |
|
{ |
|
# We've waited a minute, time to try starting the cluster. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0106"}); |
|
my $cluster_started = $anvil->Cluster->start_cluster({debug => 2, all => 1}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cluster_started => $cluster_started }}); |
|
|
|
# Mark that weve tried to start. |
|
$tried_starting = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { tried_starting => $tried_starting }}); |
|
} |
|
|
|
my $problem = $anvil->Cluster->parse_cib({debug => 2}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); |
|
if (not $problem) |
|
{ |
|
# See if both nodes are online. |
|
my $node1_ready = $anvil->Cluster->check_node_status({debug => 2, node_name => $node1_host_name}); |
|
my $node2_ready = $anvil->Cluster->check_node_status({debug => 2, node_name => $node2_host_name}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
node1_ready => $node1_ready, |
|
node2_ready => $node2_ready, |
|
}}); |
|
if (($node1_ready) && ($node2_ready)) |
|
{ |
|
$both_online = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { both_online => $both_online }}); |
|
|
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0104"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0104"}); |
|
} |
|
else |
|
{ |
|
# Not online yet, wait a bit. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0105", variables => { |
|
node1_name => $node1_host_name, |
|
node1_ready => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm}, |
|
node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd}, |
|
node1_crmd => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{'join'}, |
|
node1_join => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{ready}, |
|
node2_name => $node2_host_name, |
|
node2_ready => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{in_ccm}, |
|
node2_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{crmd}, |
|
node2_crmd => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{'join'}, |
|
node2_join => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{ready}, |
|
}}); |
|
} |
|
} |
|
} |
|
else |
|
{ |
|
# corosync.conf doesn't exist yet. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0107"}); |
|
} |
|
if (not $both_online) |
|
{ |
|
if (time > $try_starting) |
|
{ |
|
# Try starting pacemaker, in case we're rebuilding. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0164"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0164"}); |
|
|
|
$try_starting += 60; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { try_starting => $try_starting }}); |
|
|
|
my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster start"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
} |
|
else |
|
{ |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0816", variables => { delay => $delay }}); |
|
sleep $delay; |
|
} |
|
} |
|
} |
|
|
|
### TODO: Left off here; parse 'pcs cluster status' and if there's 'Unable to authenticate', re-auth. |
|
my $auth = 0; |
|
my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster status"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
|
|
my $in_pcsd = 0; |
|
foreach my $line (split/\n/, $output) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); |
|
if ($in_pcsd) |
|
{ |
|
if ($line =~ /Unable to authenticate/i) |
|
{ |
|
$auth = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { auth => $auth }}); |
|
} |
|
} |
|
if ($line =~ /PCSD Status/i) |
|
{ |
|
$in_pcsd = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_pcsd => $in_pcsd }}); |
|
} |
|
last if $auth; |
|
} |
|
if ($auth) |
|
{ |
|
# Also, re-auth. We need to run this on both hosts. |
|
my $shell_call = $anvil->data->{path}{exe}{pcs}." host auth ".$node1_host_name." ".$node2_host_name." -u hacluster -p ".$escaped_password; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({secure => 1, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
} |
|
} |
|
else |
|
{ |
|
# We're node 1, proceed with cluster setup. |
|
my $waiting = 1; |
|
my $warning_printed = 0; |
|
while($waiting) |
|
{ |
|
# Try to authenticate against the peer. |
|
my $auth_shell_call = $anvil->data->{path}{exe}{pcs}." host auth ".$node1_host_name." ".$node2_host_name." -u hacluster -p ".$escaped_password; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { auth_shell_call => $auth_shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, secure => 1, shell_call => $auth_shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
if ($return_code) |
|
{ |
|
# Something went wrong. |
|
if (not $warning_printed) |
|
{ |
|
# Update the job |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0097"); |
|
$warning_printed = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { warning_printed => $warning_printed }}); |
|
} |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0097"}); |
|
sleep 5; |
|
} |
|
else |
|
{ |
|
# We're good. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0098"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0098"}); |
|
$waiting = 0; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); |
|
} |
|
} |
|
|
|
# If there is no corosync.conf, see if the peer has it. If so, copy it. If not, initialize |
|
# the cluster. |
|
if (not -e $anvil->data->{path}{configs}{'corosync.conf'}) |
|
{ |
|
# There's no cluster yet, initialize it. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0099,!!anvil_name!".$anvil_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0099", variables => { anvil_name => $anvil_name }}); |
|
|
|
my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster setup ".$anvil_name." ".$node1_host_name." ".$node2_host_name; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
if ($return_code) |
|
{ |
|
# Something went wrong |
|
update_progress($anvil, 100, "job_0101,!!error!".$output."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0101", variables => { error => $output }}); |
|
$anvil->nice_exit({exit_code => 5}); |
|
} |
|
} |
|
|
|
# If we can parse the CIB, then pcsd is running. |
|
my $problem = $anvil->Cluster->parse_cib({debug => 2}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); |
|
if ($problem) |
|
{ |
|
# Start the cluster. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0102"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0102"}); |
|
|
|
my $cluster_started = $anvil->Cluster->start_cluster({debug => 2, all => 1}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cluster_started => $cluster_started }}); |
|
} |
|
|
|
# Now wait for both nodes to come online. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0109"); |
|
my $both_online = 0; |
|
my $start_again = time + 60; |
|
until ($both_online) |
|
{ |
|
### TODO: If we're waiting more that five minutes, call 'pcs cluster start --all' again. |
|
my $problem = $anvil->Cluster->parse_cib({debug => 2}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); |
|
if (not $problem) |
|
{ |
|
# See if both nodes are online. |
|
my $node1_ready = $anvil->Cluster->check_node_status({node_name => $node1_host_name}); |
|
my $node2_ready = $anvil->Cluster->check_node_status({node_name => $node2_host_name}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
node1_ready => $node1_ready, |
|
node2_ready => $node2_ready, |
|
}}); |
|
if (($node1_ready) && ($node2_ready)) |
|
{ |
|
$both_online = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { both_online => $both_online }}); |
|
|
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0104"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0104"}); |
|
} |
|
else |
|
{ |
|
# Not online yet, wait a bit. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0105", variables => { |
|
node1_name => $node1_host_name, |
|
node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm}, |
|
node1_crmd => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd}, |
|
node1_join => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{'join'}, |
|
node1_ready => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{ready}, |
|
node2_name => $node2_host_name, |
|
node2_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{in_ccm}, |
|
node2_crmd => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{crmd}, |
|
node2_join => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{'join'}, |
|
node2_ready => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{ready}, |
|
}}); |
|
} |
|
} |
|
if (time > $start_again) |
|
{ |
|
### NOTE: We can't just call 'start --all' again anymore. Now we need to |
|
### stop -> start. Before we do this, make sure there are no servers |
|
### running. |
|
$start_again = time + 60; |
|
my $restart = 1; |
|
my $server_count = keys %{$anvil->data->{cib}{parsed}{data}{server}}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
start_again => $start_again, |
|
server_count => $server_count, |
|
}}); |
|
foreach my $server (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{server}}) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
"cib::parsed::data::server::${server}::active" => $anvil->data->{cib}{parsed}{data}{server}{$server}{active}, |
|
}}); |
|
if ($anvil->data->{cib}{parsed}{data}{server}{$server}{active}) |
|
{ |
|
$restart = 0; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { restart => $restart }}); |
|
} |
|
} |
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { restart => $restart }}); |
|
if ($restart) |
|
{ |
|
# Call cluster start again. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0272"}); |
|
my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster stop --all"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
start_again => $start_again, |
|
shell_call => $shell_call, |
|
}}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
|
|
$shell_call = $anvil->data->{path}{exe}{pcs}." cluster start --all"; |
|
($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
} |
|
} |
|
sleep 5 if not $both_online; |
|
} |
|
|
|
### Setup fencing! |
|
$anvil->Striker->get_fence_data(); |
|
|
|
# IPMI first, if applicable. |
|
my $something_changed = {}; |
|
my $fence_order = {}; |
|
my $node1_use_delay = 0; |
|
my $node2_use_delay = 0; |
|
my $fence_devices = {}; |
|
foreach my $node ("node1", "node2") |
|
{ |
|
my $node_name = $node eq "node1" ? $node1_host_name : $node2_host_name; |
|
my $host_uuid = $node eq "node1" ? $node1_host_uuid : $node2_host_uuid; |
|
my $host_ipmi = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi}; |
|
my $ipmi_stonith_name = "ipmilan_".$node; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
node_name => $node_name, |
|
host_uuid => $host_uuid, |
|
host_ipmi => $host_ipmi =~ /passw/ ? $anvil->Log->is_secure($host_ipmi) : $host_ipmi, |
|
ipmi_stonith_name => $ipmi_stonith_name, |
|
}}); |
|
|
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0115,!!node!".$node_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0115", variables => { node => $node_name }}); |
|
|
|
# This will store the fence level order. If something changes |
|
$fence_order->{$node_name} = []; |
|
|
|
# This will switch to '1' if something changed, triggering a reconfig of the fencing levels. |
|
$something_changed->{$node_name} = 0; |
|
|
|
# Does this stonith method already exist? |
|
my $create_entry = 0; |
|
my $delete_old = 0; |
|
my $pcs_add_command = ""; |
|
if ($host_ipmi) |
|
{ |
|
push @{$fence_order->{$node_name}}, "fence_ipmilan"; |
|
$fence_devices->{$node_name}{fence_ipmilan} = [$ipmi_stonith_name]; |
|
|
|
# The --action switch needs to be 'pcmk_off_action' in pcs, so we convert it here. |
|
$host_ipmi =~ s/--action status//; |
|
$host_ipmi =~ s/--action/--pcmk_off_action/; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
host_ipmi => $host_ipmi =~ /passw/ ? $anvil->Log->is_secure($host_ipmi) : $host_ipmi, |
|
}}); |
|
|
|
# We have IPMI, so we also want fence_delay for this node. |
|
if ($node eq "node1") |
|
{ |
|
$node1_use_delay = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_use_delay => $node1_use_delay }}); |
|
} |
|
else |
|
{ |
|
$node2_use_delay = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_use_delay => $node2_use_delay }}); |
|
} |
|
|
|
# If we're here, break up the command and turn it into the pcs call. |
|
my $old_switches = {}; |
|
my ($fence_agent, $arguments) = ($host_ipmi =~ /^\/.*\/(.*?)\s+(.*)$/); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
fence_agent => $fence_agent, |
|
arguments => $arguments =~ /passw/ ? $anvil->Log->is_secure($arguments) : $arguments, |
|
}}); |
|
|
|
$pcs_add_command = $anvil->data->{path}{exe}{pcs}." stonith create ".$ipmi_stonith_name." ".$fence_agent." pcmk_host_list=\"".$node_name."\" "; |
|
my $switches = $anvil->System->parse_arguments({debug => 2, arguments => $arguments}); |
|
foreach my $switch (sort {$a cmp $b} keys %{$switches}) |
|
{ |
|
# Ignore 'delay', we handle that in Cluster->set_delay(); Also, |
|
# convert '#!SET!#' to 'true'. |
|
my $value = $switches->{$switch}; |
|
$value =~ s/"/\\"/g; |
|
$value =~ s/#!SET!#/true/g; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
switch => $switch, |
|
value => $value, |
|
}}); |
|
next if $anvil->data->{fence_data}{$fence_agent}{switch}{$switch}{name} eq "delay"; |
|
next if $anvil->data->{fence_data}{$fence_agent}{switch}{$switch}{name} eq "action"; |
|
|
|
# Find the argument=value version. |
|
my $argument = $anvil->data->{fence_data}{$fence_agent}{switch}{$switch}{name}; |
|
$pcs_add_command .= $argument."=\"".$value."\" "; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
argument => $argument, |
|
value => $argument =~ /passw/ ? $anvil->Log->is_secure($value) : $value, |
|
pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, |
|
}}); |
|
|
|
# Store this to see if it's different from what's already in the CIB. |
|
$old_switches->{$argument} = $value; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
"old_switches->{$argument}" => $old_switches->{$argument}, |
|
}}); |
|
} |
|
$pcs_add_command .= "op monitor interval=\"60\""; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, |
|
}}); |
|
|
|
# If there's an entry in the CIB, see if it's different somehow |
|
if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name}) |
|
{ |
|
foreach my $argument (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name}{argument}}) |
|
{ |
|
next if $argument eq "delay"; |
|
next if $argument eq "action"; |
|
my $old_entry = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name}{argument}{$argument}{value}; |
|
my $new_entry = exists $old_switches->{$argument} ? $old_switches->{$argument} : ""; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:argument' => $argument, |
|
's2:old_entry' => $old_entry, |
|
's3:new_entry' => $new_entry, |
|
}}); |
|
|
|
if ($old_entry ne $new_entry) |
|
{ |
|
# Changed, delete and recreate. |
|
$delete_old = 1; |
|
$create_entry = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
delete_old => $delete_old, |
|
create_entry => $create_entry, |
|
}}); |
|
|
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0117"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0117"}); |
|
last; |
|
} |
|
|
|
# Delete the old switch. |
|
delete $old_switches->{$argument}; |
|
} |
|
|
|
# Are there any old switches left? |
|
my $old_switch_count = keys %{$old_switches}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
delete_old => $delete_old, |
|
old_switch_count => $old_switch_count, |
|
}}); |
|
if ((not $delete_old) && ($old_switch_count)) |
|
{ |
|
# Delete and recreate. |
|
$delete_old = 1; |
|
$create_entry = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
delete_old => $delete_old, |
|
create_entry => $create_entry, |
|
}}); |
|
|
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0117"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0117"}); |
|
} |
|
} |
|
else |
|
{ |
|
# No existing entry, add a new one. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0116"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0116"}); |
|
|
|
$create_entry = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_entry => $create_entry }}); |
|
} |
|
} |
|
elsif (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name}) |
|
{ |
|
# There was an existing fence config, but there's no entry in 'host_ipmi'. |
|
# Remove the stonith entry. |
|
$delete_old = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { delete_old => $delete_old }}); |
|
|
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0118"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0118"}); |
|
} |
|
|
|
# Process the IPMI entry. |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
delete_old => $delete_old, |
|
create_entry => $create_entry, |
|
}}); |
|
if ($delete_old) |
|
{ |
|
# Delete |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0119,!!device!".$ipmi_stonith_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0119", variables => { device => $ipmi_stonith_name }}); |
|
|
|
my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith delete ".$ipmi_stonith_name; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
if ($return_code) |
|
{ |
|
# Something went wrong. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => { |
|
shell_call => $shell_call, |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!"); |
|
sleep 2; |
|
$anvil->nice_exit({exit_code => 6}); |
|
} |
|
|
|
$something_changed->{$node_name} = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }}); |
|
} |
|
if ($create_entry) |
|
{ |
|
# Create. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0120,!!device!".$ipmi_stonith_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0120", variables => { device => $ipmi_stonith_name }}); |
|
|
|
my $shell_call = $pcs_add_command; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
if ($return_code) |
|
{ |
|
# If this is because the method already exists, ignore the error. |
|
if ($output =~ /already exists/i) |
|
{ |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "warning_0174", variables => { |
|
method => $ipmi_stonith_name, |
|
}}); |
|
} |
|
else |
|
{ |
|
# Something went wrong. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => { |
|
shell_call => $shell_call, |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!"); |
|
sleep 2; |
|
$anvil->nice_exit({exit_code => 6}); |
|
} |
|
} |
|
|
|
$something_changed->{$node_name} = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }}); |
|
} |
|
|
|
|
|
### Now any other fence devices. |
|
foreach my $device (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$node}{fence}}) |
|
{ |
|
my $delete_old = 0; |
|
my $create_entry = 0; |
|
my $old_switches = {}; |
|
my $fence_uuid = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{fences}{$device}{uuid}; |
|
my $fence_name = $anvil->data->{fences}{fence_uuid}{$fence_uuid}{fence_name}; |
|
my $fence_arguments = $anvil->data->{fences}{fence_uuid}{$fence_uuid}{fence_arguments}; |
|
my $fence_agent = $anvil->data->{fences}{fence_uuid}{$fence_uuid}{fence_agent}; |
|
my $stonith_name = ($fence_agent =~ /^fence_(.*)$/)[0]."_".$node."_".$fence_name; |
|
my $port = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$node}{fence}{$device}{port}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
device => $device, |
|
fence_uuid => $fence_uuid, |
|
fence_name => $fence_name, |
|
fence_arguments => $fence_arguments =~ /passw/ ? $anvil->Log->is_secure($fence_arguments) : $fence_arguments, |
|
stonith_name => $stonith_name, |
|
port => $port, |
|
}}); |
|
|
|
# We use this to tell if there are two or more entries per agent. If there |
|
# are, we link them later when setting up the fence levels. |
|
if (not exists $fence_devices->{$node_name}{$fence_agent}) |
|
{ |
|
push @{$fence_order->{$node_name}}, $fence_agent; |
|
$fence_devices->{$node_name}{$fence_agent} = []; |
|
} |
|
push @{$fence_devices->{$node_name}{$fence_agent}}, $stonith_name; |
|
|
|
# Fence arguments use 'action', but pcs deprecated it in favour of 'pcmk_off_action', so rename it. |
|
$fence_arguments =~ s/action=/pcmk_off_action=/; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
fence_arguments => $fence_arguments =~ /passw/ ? $anvil->Log->is_secure($fence_arguments) : $fence_arguments, |
|
}}); |
|
|
|
# Build the pcs command |
|
my $pcs_add_command = $anvil->data->{path}{exe}{pcs}." stonith create ".$stonith_name." ".$fence_agent." pcmk_host_list=\"".$node_name."\" ".$fence_arguments." "; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, |
|
}}); |
|
while ($fence_arguments =~ /=/) |
|
{ |
|
# Ignore 'delay', we handle that in Cluster->set_delay(); |
|
my $pair = ($fence_arguments =~ /(\S*?=".*?")/)[0]; |
|
$fence_arguments =~ s/$pair//; |
|
$fence_arguments =~ s/^\s+//; |
|
$fence_arguments =~ s/\s+$//; |
|
my ($argument, $value) = ($pair =~ /(.*)="(.*)"/); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:fence_arguments' => $fence_arguments, |
|
's2:pair' => $pair =~ /passw/ ? $anvil->Log->is_secure($pair) : $pair, |
|
's3:argument' => $argument, |
|
's4:value' => $argument =~ /passw/ ? $anvil->Log->is_secure($value) : $value, |
|
}}); |
|
|
|
# Ignore 'delay', we handle that in Cluster->set_delay(); |
|
if (($argument ne "pcmk_off_action") && |
|
(exists $anvil->data->{fence_data}{$fence_agent}{switch}{$argument}{name}) && |
|
($anvil->data->{fence_data}{$fence_agent}{switch}{$argument}{name} eq "delay")) |
|
{ |
|
next; |
|
} |
|
|
|
# Store this to see if it's different from what's already in the CIB. |
|
$old_switches->{$argument} = $value; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
"old_switches->{$argument}" => $old_switches->{$argument}, |
|
}}); |
|
} |
|
if ($port) |
|
{ |
|
$port =~ s/"/\\"/g; |
|
$pcs_add_command .= "port=\"".$port."\" "; |
|
$old_switches->{port} = $port; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, |
|
"old_switches->{port}" => $old_switches->{port}, |
|
}}); |
|
} |
|
$pcs_add_command .= "op monitor interval=\"60\""; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, |
|
}}); |
|
|
|
# Does this device exist already? |
|
if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name}) |
|
{ |
|
foreach my $argument (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name}{argument}}) |
|
{ |
|
next if $argument eq "delay"; |
|
my $old_entry = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name}{argument}{$argument}{value}; |
|
my $new_entry = exists $old_switches->{$argument} ? $old_switches->{$argument} : ""; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:argument' => $argument, |
|
's2:old_entry' => $old_entry, |
|
's3:new_entry' => $new_entry, |
|
}}); |
|
|
|
if ($old_entry ne $new_entry) |
|
{ |
|
# Changed, delete and recreate. |
|
$delete_old = 1; |
|
$create_entry = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
delete_old => $delete_old, |
|
create_entry => $create_entry, |
|
}}); |
|
|
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0121,!!device!".$stonith_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0121", variables => { device => $stonith_name }}); |
|
last; |
|
} |
|
|
|
# Delete the old switch. |
|
delete $old_switches->{$argument}; |
|
} |
|
|
|
# Are there any old switches left? |
|
my $old_switch_count = keys %{$old_switches}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_switch_count => $old_switch_count }}); |
|
if ((not $delete_old) && ($old_switch_count)) |
|
{ |
|
# Delete and recreate. |
|
$delete_old = 1; |
|
$create_entry = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
delete_old => $delete_old, |
|
create_entry => $create_entry, |
|
}}); |
|
|
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0121,!!device!".$stonith_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0121", variables => { device => $stonith_name }}); |
|
} |
|
} |
|
else |
|
{ |
|
# No existing entry, add a new one. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0122,!!device!".$stonith_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0122", variables => { device => $stonith_name }}); |
|
|
|
$create_entry = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_entry => $create_entry }}); |
|
} |
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
delete_old => $delete_old, |
|
create_entry => $create_entry, |
|
}}); |
|
if ($delete_old) |
|
{ |
|
# Delete |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0119,!!device!".$stonith_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0119", variables => { device => $stonith_name }}); |
|
|
|
my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith delete ".$stonith_name; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
if ($return_code) |
|
{ |
|
# Something went wrong. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => { |
|
shell_call => $shell_call, |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!"); |
|
sleep 2; |
|
$anvil->nice_exit({exit_code => 6}); |
|
} |
|
|
|
$something_changed->{$node_name} = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }}); |
|
} |
|
if ($create_entry) |
|
{ |
|
# Create. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0120,!!device!".$stonith_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0120", variables => { device => $stonith_name }}); |
|
|
|
my $shell_call = $pcs_add_command; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
if ($return_code) |
|
{ |
|
# See if this was because the fence method already existed. |
|
if ($output =~ /already exists/) |
|
{ |
|
# It already existed, we're fine. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0481,!!device!".$stonith_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0120", variables => { device => $stonith_name }}); |
|
} |
|
else |
|
{ |
|
# Something went wrong. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => { |
|
shell_call => $shell_call, |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!"); |
|
sleep 2; |
|
$anvil->nice_exit({exit_code => 6}); |
|
} |
|
} |
|
else |
|
{ |
|
$something_changed->{$node_name} = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }}); |
|
} |
|
} |
|
} |
|
|
|
|
|
### If we had a fence_ipmilan entry, add a 'fence_delay' entry, if needed. |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
node => $node, |
|
node1_use_delay => $node1_use_delay, |
|
node2_use_delay => $node2_use_delay, |
|
}}); |
|
if ((($node eq "node1") && ($node1_use_delay)) or |
|
(($node eq "node2") && ($node2_use_delay))) |
|
{ |
|
my $stonith_name = "delay_".$node; |
|
push @{$fence_order->{$node_name}}, "fence_delay"; |
|
$fence_devices->{$node_name}{fence_delay} = [$stonith_name]; |
|
|
|
# Add the fence delay if it doesn't exist yet. |
|
if (not exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name}) |
|
{ |
|
my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith create ".$stonith_name." fence_delay pcmk_host_list=\"".$node_name."\" wait=\"60\" op monitor interval=\"60\""; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
if ($return_code) |
|
{ |
|
# Something went wrong. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => { |
|
shell_call => $shell_call, |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!"); |
|
sleep 2; |
|
$anvil->nice_exit({exit_code => 6}); |
|
} |
|
} |
|
} |
|
} |
|
|
|
# Setup fence levels. |
|
foreach my $node_name (sort {$a cmp $b} keys %{$fence_order}) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }}); |
|
if ($something_changed->{$node_name}) |
|
{ |
|
# Update our view of the cluster. |
|
my $problem = $anvil->Cluster->parse_cib({debug => 2}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); |
|
|
|
# Delete any existing fence levels |
|
if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}) |
|
{ |
|
foreach my $index (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{order}}) |
|
{ |
|
# pcs stonith level delete <index> <target> |
|
my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith level delete ".$index." ".$node_name; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
if ($return_code) |
|
{ |
|
# Something went wrong. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => { |
|
shell_call => $shell_call, |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!"); |
|
sleep 2; |
|
$anvil->nice_exit({exit_code => 6}); |
|
} |
|
} |
|
} |
|
|
|
# Create the new fence levels |
|
my $i = 1; |
|
foreach my $fence_agent (@{$fence_order->{$node_name}}) |
|
{ |
|
my $devices = ""; |
|
foreach my $device (sort {$a cmp $b} @{$fence_devices->{$node_name}{$fence_agent}}) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { device => $device }}); |
|
$devices .= $device.","; |
|
} |
|
$devices =~ s/,$//; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { devices => $devices }}); |
|
|
|
my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith level add ".$i." ".$node_name." ".$devices; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
if ($return_code) |
|
{ |
|
# Something went wrong. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => { |
|
shell_call => $shell_call, |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!"); |
|
sleep 2; |
|
$anvil->nice_exit({exit_code => 6}); |
|
} |
|
|
|
$i++; |
|
} |
|
} |
|
} |
|
} |
|
|
|
# Make sure logind is update to handle fencing properly |
|
# see - https://access.redhat.com/solutions/1578823 |
|
$anvil->Cluster->configure_logind({debug => 2}); |
|
|
|
# Enable fencing and set the retry to INFINITY, if needed. |
|
$anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'} = "" if not defined $anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
"cib::parsed::data::stonith::max-attempts" => $anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'}, |
|
}}); |
|
if ($anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'} ne "INFINITY") |
|
{ |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0124"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0124"}); |
|
|
|
my $shell_call = $anvil->data->{path}{exe}{pcs}." property set stonith-max-attempts=INFINITY"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
if ($return_code) |
|
{ |
|
# Something went wrong. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => { |
|
shell_call => $shell_call, |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!"); |
|
sleep 2; |
|
$anvil->nice_exit({exit_code => 6}); |
|
} |
|
} |
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
"cib::parsed::data::stonith::enabled" => $anvil->data->{cib}{parsed}{data}{stonith}{enabled}, |
|
}}); |
|
if (not $anvil->data->{cib}{parsed}{data}{stonith}{enabled}) |
|
{ |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0125"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0125"}); |
|
|
|
my $shell_call = $anvil->data->{path}{exe}{pcs}." property set stonith-enabled=true"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
if ($return_code) |
|
{ |
|
# Something went wrong. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => { |
|
shell_call => $shell_call, |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!"); |
|
sleep 2; |
|
$anvil->nice_exit({exit_code => 6}); |
|
} |
|
} |
|
|
|
# Update (if needed) corosync.conf to use the BCN1, MN1 and SN1 as knet networks. |
|
if ($machine eq "node1") |
|
{ |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0344"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0344"}); |
|
|
|
check_corosync($anvil); |
|
} |
|
|
|
# Record that we're done configuring pacemaker. |
|
$anvil->Database->insert_or_update_variables({ |
|
variable_name => "system::pacemaker::configured", |
|
variable_value => 1, |
|
variable_default => "", |
|
variable_description => "striker_0295", |
|
variable_section => "system", |
|
variable_source_uuid => $anvil->data->{sys}{host_uuid}, |
|
variable_source_table => "hosts", |
|
}); |
|
|
|
return(0); |
|
} |
|
|
|
sub check_corosync |
|
{ |
|
my ($anvil) = @_; |
|
|
|
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0345"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0345"}); |
|
my $waiting = 1; |
|
my $anvil_uuid = $anvil->data->{sys}{anvil_uuid}; |
|
my $new_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password}; |
|
while ($waiting) |
|
{ |
|
### TODO: Add MN where appropriate |
|
my $problem = $anvil->Cluster->parse_cib({debug => 3}); |
|
my $peer_ready = $anvil->data->{cib}{parsed}{peer}{ready}; |
|
my $peer_name = $anvil->data->{cib}{parsed}{peer}{name}; |
|
my $peer_short_name = $peer_name; |
|
$peer_short_name =~ s/\..*$//; |
|
my $peer_bcn_name = $peer_short_name.".bcn1"; |
|
my $peer_sn_name = $peer_short_name.".sn1"; |
|
my $local_ready = $anvil->data->{cib}{parsed}{'local'}{ready}; |
|
my $local_name = $anvil->data->{cib}{parsed}{'local'}{name}; |
|
my $local_short_name = $anvil->Get->short_host_name; |
|
my $local_bcn_name = $local_short_name.".bcn1"; |
|
my $local_sn_name = $local_short_name.".sn1"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
problem => $problem, |
|
peer_ready => $peer_ready, |
|
peer_name => $peer_name, |
|
peer_short_name => $peer_short_name, |
|
peer_bcn_name => $peer_bcn_name, |
|
peer_sn_name => $peer_sn_name, |
|
local_ready => $local_ready, |
|
local_name => $local_name, |
|
local_short_name => $local_short_name, |
|
local_bcn_name => $local_bcn_name, |
|
local_sn_name => $local_sn_name, |
|
}}); |
|
if ((not $problem) && ($peer_ready) && ($local_ready)) |
|
{ |
|
update_progress($anvil, $anvil->data->{job}{progress}, "job_0346"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0346"}); |
|
|
|
# Verify we can ping the peer on the BCN and SN. |
|
my $bcn_access = $anvil->Remote->test_access({ |
|
target => $peer_bcn_name, |
|
password => $new_password, |
|
}); |
|
my $sn_access = $anvil->Remote->test_access({ |
|
target => $peer_sn_name, |
|
password => $new_password, |
|
}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
bcn_access => $bcn_access, |
|
sn_access => $sn_access, |
|
}}); |
|
if (($bcn_access) && ($sn_access)) |
|
{ |
|
# We're ready! |
|
update_progress($anvil, ($anvil->data->{job}{progress}+1), "job_0347"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0347"}); |
|
|
|
$waiting = 0; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); |
|
|
|
my $in_totem = 0; |
|
my $token_seen = 0; |
|
my $in_nodelist = 0; |
|
my $in_node = 0; |
|
my $ring0_addr = ""; |
|
my $ring1_addr = ""; |
|
my $in_node_name = ""; |
|
my $nodelist_body = ""; |
|
my $old_corosync_conf = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'corosync.conf'}}); |
|
my $new_corosync_conf = ""; |
|
foreach my $line (split/\n/, $old_corosync_conf) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); |
|
if ($line =~ /totem \{/) |
|
{ |
|
$in_totem = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_totem => $in_totem }}); |
|
|
|
$new_corosync_conf .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }}); |
|
next; |
|
} |
|
if ($line =~ /nodelist \{/) |
|
{ |
|
$in_nodelist = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); |
|
|
|
$new_corosync_conf .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }}); |
|
next; |
|
} |
|
|
|
if ($in_nodelist) |
|
{ |
|
if ($line =~ /node \{/) |
|
{ |
|
$in_node = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_node => $in_node }}); |
|
|
|
$new_corosync_conf .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }}); |
|
next; |
|
} |
|
if ($in_node) |
|
{ |
|
if ($line =~ /name:(.*)$/) |
|
{ |
|
$in_node_name = $anvil->Words->clean_spaces({string => $1}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_node_name => $in_node_name }}); |
|
|
|
$nodelist_body .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }}); |
|
next; |
|
} |
|
elsif ($line =~ /ring0_addr:(.*)$/) |
|
{ |
|
$ring0_addr = $anvil->Words->clean_spaces({string => $1}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ring0_addr => $ring0_addr }}); |
|
|
|
$nodelist_body .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }}); |
|
next; |
|
} |
|
elsif ($line =~ /ring1_addr:(.*)$/) |
|
{ |
|
$ring1_addr = $anvil->Words->clean_spaces({string => $1}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ring1_addr => $ring1_addr }}); |
|
|
|
$nodelist_body .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }}); |
|
next; |
|
} |
|
elsif ($line =~ /}/) |
|
{ |
|
$in_node = 0; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_node => $in_node }}); |
|
|
|
# Is this the local or peer node? |
|
my $test_ring0_addr = $peer_bcn_name; |
|
my $test_ring1_addr = $peer_sn_name; |
|
if (($in_node_name eq $anvil->Get->host_name) or ($in_node_name eq $anvil->Get->short_host_name)) |
|
{ |
|
# It's us |
|
$test_ring0_addr = $local_bcn_name; |
|
$test_ring1_addr = $local_sn_name; |
|
} |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:ring0_addr' => $ring0_addr, |
|
's2:test_ring0_addr' => $test_ring0_addr, |
|
's3:ring1_addr' => $ring1_addr, |
|
's4:test_ring1_addr' => $test_ring1_addr, |
|
}}); |
|
|
|
if ((not $ring1_addr) or ($ring0_addr ne $test_ring0_addr) or ($ring1_addr ne $test_ring1_addr)) |
|
{ |
|
# Rewrite the nodelist body. |
|
my $new_nodelist_body = " ring0_addr: ".$test_ring0_addr."\n"; |
|
$new_nodelist_body .= " ring1_addr: ".$test_ring1_addr."\n"; |
|
foreach my $nodelist_line (split/\n/, $nodelist_body) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_line => $nodelist_line }}); |
|
next if $nodelist_line =~ /ring\d_addr/; |
|
|
|
$new_nodelist_body .= $nodelist_line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_nodelist_body => $new_nodelist_body }}); |
|
} |
|
|
|
$nodelist_body = $new_nodelist_body; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_nodelist_body => $new_nodelist_body }}); |
|
} |
|
|
|
$new_corosync_conf .= $nodelist_body; |
|
$new_corosync_conf .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }}); |
|
|
|
$ring0_addr = ""; |
|
$ring1_addr = ""; |
|
$in_node_name = ""; |
|
$nodelist_body = ""; |
|
next; |
|
} |
|
else |
|
{ |
|
# Normal line, stash it. |
|
$nodelist_body .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }}); |
|
next; |
|
} |
|
} |
|
elsif ($line =~ /}/) |
|
{ |
|
$in_nodelist = 0; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_nodelist => $in_nodelist }}); |
|
|
|
$new_corosync_conf .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }}); |
|
next; |
|
} |
|
} |
|
if ($in_totem) |
|
{ |
|
if ($line =~ /}/) |
|
{ |
|
if (not $token_seen) |
|
{ |
|
$new_corosync_conf .= " token: 10000\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }}); |
|
} |
|
$new_corosync_conf .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }}); |
|
|
|
$in_totem = 0; |
|
$token_seen = 0; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
in_totem => $in_totem, |
|
token_seen => $token_seen, |
|
}}); |
|
next; |
|
} |
|
if ($line =~ /token:/) |
|
{ |
|
$token_seen = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); |
|
} |
|
$new_corosync_conf .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }}); |
|
} |
|
else |
|
{ |
|
# Normal line |
|
$new_corosync_conf .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }}); |
|
} |
|
} |
|
|
|
# Take the last new lines of the file bodies. |
|
$old_corosync_conf =~ s/\n$//g; |
|
$new_corosync_conf =~ s/\n$//g; |
|
|
|
my $difference = diff \$old_corosync_conf, \$new_corosync_conf, { STYLE => 'Unified' }; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }}); |
|
|
|
if ($difference) |
|
{ |
|
# Update the corosync.conf, sync it and reload corosync. |
|
update_progress($anvil, ($anvil->data->{job}{progress}+1), "log_0643,!!difference!".$difference."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0643", variables => { difference => $difference }}); |
|
$anvil->Storage->write_file({ |
|
file => $anvil->data->{path}{configs}{'corosync.conf'}, |
|
body => $new_corosync_conf, |
|
overwrite => 1, |
|
backup => 1, |
|
}); |
|
|
|
# Sync |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0644"}); |
|
my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster sync"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
# RC 0 is OK |
|
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
update_progress($anvil, ($anvil->data->{job}{progress}+1), "job_0348,!!return_code!".$return_code."!!,!!output!".$output."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0348", variables => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
|
|
# Reload |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0645"}); |
|
$shell_call = $anvil->data->{path}{exe}{pcs}." cluster reload corosync"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
|
|
|
# RC 0 is OK |
|
($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
update_progress($anvil, ($anvil->data->{job}{progress}+1), "job_0349,!!return_code!".$return_code."!!,!!output!".$output."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0349", variables => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
} |
|
} |
|
} |
|
|
|
if ($waiting) |
|
{ |
|
sleep 5; |
|
} |
|
} |
|
|
|
return(0); |
|
} |
|
|
|
# Check if we need to change any IPs or our hostname. |
|
sub check_local_network |
|
{ |
|
my ($anvil) = @_; |
|
|
|
# What host name and IP(s) should I have? |
|
my $local_host = $anvil->Get->short_host_name(); |
|
my $machine = $anvil->data->{sys}{machine}; |
|
my $manifest_uuid = $anvil->data->{sys}{manifest_uuid}; |
|
my $domain = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{domain}; |
|
my $old_host_name = $anvil->Get->host_name; |
|
my $new_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{name}; |
|
if ($domain) |
|
{ |
|
$new_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{name}.".".$domain; |
|
} |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's2:domain' => $domain, |
|
's3:old_host_name' => $old_host_name, |
|
's4:new_host_name' => $new_host_name, |
|
}}); |
|
|
|
$anvil->data->{sys}{host_name} = $new_host_name; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
'sys::host_name' => $anvil->data->{sys}{host_name}, |
|
}}); |
|
|
|
# If the hostname isn't the same, change it. |
|
if ($old_host_name ne $new_host_name) |
|
{ |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0061,!!host_name!".$new_host_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0061", variables => { host_name => $new_host_name }}); |
|
my ($now_host_name) = $anvil->System->host_name({ |
|
debug => 2, |
|
set => $new_host_name, |
|
}); |
|
if ($now_host_name eq $new_host_name) |
|
{ |
|
# Success! |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0045"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0045"}); |
|
} |
|
else |
|
{ |
|
# Failed |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0063", variables => { |
|
host_name => $new_host_name, |
|
current_host_name => $now_host_name, |
|
}}); |
|
update_progress($anvil, 0, "job_0063,!!host_name!".$new_host_name."!!,!!current_host_name!".$now_host_name."!!"); |
|
sleep 2; |
|
$anvil->nice_exit({exit_code => 3}); |
|
} |
|
} |
|
else |
|
{ |
|
# No need to change |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0077,!!host_name!".$new_host_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0077", variables => { host_name => $new_host_name }}); |
|
} |
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { |
|
'sys::host_name' => $anvil->data->{sys}{host_name}, |
|
}}); |
|
|
|
# Read the local network manager data. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0080"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0080"}); |
|
$anvil->Network->read_nmcli({debug => 2}); |
|
|
|
# Now check IP addresses. |
|
$anvil->Network->get_ips({debug => 2}); |
|
|
|
# This will be set when the first IFN with a gateway is set. |
|
my $default_gateway_interface = ""; |
|
foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}}) |
|
{ |
|
if ($anvil->data->{network}{$local_host}{interface}{$in_iface}{default_gateway}) |
|
{ |
|
$default_gateway_interface = $in_iface; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { default_gateway_interface => $default_gateway_interface }}); |
|
last; |
|
} |
|
} |
|
|
|
my $dns = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{dns}; |
|
my $mtu = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{mtu}; |
|
my $ntp = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{ntp}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
dns => $dns, |
|
mtu => $mtu, |
|
ntp => $ntp, |
|
}}); |
|
|
|
if (not $dns) |
|
{ |
|
$dns = "8.8.8.8,8.8.4.4"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dns => $dns }}); |
|
} |
|
if (not $mtu) |
|
{ |
|
$mtu = "1500"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mtu => $mtu }}); |
|
} |
|
|
|
### TODO: sorting the array seems inconsistent, so sorting in a hash |
|
# The DNS are comma-separated lists, that may or may not have spaces and may or may not be in |
|
# alphabetical order. To properly compare, we'll rebuild the CSV string of the current and desired |
|
# DNS settings. |
|
my @dns_array = split/,/, $dns; |
|
my $wanted_dns_hash = {}; |
|
foreach my $this_dns (sort {$a cmp $b} @dns_array) |
|
{ |
|
$this_dns = $anvil->Words->clean_spaces({ string => $this_dns }); |
|
$wanted_dns_hash->{$this_dns} = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
"wanted_dns_hash->${this_dns}" => $wanted_dns_hash->{$this_dns}, |
|
}}); |
|
} |
|
my $cleaned_wanted_dns = ""; |
|
foreach my $wanted_dns (sort {$a cmp $b} keys %{$wanted_dns_hash}) |
|
{ |
|
$cleaned_wanted_dns .= $wanted_dns.","; |
|
} |
|
$cleaned_wanted_dns =~ s/,$//; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cleaned_wanted_dns => $cleaned_wanted_dns }}); |
|
|
|
# If any network interface is changed, we'll write out the config file then, when done, disconnect |
|
# from the database, restart networking and then reconnect before moving on. |
|
my $restart_interfaces = []; |
|
|
|
foreach my $network (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}}) |
|
{ |
|
my $ip = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}{$network}{ip}; |
|
my $subnet = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network}{subnet}; |
|
my $cidr = $anvil->Convert->cidr({subnet_mask => $subnet}); |
|
my $gateway = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network}{gateway}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { |
|
's1:network' => $network, |
|
's2:ip' => $ip, |
|
's3:subnet' => $subnet, |
|
's4:cidr' => $cidr, |
|
's5:gateway' => $gateway, |
|
}}); |
|
foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}}) |
|
{ |
|
# Only one interface will start with the network name and have an IP address. |
|
next if $in_iface !~ /^${network}_/; |
|
next if not $anvil->data->{network}{$local_host}{interface}{$in_iface}{ip}; |
|
my $current_ip = $anvil->data->{network}{$local_host}{interface}{$in_iface}{ip}; |
|
my $current_subnet = $anvil->data->{network}{$local_host}{interface}{$in_iface}{subnet_mask}; |
|
my $current_gateway = $anvil->data->{network}{$local_host}{interface}{$in_iface}{gateway}; |
|
my $current_dns = $anvil->data->{network}{$local_host}{interface}{$in_iface}{dns}; |
|
my $current_mtu = $anvil->data->{network}{$local_host}{interface}{$in_iface}{mtu}; |
|
my $mac_address = $anvil->data->{network}{$local_host}{interface}{$in_iface}{mac_address}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:in_iface' => $in_iface, |
|
's2:current_ip' => $current_ip, |
|
's4:current_subnet' => $current_subnet, |
|
's5:current_gateway' => $current_gateway, |
|
's6:current_dns' => $current_dns, |
|
's7:current_mtu' => $current_mtu, |
|
}}); |
|
|
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0081,!!name!".$in_iface."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0081", variables => { name => $in_iface }}); |
|
|
|
if ((not $default_gateway_interface) && ($in_iface =~ /^ifn/) && ($gateway)) |
|
{ |
|
# No existing default gateway, but this is the first IFN we've seen with a |
|
# gateway defined, so we'll use this one. |
|
$default_gateway_interface = $in_iface; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_gateway_interface => $default_gateway_interface }}); |
|
} |
|
|
|
my $change = 0; |
|
my $current_dns_hash = {}; |
|
my @current_dns_array = split/,/, $current_dns; |
|
foreach my $this_current_dns (sort {$a cmp $b} @current_dns_array) |
|
{ |
|
$this_current_dns = $anvil->Words->clean_spaces({ string => $this_current_dns }); |
|
$current_dns_hash->{$this_current_dns} = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
"current_dns_hash->::${this_current_dns}" => $current_dns_hash->{$this_current_dns}, |
|
}}); |
|
} |
|
my $cleaned_current_dns = ""; |
|
foreach my $current_dns (sort {$a cmp $b} keys %{$current_dns_hash}) |
|
{ |
|
$cleaned_current_dns .= $current_dns.","; |
|
} |
|
$cleaned_current_dns =~ s/,$//; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
cleaned_wanted_dns => $cleaned_wanted_dns, |
|
cleaned_current_dns => $cleaned_current_dns, |
|
}}); |
|
|
|
if (($current_ip ne $ip) or ($current_subnet ne $subnet)) |
|
{ |
|
# IP / subnet changed. |
|
#print "IP: .... [".$current_ip."] -> [".$ip."]\n"; |
|
#print "Subnet: [".$current_subnet."] -> [".$subnet."]\n"; |
|
$change = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }}); |
|
} |
|
|
|
# gateway? |
|
if ($current_gateway ne $gateway) |
|
{ |
|
#print "Gateway: [".$current_gateway."] -> [".$gateway."]\n"; |
|
$change = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }}); |
|
} |
|
|
|
# Check DNS only if this is the default gateway interface. |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
in_iface => $in_iface, |
|
default_gateway_interface => $default_gateway_interface, |
|
cleaned_current_dns => $cleaned_current_dns, |
|
}}); |
|
if ($in_iface eq $default_gateway_interface) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
cleaned_current_dns => $cleaned_current_dns, |
|
cleaned_wanted_dns => $cleaned_wanted_dns, |
|
}}); |
|
if ($cleaned_wanted_dns ne $cleaned_current_dns) |
|
{ |
|
$change = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }}); |
|
} |
|
} |
|
elsif ($cleaned_current_dns) |
|
{ |
|
# Remove the old DNS entries. |
|
$change = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }}); |
|
} |
|
|
|
if (not $change) |
|
{ |
|
# No change |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0082"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0082"}); |
|
} |
|
else |
|
{ |
|
# Update the config. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0078,!!interface!".$in_iface."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0078", variables => { interface => $in_iface }}); |
|
|
|
my $interface_uuid = $anvil->data->{nmcli}{$local_host}{device_to_uuid}{$in_iface}; |
|
my $filename = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{filename}; |
|
my $interface_name = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{name}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:in_iface' => $in_iface, |
|
's2:interface_name' => $interface_name, |
|
's3:filename' => $filename, |
|
's4:interface_uuid' => $interface_uuid, |
|
}}); |
|
|
|
# Record the name to restart |
|
push @{$restart_interfaces}, $interface_name; |
|
|
|
# Read in the file. We'll update the date stamp and change the variables as |
|
# needed and write it back out. |
|
my $ip_seen = 0; |
|
my $subnet_seen = 0; |
|
my $gateway_seen = 0; |
|
my $dns_seen = 0; |
|
my $mtu_seen = 0; |
|
my $boot_proto_seen = 0; |
|
my $defroute_seen = 0; |
|
my $say_default = $default_gateway_interface eq $in_iface ? "YES" : "NO"; |
|
my $new_config = ""; |
|
my $old_config = $anvil->Storage->read_file({file => $filename}); |
|
foreach my $line (split/\n/, $old_config) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); |
|
if ($line =~ /^BOOTPROTO="(.*?)"/) |
|
{ |
|
my $boot_protocol = $1; |
|
$boot_proto_seen = 1; |
|
$new_config .= "BOOTPROTO=\"none\"\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:boot_proto_seen' => $boot_proto_seen, |
|
's2:boot_protocol' => $boot_protocol, |
|
's3:new_config' => $new_config, |
|
}}); |
|
if ($boot_protocol eq "dhcp") |
|
{ |
|
# Inject the network config. |
|
if (not $ip_seen) |
|
{ |
|
$ip_seen = 1; |
|
$new_config .= "IPADDR=\"".$ip."\"\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:ip_seen' => $boot_proto_seen, |
|
's2:new_config' => $new_config, |
|
}}); |
|
} |
|
if (not $subnet_seen) |
|
{ |
|
if ($cidr) |
|
{ |
|
$new_config .= "PREFIX=\"".$cidr."\"\n"; |
|
} |
|
else |
|
{ |
|
$new_config .= "NETMASK=\"".$subnet."\"\n"; |
|
} |
|
$subnet_seen = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:subnet_seen' => $subnet_seen, |
|
's2:new_config' => $new_config, |
|
}}); |
|
} |
|
if (($gateway) && (not $gateway_seen)) |
|
{ |
|
$gateway_seen = 1; |
|
$new_config .= "GATEWAY=\"".$gateway."\"\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:gateway_seen' => $gateway_seen, |
|
's2:new_config' => $new_config, |
|
}}); |
|
} |
|
if (not $defroute_seen) |
|
{ |
|
$defroute_seen = 1; |
|
$new_config .= "DEFROUTE=\"".$say_default."\"\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:defroute_seen' => $defroute_seen, |
|
's2:new_config' => $new_config, |
|
}}); |
|
} |
|
if (($default_gateway_interface eq $in_iface) && ($cleaned_wanted_dns)) |
|
{ |
|
if (not $dns_seen) |
|
{ |
|
my $i = 1; |
|
foreach my $this_dns (split/,/, $cleaned_wanted_dns) |
|
{ |
|
$new_config .= "DNS".$i."=\"".$this_dns."\"\n"; |
|
$i++; |
|
} |
|
|
|
$dns_seen = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:dns_seen' => $dns_seen, |
|
's2:new_config' => $new_config, |
|
}}); |
|
} |
|
} |
|
} |
|
} |
|
elsif ($line =~ /^IPADDR=".*?"/) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ip_seen => $ip_seen }}); |
|
if (not $ip_seen) |
|
{ |
|
$ip_seen = 1; |
|
$new_config .= "IPADDR=\"".$ip."\"\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:ip_seen' => $boot_proto_seen, |
|
's2:new_config' => $new_config, |
|
}}); |
|
} |
|
} |
|
elsif (($line =~ /^PREFIX=".*?"/) or ($line =~ /^NETMASK=".*?"/)) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { subnet_seen => $subnet_seen }}); |
|
if (not $subnet_seen) |
|
{ |
|
$subnet_seen = 1; |
|
if ($cidr) |
|
{ |
|
$new_config .= "PREFIX=\"".$cidr."\"\n"; |
|
} |
|
else |
|
{ |
|
$new_config .= "NETMASK=\"".$subnet."\"\n"; |
|
} |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:subnet_seen' => $subnet_seen, |
|
's2:new_config' => $new_config, |
|
}}); |
|
} |
|
} |
|
elsif ($line =~ /^GATEWAY=".*"/) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { gateway_seen => $gateway_seen }}); |
|
if ($in_iface ne $default_gateway_interface) |
|
{ |
|
# This interface shouldn't have DNS. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0111", variables => { |
|
interface => $in_iface, |
|
gateway_line => $line, |
|
}}); |
|
next; |
|
} |
|
if (not $gateway_seen) |
|
{ |
|
$gateway_seen = 1; |
|
$new_config .= "GATEWAY=\"".$gateway."\"\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:gateway_seen' => $gateway_seen, |
|
's2:new_config' => $new_config, |
|
}}); |
|
} |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { gateway_seen => $gateway_seen }}); |
|
} |
|
elsif ($line =~ /^DNS\d+=".*"/) |
|
{ |
|
# The first time we see a DNS entry, we inject the DNS we've |
|
# got and then ignore the rest. |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dns_seen => $dns_seen }}); |
|
if ($in_iface ne $default_gateway_interface) |
|
{ |
|
# This interface shouldn't have DNS. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0110", variables => { |
|
interface => $in_iface, |
|
dns_line => $line, |
|
}}); |
|
next; |
|
} |
|
if (not $dns_seen) |
|
{ |
|
|
|
# If there's no DNS, this will do nothing. |
|
my $i = 1; |
|
foreach my $this_dns (split/,/, $cleaned_wanted_dns) |
|
{ |
|
$new_config .= "DNS".$i."=\"".$this_dns."\"\n"; |
|
$i++; |
|
} |
|
$dns_seen = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:dns_seen' => $dns_seen, |
|
's2:new_config' => $new_config, |
|
}}); |
|
} |
|
} |
|
elsif ($line =~ /^MTU=".*"/) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mtu_seen => $mtu_seen }}); |
|
if (not $mtu_seen) |
|
{ |
|
$mtu_seen = 1; |
|
$new_config .= "MTU=\"".$mtu."\"\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:mtu_seen' => $mtu_seen, |
|
's2:new_config' => $new_config, |
|
}}); |
|
} |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mtu_seen => $mtu_seen }}); |
|
} |
|
elsif ($line =~ /^DEFROUTE=".*"/) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }}); |
|
if (not $defroute_seen) |
|
{ |
|
$defroute_seen = 1; |
|
$new_config .= "DEFROUTE=\"".$say_default."\"\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:defroute_seen' => $defroute_seen, |
|
's2:new_config' => $new_config, |
|
}}); |
|
} |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }}); |
|
} |
|
else |
|
{ |
|
$new_config .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }}); |
|
} |
|
} |
|
|
|
# Add any values I've not yet seen. |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
ip_seen => $ip_seen, |
|
subnet_seen => $subnet_seen, |
|
gateway_seen => $gateway_seen, |
|
dns_seen => $dns_seen, |
|
mtu_seen => $mtu_seen, |
|
defroute_seen => $defroute_seen, |
|
}}); |
|
if (not $ip_seen) |
|
{ |
|
$new_config .= "IPADDR=\"".$ip."\"\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }}); |
|
} |
|
if (not $subnet_seen) |
|
{ |
|
if ($cidr) |
|
{ |
|
$new_config .= "PREFIX=\"".$cidr."\"\n"; |
|
} |
|
else |
|
{ |
|
$new_config .= "NETMASK=\"".$subnet."\"\n"; |
|
} |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }}); |
|
} |
|
if ($in_iface eq $default_gateway_interface) |
|
{ |
|
if (not $gateway_seen) |
|
{ |
|
$new_config .= "GATEWAY=\"".$gateway."\"\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }}); |
|
} |
|
if (not $dns_seen) |
|
{ |
|
# If there's no DNS, this will do nothing. |
|
my $i = 1; |
|
foreach my $this_dns (split/,/, $cleaned_wanted_dns) |
|
{ |
|
$new_config .= "DNS".$i."=\"".$this_dns."\"\n"; |
|
$i++; |
|
} |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }}); |
|
} |
|
} |
|
if (not $mtu_seen) |
|
{ |
|
$new_config .= "MTU=\"".$mtu."\"\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }}); |
|
} |
|
if (not $defroute_seen) |
|
{ |
|
$new_config .= "DEFROUTE=\"".$say_default."\"\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }}); |
|
} |
|
|
|
# Write out the new file. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0083,!!file!".$filename."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0083", variables => { file => $filename }}); |
|
$anvil->Storage->write_file({ |
|
debug => 2, |
|
file => $filename, |
|
body => $new_config, |
|
user => "root", |
|
group => "root", |
|
mode => "0644", |
|
overwrite => 1, |
|
}); |
|
} |
|
last; |
|
} |
|
} |
|
|
|
# If there are any entries in '$restart_interfaces', restart |
|
my $restart_interface_count = @{$restart_interfaces}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { restart_interface_count => $restart_interface_count }}); |
|
if ($restart_interface_count) |
|
{ |
|
# Disconnect from the database, as we're about to tear down our connection. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0079"}); |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0079"); |
|
$anvil->Database->disconnect(); |
|
|
|
# Tell nmcli to re-read the config files. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0463"}); |
|
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection reload"}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { |
|
output => $output, |
|
return_code => $return_code, |
|
}}); |
|
|
|
foreach my $interface_name (sort {$a cmp $b} @{$restart_interfaces}) |
|
{ |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0462", variables => { interface => $interface_name }}); |
|
$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection down \"".$interface_name."\""}); |
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0464", variables => { interface => $interface_name }}); |
|
$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection up \"".$interface_name."\""}); |
|
} |
|
|
|
# Wait for a DB connection. We'll wait up to 130 seconds (updelay is 120 seconds, plus a small buffer). |
|
my $wait_until = time + 130; |
|
until ($anvil->data->{sys}{database}{connections}) |
|
{ |
|
$anvil->refresh(); |
|
$anvil->Database->connect(); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0132"}); |
|
if (not $anvil->data->{sys}{database}{connections}) |
|
{ |
|
if (time > $wait_until) |
|
{ |
|
# Failed to reconnect, exit. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0107"}); |
|
$anvil->nice_exit({exit_code => 4}); |
|
} |
|
|
|
# No databases, sleep and then try again. |
|
sleep 2; |
|
} |
|
|
|
# reload the job data. |
|
load_job($anvil); |
|
} |
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0084"}); |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0084"); |
|
} |
|
|
|
# Remove virbr0 if it exists. |
|
if (exists $anvil->data->{network}{$local_host}{interface}{virbr0}) |
|
{ |
|
# Remove the NAT'ed bridge |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0085"}); |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0085"); |
|
|
|
$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-destroy default"}); |
|
$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-undefine default "}); |
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0034"}); |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0034"); |
|
} |
|
|
|
# Update network view |
|
$anvil->Network->read_nmcli({debug => 3}); |
|
$anvil->Network->get_ips({debug => 3}); |
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0086"}); |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0086"); |
|
|
|
# Update MTUs (running interface and config) if needed. |
|
foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}}) |
|
{ |
|
# Only one interface will start with the network name and have an IP address. |
|
my $current_mtu = $anvil->data->{network}{$local_host}{interface}{$in_iface}{mtu}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { |
|
's1:in_iface' => $in_iface, |
|
's2:current_mtu' => $current_mtu, |
|
}}); |
|
|
|
if (($mtu) && ($current_mtu eq $mtu)) |
|
{ |
|
# It's fine |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0087,!!interface!".$in_iface."!!,!!mtu!".$mtu."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0087", variables => { |
|
interface => $in_iface, |
|
mtu => $mtu, |
|
}}); |
|
} |
|
else |
|
{ |
|
# Change the MTU both on the running interface and in the config file. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0088,!!interface!".$in_iface."!!,!!old_mtu!".$current_mtu."!!,!!mtu!".$mtu."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0088", variables => { |
|
interface => $in_iface, |
|
old_mtu => $current_mtu, |
|
mtu => $mtu, |
|
}}); |
|
|
|
# Change the live MTU. |
|
$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{ip}." link set ".$in_iface." mtu ".$mtu}); |
|
|
|
# Now update the config file. |
|
my $interface_uuid = $anvil->data->{nmcli}{$local_host}{device_to_uuid}{$in_iface}; |
|
my $filename = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{filename}; |
|
my $interface_name = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{name}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { |
|
's1:interface_name' => $interface_name, |
|
's2:filename' => $filename, |
|
's3:interface_uuid' => $interface_uuid, |
|
}}); |
|
|
|
my $mtu_seen = 0; |
|
my $new_config = ""; |
|
my $old_config = $anvil->Storage->read_file({file => $filename}); |
|
foreach my $line (split/\n/, $old_config) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }}); |
|
if ($line =~ /^MTU=".*?"/) |
|
{ |
|
$mtu_seen = 1; |
|
$new_config .= "MTU=\"".$mtu."\"\n"; |
|
} |
|
else |
|
{ |
|
$new_config .= $line."\n"; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { new_config => $new_config }}); |
|
} |
|
} |
|
if (not $mtu_seen) |
|
{ |
|
# Inject the MTU variable |
|
$new_config .= "MTU=\"".$mtu."\"\n"; |
|
} |
|
|
|
# Write out the new file. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0083,!!file!".$filename."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0083", variables => { file => $filename }}); |
|
$anvil->Storage->write_file({ |
|
debug => 2, |
|
file => $filename, |
|
body => $new_config, |
|
user => "root", |
|
group => "root", |
|
mode => "0644", |
|
overwrite => 1, |
|
}); |
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0034"}); |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0034"); |
|
} |
|
} |
|
|
|
# Update NTP if set and needed. |
|
if ($ntp) |
|
{ |
|
# Break up the NTP servers into a list, we'll set to '1' the ones we find. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0089"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0089"}); |
|
my $new_ntp = {}; |
|
foreach my $ntp_server (split/,/, $ntp) |
|
{ |
|
$ntp_server = $anvil->Words->clean_spaces({ string => $ntp_server }); |
|
$new_ntp->{$ntp_server} = 0; |
|
} |
|
|
|
# Call chrony to see what servers are setup already. |
|
my $change = 0; |
|
my $new_config = ""; |
|
my $old_config = $anvil->Storage->read_file({file => $anvil->data->{path}{data}{'chrony.conf'}}); |
|
foreach my $line (split/\n/, $old_config) |
|
{ |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }}); |
|
$new_config .= $line."\n"; |
|
if ($line =~ /^Server (.*)$/) |
|
{ |
|
# This is a source. |
|
my $source = $1; |
|
$new_ntp->{$source} = 1; |
|
} |
|
} |
|
|
|
# Are any sources not enabled? |
|
foreach my $source (sort {$a cmp $b} keys %{$new_ntp}) |
|
{ |
|
if (not $new_ntp->{$source}) |
|
{ |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0090,!!server!".$source."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0090", variables => { server => $source }}); |
|
$change = 1; |
|
$new_config .= "Server ".$source."\n"; |
|
} |
|
} |
|
|
|
if ($change) |
|
{ |
|
# Write out the updated file. |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0055,!!file!".$anvil->data->{path}{data}{'chrony.conf'}."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0055", variables => { file => $anvil->data->{path}{data}{'chrony.conf'} }}); |
|
$anvil->Storage->write_file({ |
|
debug => 2, |
|
file => $anvil->data->{path}{data}{'chrony.conf'}, |
|
body => $new_config, |
|
user => "root", |
|
group => "root", |
|
mode => "0644", |
|
overwrite => 1, |
|
}); |
|
|
|
# Restart the daemon. |
|
my $daemon = "chronyd"; |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0091,!!daemon!".$daemon."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0091", variables => { daemon => $daemon }}); |
|
$anvil->System->restart_daemon({ |
|
debug => 3, |
|
daemon => $daemon, |
|
}) |
|
} |
|
} |
|
|
|
# Configure SSH by adding ours and our peer's SSH keys to ~/.ssh/known_hosts |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0113"}); |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0113"); |
|
$anvil->System->check_ssh_keys({debug => 2}); |
|
|
|
# Setup IPMI, if needed. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0114"}); |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0114"); |
|
$anvil->System->configure_ipmi({debug => 2, manifest_uuid => $manifest_uuid}); |
|
|
|
# Wait now until our peer's host name matches what's in the manifest. |
|
my $anvil_uuid = $anvil->data->{sys}{anvil_uuid}; |
|
my $peer_machine = $anvil->data->{sys}{peer_machine}; |
|
my $peer_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$peer_machine}{name}; |
|
my $peer_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{"anvil_".$peer_machine."_host_uuid"}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
"s1:anvil_uuid" => $anvil_uuid, |
|
"s2:peer_machine" => $peer_machine, |
|
"s3:peer_host_name" => $peer_host_name, |
|
"s4:peer_host_uuid" => $peer_host_uuid, |
|
}}); |
|
|
|
my $waiting = 1; |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0482,!!peer_name!".$peer_host_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0482", variables => { peer_name => $peer_host_name }}); |
|
while ($waiting) |
|
{ |
|
$anvil->Database->get_hosts(); |
|
my $current_host_name = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{host_name}; |
|
my $current_short_host_name = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{short_host_name}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
"s1:current_host_name" => $current_host_name, |
|
"s2:current_short_host_name" => $current_short_host_name, |
|
}}); |
|
if (($peer_host_name eq $current_host_name) or |
|
($peer_host_name eq $current_short_host_name)) |
|
{ |
|
# Done! |
|
$waiting = 0; |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0483,!!peer_name!".$peer_host_name."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0483", variables => { peer_name => $peer_host_name }}); |
|
} |
|
else |
|
{ |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0484", variables => { |
|
old_peer_name => $current_short_host_name." / ".$current_host_name, |
|
peer_name => $peer_host_name, |
|
}}); |
|
sleep 5; |
|
} |
|
} |
|
|
|
return(0); |
|
} |
|
|
|
# Load the job details. |
|
sub load_job |
|
{ |
|
my ($anvil) = @_; |
|
|
|
# See if we can find the job details. This method checks for the 'job-uuid' switch if it was used. |
|
$anvil->data->{switches}{'job-uuid'} = "" if not exists $anvil->data->{switches}{'job-uuid'}; |
|
$anvil->data->{jobs}{job_uuid} = "" if not exists $anvil->data->{jobs}{job_uuid}; |
|
|
|
$anvil->Job->get_job_details({debug => 3}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { |
|
"jobs::job_uuid" => $anvil->data->{jobs}{job_uuid}, |
|
}}); |
|
|
|
if (not $anvil->data->{jobs}{job_uuid}) |
|
{ |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0130"}); |
|
sleep 1; |
|
$anvil->nice_exit({exit_code => 7}); |
|
} |
|
|
|
$anvil->data->{job}{progress} = 0; |
|
update_progress($anvil, $anvil->data->{job}{progress}, "clear"); |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0074,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0074", variables => { 'job-uuid' => $anvil->data->{switches}{'job-uuid'} }}); |
|
|
|
my ($machine, $manifest_uuid, $anvil_uuid) = ($anvil->data->{jobs}{job_data} =~ /as_machine=(.*?),manifest_uuid=(.*?),anvil_uuid=(.*?)$/); |
|
$machine = "" if not defined $machine; |
|
$manifest_uuid = "" if not defined $manifest_uuid; |
|
$anvil_uuid = "" if not defined $anvil_uuid; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { |
|
's1:job_data' => $anvil->data->{jobs}{job_data}, |
|
's2:machine' => $machine, |
|
's3:manifest_uuid' => $manifest_uuid, |
|
's4:anvil_uuid' => $anvil_uuid, |
|
}}); |
|
|
|
if ((not $machine) or |
|
(not $manifest_uuid) or |
|
(not $anvil_uuid)) |
|
{ |
|
# This occassionally is hit, but then works when tried again. |
|
update_progress($anvil, 100, "error_0308,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!,!!raw!".$anvil->data->{jobs}{job_data}."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0308", variables => { |
|
'job-uuid' => $anvil->data->{switches}{'job-uuid'}, |
|
raw => $anvil->data->{jobs}{job_data}, |
|
}}); |
|
sleep 2; |
|
$anvil->nice_exit({exit_code => 5}); |
|
} |
|
|
|
$anvil->data->{sys}{machine} = $machine; |
|
$anvil->data->{sys}{peer_machine} = $machine eq "node1" ? "node2" : "node1"; |
|
$anvil->data->{sys}{manifest_uuid} = $manifest_uuid; |
|
$anvil->data->{sys}{anvil_uuid} = $anvil_uuid; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
"s1:sys::machine" => $anvil->data->{sys}{machine}, |
|
"s2:sys::peer_machine" => $anvil->data->{sys}{peer_machine}, |
|
"s3:sys::manifest_uuid" => $anvil->data->{sys}{manifest_uuid}, |
|
"s4:sys::anvil_uuid" => $anvil->data->{sys}{anvil_uuid}, |
|
}}); |
|
|
|
# Load in the host, manifest and anvil data. |
|
$anvil->Database->get_hosts(); |
|
$anvil->Database->get_manifests(); |
|
$anvil->Database->get_anvils(); |
|
|
|
# Parse the manifest |
|
if (not exists $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}) |
|
{ |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "warning_0046", variables => { uuid => $anvil->data->{sys}{manifest_uuid} }}); |
|
update_progress($anvil, 0, "warning_0046,!!uuid!".$anvil->data->{sys}{manifest_uuid}."!!"); |
|
sleep 10; |
|
$anvil->nice_exit({exit_code => 2}); |
|
} |
|
|
|
# Parse the manifest |
|
my $problem = $anvil->Striker->load_manifest({manifest_uuid => $anvil->data->{sys}{manifest_uuid}}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { |
|
manifest_uuid => $anvil->data->{sys}{manifest_uuid}, |
|
problem => $problem, |
|
}}); |
|
if ($problem) |
|
{ |
|
# Something went wrong, fatally. Abort the job. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0076", variables => { uuid => $anvil->data->{sys}{manifest_uuid} }}); |
|
update_progress($anvil, 100, "job_0076,!!uuid!".$anvil->data->{sys}{manifest_uuid}."!!"); |
|
$anvil->nice_exit({exit_code => 2}); |
|
} |
|
|
|
# Make sure we have a valid Anvil! |
|
if (not exists $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}) |
|
{ |
|
# Odd. Error out, the Anvil! might not be loaded yet. |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "warning_0121", variables => { uuid => $anvil->data->{sys}{anvil_uuid} }}); |
|
update_progress($anvil, 0, "warning_0121,!!uuid!".$anvil->data->{sys}{anvil_uuid}."!!"); |
|
sleep 10; |
|
$anvil->nice_exit({exit_code => 5}); |
|
} |
|
|
|
# Load the anvil |
|
$anvil->data->{sys}{node1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; |
|
$anvil->data->{sys}{node2_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { |
|
"sys::node1_host_uuid" => $anvil->data->{sys}{node1_host_uuid}, |
|
"sys::node2_host_uuid" => $anvil->data->{sys}{node2_host_uuid}, |
|
}}); |
|
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0075,!!machine!".$anvil->data->{sys}{machine}."!!,!!manifest_uuid!".$anvil->data->{sys}{manifest_uuid}."!!"); |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0075", variables => { |
|
machine => $anvil->data->{sys}{machine}, |
|
manifest_uuid => $anvil->data->{sys}{manifest_uuid}, |
|
}}); |
|
|
|
return(0); |
|
} |
|
|
|
# If this is being called as a job, this will allow the progress to be updated. |
|
sub update_progress |
|
{ |
|
my ($anvil, $progress, $message) = @_; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { |
|
's1:progress' => $progress, |
|
's2:message' => $message, |
|
}}); |
|
|
|
$progress = 98 if $progress > 100; |
|
if (not $anvil->data->{switches}{'job-uuid'}) |
|
{ |
|
return(0); |
|
} |
|
|
|
$anvil->Job->update_progress({ |
|
debug => 3, |
|
progress => $progress, |
|
message => $message, |
|
job_uuid => $anvil->data->{switches}{'job-uuid'}, |
|
}); |
|
|
|
return(0); |
|
} |
|
|
|
sub wait_for_subnodes |
|
{ |
|
my ($anvil) = @_; |
|
|
|
my $anvil_uuid = $anvil->data->{sys}{anvil_uuid}; |
|
my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; |
|
my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
anvil_uuid => $anvil_uuid, |
|
node1_host_uuid => $node1_host_uuid, |
|
node2_host_uuid => $node2_host_uuid, |
|
}}); |
|
|
|
my $waiting = 1; |
|
update_progress($anvil, $anvil->data->{job}{progress}, "job_0477"); |
|
while($waiting) |
|
{ |
|
my $ready = 1; |
|
foreach my $host_uuid ($node1_host_uuid, $node2_host_uuid) |
|
{ |
|
my $host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
host_uuid => $host_uuid, |
|
host_name => $host_name, |
|
}}); |
|
|
|
my ($maintenance_mode, $variable_uuid, $modified_date) = $anvil->Database->read_variable({ |
|
variable_name => "maintenance_mode", |
|
variable_source_table => "hosts", |
|
variable_source_uuid => $host_uuid, |
|
}); |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { maintenance_mode => $maintenance_mode }}); |
|
(my $configured, $variable_uuid, $modified_date) = $anvil->Database->read_variable({ |
|
variable_name => "system::configured", |
|
variable_source_uuid => $host_uuid, |
|
variable_source_table => "hosts", |
|
}); |
|
$maintenance_mode = 1 if not defined $maintenance_mode; |
|
$configured = 0 if not defined $configured; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { configured => $configured }}); |
|
|
|
# Is anvil-configure-host running? |
|
my $job_running = 0; |
|
$anvil->Database->get_jobs({job_host_uuid => $host_uuid}); |
|
foreach my $job_uuid (sort {$a cmp $b} keys %{$anvil->data->{jobs}{running}}) |
|
{ |
|
my $job_command = $anvil->data->{jobs}{running}{$job_uuid}{job_command}; |
|
my $job_data = $anvil->data->{jobs}{running}{$job_uuid}{job_data}; |
|
my $job_progress = $anvil->data->{jobs}{running}{$job_uuid}{job_progress}; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
|
's1:job_uuid' => $job_uuid, |
|
's2:job_command' => $job_command, |
|
's3:job_data' => $job_data, |
|
's4:job_progress' => $job_progress, |
|
}}); |
|
next if $job_progress == 100; |
|
if ($job_command =~ /anvil-configure-host/) |
|
{ |
|
$job_running = 1; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_running => $job_running }}); |
|
} |
|
} |
|
|
|
if (($maintenance_mode) or ($job_running) or (not $configured)) |
|
{ |
|
$ready = 0; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ready => $ready }}); |
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0475", variables => { |
|
subnode => $host_name, |
|
configured => $configured, |
|
maintenance_mode => $maintenance_mode, |
|
job_running => $job_running, |
|
}}); |
|
} |
|
} |
|
|
|
if ($ready) |
|
{ |
|
$waiting = 0; |
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); |
|
} |
|
else |
|
{ |
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0476"}); |
|
sleep 5; |
|
} |
|
} |
|
|
|
return(0); |
|
}
|
|
|