seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Local modifications to ClusterLabs/Anvil by Alteeve
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
3589 Commits
2 Branches
0 Tags
21 MiB
 
 
 
 
 
 
Tag: Branch: Tree: 78b75c649a
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '78b75c649a'
${ noResults }
anvil/tools/anvil-join-anvil

2501 lines
107 KiB
Raw Blame History

#!/usr/bin/perl
#
# This adds a node (or gets a node to join to a new/rebuilt peer or added DR) to an Anvil!.
#
# Exit codes;
# 0 = Normal exit.
# 1 = Failed to connect to any database.
# 2 = Failed to load/parse the manifest.
# 3 = Failed to change the host name.
# 4 = Failed to reconnect to any database after the network was reconfigured
# 5 = Problem parsing job data or loading manifest or anvil data using job data.
# 6 = A pcs call failed.
# 7 = No job was found to run.
#
# TODO:
# - Check to see if this is a cluster node and/or running VMs, and if so, refuse to run.
#
use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
use String::ShellQuote;
use Text::Diff;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;
my $anvil = Anvil::Tools->new();
# Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is
# passed directly, it will be used. Otherwise, the password will be read from the database.
$anvil->Get->switches;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try
# again after we exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0129"});
sleep 2;
$anvil->nice_exit({exit_code => 1});
}
# Get the job details
load_job($anvil);
# Hold until both subnodes are marked as configured and not in maintenance mode.
wait_for_subnodes($anvil);
# Update the user passwords
update_passwords($anvil);
# Check if we need to change any IPs or our hostname.
check_local_network($anvil);
# (wait for out peer and) Configure pacemaker
configure_pacemaker($anvil);
# Configure DRBD
configure_drbd($anvil);
# Enable anvil-safe-start
if (1)
{
my ($return_code) = $anvil->System->enable_daemon({daemon => "anvil-safe-start.service"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
update_progress($anvil, 99, "job_0094,!!daemon!anvil-safe-start.service!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "anvil-safe-start.service" }});
}
update_progress($anvil, 100, "job_0128");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0128"});
# Record that we're done configuring pacemaker.
$anvil->Database->insert_or_update_variables({
variable_name => "system::all::configured",
variable_value => 1,
variable_default => "",
variable_description => "striker_0297",
variable_section => "system",
variable_source_uuid => $anvil->data->{sys}{host_uuid},
variable_source_table => "hosts",
});
$anvil->nice_exit({exit_code => 0});
#############################################################################################################
# Functions #
#############################################################################################################
# Configure DRBD
sub configure_drbd
{
my ($anvil) = @_;
### TODO: See if there is a hardware RAID controller and, if so, auto-enable
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0126");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0126"});
my $updated = $anvil->DRBD->update_global_common({
usage_count => $anvil->data->{sys}{privacy}{strong} ? 0 : 1,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { updated => $updated }});
if ($updated)
{
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0127");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0127"});
}
else
{
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0128");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0128"});
}
# Record that we're done configuring pacemaker.
$anvil->Database->insert_or_update_variables({
variable_name => "system::drbd::configured",
variable_value => 1,
variable_default => "",
variable_description => "striker_0296",
variable_section => "system",
variable_source_uuid => $anvil->data->{sys}{host_uuid},
variable_source_table => "hosts",
});
return(0);
}
# Update the passwords for user accounts.
sub update_passwords
{
my ($anvil) = @_;
my $machine = $anvil->data->{sys}{machine};
my $manifest_uuid = $anvil->data->{sys}{manifest_uuid};
my $anvil_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{name};
my $anvil_uuid = $anvil->data->{sys}{anvil_uuid};
my $new_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
my $users = $machine =~ /dr\d+/ ? ["root", "admin"] : ["root", "admin", "hacluster"];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
machine => $machine,
manifest_uuid => $manifest_uuid,
anvil_name => $anvil_name,
anvil_uuid => $anvil_uuid,
new_password => $anvil->Log->is_secure($new_password),
}});
foreach my $user (@{$users})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { user => $user }});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0093,!!user!".$user."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0093", variables => { user => $user }});
my ($return_code) = $anvil->System->change_shell_user_password({
debug => 3,
user => $user,
new_password => $new_password,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
}
return(0);
}
# (wait for our peer and) Configure pacemaker. If this is a DR host, this is skipped.
sub configure_pacemaker
{
my ($anvil) = @_;
my $machine = $anvil->data->{sys}{machine};
my $manifest_uuid = $anvil->data->{sys}{manifest_uuid};
### TODO: Move these to variables in the 'sys' hash
my $anvil_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{name};
my $anvil_uuid = $anvil->data->{sys}{anvil_uuid};
my $host_name = $anvil->Get->host_name;
my $new_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
my $node1_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
$node1_host_name =~ s/\..*$//;
my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
my $node2_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
$node2_host_name =~ s/\..*$//;
my $peer_host_name = $anvil->Get->host_uuid() eq $node1_host_uuid ? $node2_host_name : $node1_host_name;
my $peer_host_uuid = $anvil->Get->host_uuid() eq $node1_host_uuid ? $node2_host_uuid : $node1_host_uuid;
my $escaped_password = shell_quote($new_password);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
machine => $machine,
anvil_uuid => $anvil_uuid,
anvil_name => $anvil_name,
host_name => $host_name,
manifest_uuid => $manifest_uuid,
node1_host_uuid => $node1_host_uuid,
node1_host_name => $node1_host_name,
node2_host_uuid => $node2_host_uuid,
node2_host_name => $node2_host_name,
peer_host_name => $peer_host_name,
peer_host_uuid => $peer_host_uuid,
new_password => $anvil->Log->is_secure($new_password),
escaped_password => $anvil->Log->is_secure($escaped_password),
}});
# If this is a DR box, we don't use pacemaker.
if ($machine =~ /dr\d+/)
{
update_progress($anvil, ($anvil->data->{job}{progress} += 10), "job_0096");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0096"});
return(0);
}
### Run on both nodes.
# Enable pcsd and start the pcsd daemon.
my ($return_code) = $anvil->System->enable_daemon({daemon => "pcsd.service"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
$return_code = undef;
($return_code) = $anvil->System->start_daemon({daemon => "pcsd.service"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
$return_code = undef;
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0094,!!daemon!pcsd.service!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "pcsd.service" }});
# Enable libvirtd and start the libvirtd daemon.
($return_code) = $anvil->System->enable_daemon({daemon => "libvirtd.service"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
$return_code = undef;
($return_code) = $anvil->System->start_daemon({daemon => "libvirtd.service"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
$return_code = undef;
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0094,!!daemon!libvirtd.service!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "libvirtd.service" }});
# Disabled and stop the drbd daemon.
($return_code) = $anvil->System->disable_daemon({daemon => "drbd.service"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
$return_code = undef;
($return_code) = $anvil->System->stop_daemon({daemon => "drbd.service"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
$return_code = undef;
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!drbd.service!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "drbd.service" }});
# Disabled and stop the ksm and ksmtuned daemon.
($return_code) = $anvil->System->disable_daemon({daemon => "ksm.service"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
$return_code = undef;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "ksm.service" }});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!ksm.service!!");
($return_code) = $anvil->System->stop_daemon({daemon => "ksmtuned.service"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
$return_code = undef;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "ksmtuned.service" }});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!ksmtuned.service!!");
# If there is no corosync.conf, see if the peer has it. If so, copy it. If not, we'll initialize the
# cluster shortly.
if (not -e $anvil->data->{path}{configs}{'corosync.conf'})
{
my $corosync_conf = $anvil->Storage->read_file({
file => $anvil->data->{path}{configs}{'corosync.conf'},
target => $peer_host_name,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { corosync_conf => $corosync_conf }});
if ($corosync_conf ne "!!error!!")
{
# Write the file out.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0100"});
$anvil->Storage->write_file({
debug => 2,
body => $corosync_conf,
file => $anvil->data->{path}{configs}{'corosync.conf'},
user => "root",
group => "root",
mode => "0644",
});
}
}
# Node 1 initializes, node 2 waits.
if ($machine eq "node2")
{
my $start_time = 0;
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0103");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0103"});
# We loop until the peer finishes or the peer's job hit's 100.
my $tried_starting = 0;
my $both_online = 0;
until($both_online)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
'path::configs::corosync.conf' => $anvil->data->{path}{configs}{'corosync.conf'},
}});
if (-e $anvil->data->{path}{configs}{'corosync.conf'})
{
if (not $start_time)
{
# Corosync is configured, we'll wait up to two minutes and then try
# joining the cluster ourselves.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0108"});
$start_time = time + 120;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { start_time => $start_time }});
}
elsif ((time > $start_time) && (not $tried_starting))
{
# We've waited a minute, time to try starting the cluster.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0106"});
my $cluster_started = $anvil->Cluster->start_cluster({debug => 2, all => 1});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { cluster_started => $cluster_started }});
# Mark that weve tried to start.
$tried_starting = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { tried_starting => $tried_starting }});
}
my $problem = $anvil->Cluster->parse_cib({debug => 3});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
if (not $problem)
{
# See if both nodes are online.
my $node1_ready = $anvil->Cluster->check_node_status({node_name => $node1_host_name});
my $node2_ready = $anvil->Cluster->check_node_status({node_name => $node2_host_name});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
node1_ready => $node1_ready,
node2_ready => $node2_ready,
}});
if (($node1_ready) && ($node2_ready))
{
$both_online = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { both_online => $both_online }});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0104");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0104"});
}
else
{
# Not online yet, wait a bit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0105", variables => {
node1_name => $node1_host_name,
node1_ready => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm},
node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd},
node1_crmd => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{'join'},
node1_join => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{ready},
node2_name => $node2_host_name,
node2_ready => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{in_ccm},
node2_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{crmd},
node2_crmd => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{'join'},
node2_join => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{ready},
}});
}
}
}
else
{
# corosync.conf doesn't exist yet.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0107"});
}
if (not $both_online)
{
my $delay = 5;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0816", variables => { delay => $delay }});
sleep $delay;
}
}
}
else
{
# We're node 1, proceed with cluster setup.
my $waiting = 1;
my $warning_printed = 0;
while($waiting)
{
# Try to authenticate against the peer.
my $auth_shell_call = $anvil->data->{path}{exe}{pcs}." host auth ".$node1_host_name." ".$node2_host_name." -u hacluster -p ".$escaped_password;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { auth_shell_call => $auth_shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 3, secure => 1, shell_call => $auth_shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
if ($return_code)
{
# Something went wrong.
if (not $warning_printed)
{
# Update the job
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0097");
$warning_printed = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { warning_printed => $warning_printed }});
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0097"});
sleep 5;
}
else
{
# We're good.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0098");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0098"});
$waiting = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
}
}
# If there is no corosync.conf, see if the peer has it. If so, copy it. If not, initialize
# the cluster.
if (not -e $anvil->data->{path}{configs}{'corosync.conf'})
{
# There's no cluster yet, initialize it.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0099,!!anvil_name!".$anvil_name."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0099", variables => { anvil_name => $anvil_name }});
my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster setup ".$anvil_name." ".$node1_host_name." ".$node2_host_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
if ($return_code)
{
# Something went wrong
update_progress($anvil, 100, "job_0101,!!error!".$output."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0101", variables => { error => $output }});
$anvil->nice_exit({exit_code => 5});
}
}
# If we can parse the CIB, then pcsd is running.
my $problem = $anvil->Cluster->parse_cib({debug => 3});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
if ($problem)
{
# Start the cluster.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0102");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0102"});
my $cluster_started = $anvil->Cluster->start_cluster({debug => 3, all => 1});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cluster_started => $cluster_started }});
}
# Now wait for both nodes to come online.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0109");
my $both_online = 0;
my $start_again = time + 60;
until ($both_online)
{
### TODO: If we're waiting more that five minutes, call 'pcs cluster start --all' again.
my $problem = $anvil->Cluster->parse_cib({debug => 3});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
if (not $problem)
{
# See if both nodes are online.
my $node1_ready = $anvil->Cluster->check_node_status({node_name => $node1_host_name});
my $node2_ready = $anvil->Cluster->check_node_status({node_name => $node2_host_name});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
node1_ready => $node1_ready,
node2_ready => $node2_ready,
}});
if (($node1_ready) && ($node2_ready))
{
$both_online = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { both_online => $both_online }});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0104");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0104"});
}
else
{
# Not online yet, wait a bit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0105", variables => {
node1_name => $node1_host_name,
node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm},
node1_crmd => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd},
node1_join => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{'join'},
node1_ready => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{ready},
node2_name => $node2_host_name,
node2_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{in_ccm},
node2_crmd => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{crmd},
node2_join => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{'join'},
node2_ready => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{ready},
}});
}
}
if (time > $start_again)
{
# Call cluster start again.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0272"});
$start_again = time + 60;
my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster start --all";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
start_again => $start_again,
shell_call => $shell_call,
}});
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
}
sleep 5 if not $both_online;
}
### Setup fencing!
$anvil->Striker->get_fence_data();
# IPMI first, if applicable.
my $something_changed = {};
my $fence_order = {};
my $node1_use_delay = 0;
my $node2_use_delay = 0;
my $fence_devices = {};
foreach my $node ("node1", "node2")
{
my $node_name = $node eq "node1" ? $node1_host_name : $node2_host_name;
my $host_uuid = $node eq "node1" ? $node1_host_uuid : $node2_host_uuid;
my $host_ipmi = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi};
my $ipmi_stonith_name = "ipmilan_".$node;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
node_name => $node_name,
host_uuid => $host_uuid,
host_ipmi => $host_ipmi =~ /passw/ ? $anvil->Log->is_secure($host_ipmi) : $host_ipmi,
ipmi_stonith_name => $ipmi_stonith_name,
}});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0115,!!node!".$node_name."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0115", variables => { node => $node_name }});
# This will store the fence level order. If something changes
$fence_order->{$node_name} = [];
# This will switch to '1' if something changed, triggering a reconfig of the fencing levels.
$something_changed->{$node_name} = 0;
# Does this stonith method already exist?
my $create_entry = 0;
my $delete_old = 0;
my $pcs_add_command = "";
if ($host_ipmi)
{
push @{$fence_order->{$node_name}}, "fence_ipmilan";
$fence_devices->{$node_name}{fence_ipmilan} = [$ipmi_stonith_name];
# The --action switch needs to be 'pcmk_off_action' in pcs, so we convert it here.
$host_ipmi =~ s/--action status//;
$host_ipmi =~ s/--action/--pcmk_off_action/;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
host_ipmi => $host_ipmi =~ /passw/ ? $anvil->Log->is_secure($host_ipmi) : $host_ipmi,
}});
# We have IPMI, so we also want fence_delay for this node.
if ($node eq "node1")
{
$node1_use_delay = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { node1_use_delay => $node1_use_delay }});
}
else
{
$node2_use_delay = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { node2_use_delay => $node2_use_delay }});
}
# If we're here, break up the command and turn it into the pcs call.
my $old_switches = {};
my ($fence_agent, $arguments) = ($host_ipmi =~ /^\/.*\/(.*?)\s+(.*)$/);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
fence_agent => $fence_agent,
arguments => $arguments =~ /passw/ ? $anvil->Log->is_secure($arguments) : $arguments,
}});
$pcs_add_command = $anvil->data->{path}{exe}{pcs}." stonith create ".$ipmi_stonith_name." ".$fence_agent." pcmk_host_list=\"".$node_name."\" ";
my $switches = $anvil->System->parse_arguments({debug => 2, arguments => $arguments});
foreach my $switch (sort {$a cmp $b} keys %{$switches})
{
# Ignore 'delay', we handle that in Cluster->set_delay(); Also,
# convert '#!SET!#' to 'true'.
my $value = $switches->{$switch};
$value =~ s/"/\\"/g;
$value =~ s/#!SET!#/true/g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
switch => $switch,
value => $value,
}});
next if $anvil->data->{fence_data}{$fence_agent}{switch}{$switch}{name} eq "delay";
next if $anvil->data->{fence_data}{$fence_agent}{switch}{$switch}{name} eq "action";
# Find the argument=value version.
my $argument = $anvil->data->{fence_data}{$fence_agent}{switch}{$switch}{name};
$pcs_add_command .= $argument."=\"".$value."\" ";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
argument => $argument,
value => $argument =~ /passw/ ? $anvil->Log->is_secure($value) : $value,
pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command,
}});
# Store this to see if it's different from what's already in the CIB.
$old_switches->{$argument} = $value;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"old_switches->{$argument}" => $old_switches->{$argument},
}});
}
$pcs_add_command .= "op monitor interval=\"60\"";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command,
}});
# If there's an entry in the CIB, so if it's different somehow
if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name})
{
foreach my $argument (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name}{argument}})
{
next if $argument eq "delay";
next if $argument eq "action";
my $old_entry = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name}{argument}{$argument}{value};
my $new_entry = exists $old_switches->{$argument} ? $old_switches->{$argument} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:argument' => $argument,
's2:old_entry' => $old_entry,
's3:new_entry' => $new_entry,
}});
if ($old_entry ne $new_entry)
{
# Changed, delete and recreate.
$delete_old = 1;
$create_entry = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
delete_old => $delete_old,
create_entry => $create_entry,
}});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0117");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0117"});
last;
}
# Delete the old switch.
delete $old_switches->{$argument};
}
# Are there any old switches left?
my $old_switch_count = keys %{$old_switches};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
delete_old => $delete_old,
old_switch_count => $old_switch_count,
}});
if ((not $delete_old) && ($old_switch_count))
{
# Delete and recreate.
$delete_old = 1;
$create_entry = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
delete_old => $delete_old,
create_entry => $create_entry,
}});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0117");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0117"});
}
}
else
{
# No existing entry, add a new one.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0116");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0116"});
$create_entry = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { create_entry => $create_entry }});
}
}
elsif (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name})
{
# There was an existing fence config, but there's no entry in 'host_ipmi'.
# Remove the stonith entry.
$delete_old = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { delete_old => $delete_old }});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0118");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0118"});
}
# Process the IPMI entry.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
delete_old => $delete_old,
create_entry => $create_entry,
}});
if ($delete_old)
{
# Delete
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0119,!!device!".$ipmi_stonith_name."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0119", variables => { device => $ipmi_stonith_name }});
my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith delete ".$ipmi_stonith_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
output => $output,
return_code => $return_code,
}});
if ($return_code)
{
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
shell_call => $shell_call,
output => $output,
return_code => $return_code,
}});
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
sleep 2;
$anvil->nice_exit({exit_code => 6});
}
$something_changed->{$node_name} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
}
if ($create_entry)
{
# Create.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0120,!!device!".$ipmi_stonith_name."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0120", variables => { device => $ipmi_stonith_name }});
my $shell_call = $pcs_add_command;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
output => $output,
return_code => $return_code,
}});
if ($return_code)
{
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
shell_call => $shell_call,
output => $output,
return_code => $return_code,
}});
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
sleep 2;
$anvil->nice_exit({exit_code => 6});
}
$something_changed->{$node_name} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
}
### Now any other fence devices.
foreach my $device (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$node}{fence}})
{
my $delete_old = 0;
my $create_entry = 0;
my $old_switches = {};
my $fence_uuid = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{fences}{$device}{uuid};
my $fence_name = $anvil->data->{fences}{fence_uuid}{$fence_uuid}{fence_name};
my $fence_arguments = $anvil->data->{fences}{fence_uuid}{$fence_uuid}{fence_arguments};
my $fence_agent = $anvil->data->{fences}{fence_uuid}{$fence_uuid}{fence_agent};
my $stonith_name = ($fence_agent =~ /^fence_(.*)$/)[0]."_".$node."_".$fence_name;
my $port = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$node}{fence}{$device}{port};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
device => $device,
fence_uuid => $fence_uuid,
fence_name => $fence_name,
fence_arguments => $fence_arguments =~ /passw/ ? $anvil->Log->is_secure($fence_arguments) : $fence_arguments,
stonith_name => $stonith_name,
port => $port,
}});
# We use this to tell if there are two or more entries per agent. If there
# are, we link them later when setting up the fence levels.
if (not exists $fence_devices->{$node_name}{$fence_agent})
{
push @{$fence_order->{$node_name}}, $fence_agent;
$fence_devices->{$node_name}{$fence_agent} = [];
}
push @{$fence_devices->{$node_name}{$fence_agent}}, $stonith_name;
# Fence arguments use 'action', but pcs deprecated it in favour of 'pcmk_off_action', so rename it.
$fence_arguments =~ s/action=/pcmk_off_action=/;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
fence_arguments => $fence_arguments =~ /passw/ ? $anvil->Log->is_secure($fence_arguments) : $fence_arguments,
}});
# Build the pcs command
my $pcs_add_command = $anvil->data->{path}{exe}{pcs}." stonith create ".$stonith_name." ".$fence_agent." pcmk_host_list=\"".$node_name."\" ".$fence_arguments." ";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command,
}});
while ($fence_arguments =~ /=/)
{
# Ignore 'delay', we handle that in Cluster->set_delay();
my $pair = ($fence_arguments =~ /(\S*?=".*?")/)[0];
$fence_arguments =~ s/$pair//;
$fence_arguments =~ s/^\s+//;
$fence_arguments =~ s/\s+$//;
my ($argument, $value) = ($pair =~ /(.*)="(.*)"/);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:fence_arguments' => $fence_arguments,
's2:pair' => $pair =~ /passw/ ? $anvil->Log->is_secure($pair) : $pair,
's3:argument' => $argument,
's4:value' => $argument =~ /passw/ ? $anvil->Log->is_secure($value) : $value,
}});
# Ignore 'delay', we handle that in Cluster->set_delay();
if (($argument ne "pcmk_off_action") &&
(exists $anvil->data->{fence_data}{$fence_agent}{switch}{$argument}{name}) &&
($anvil->data->{fence_data}{$fence_agent}{switch}{$argument}{name} eq "delay"))
{
next;
}
# Store this to see if it's different from what's already in the CIB.
$old_switches->{$argument} = $value;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"old_switches->{$argument}" => $old_switches->{$argument},
}});
}
if ($port)
{
$port =~ s/"/\\"/g;
$pcs_add_command .= "port=\"".$port."\" ";
$old_switches->{port} = $port;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command,
"old_switches->{port}" => $old_switches->{port},
}});
}
$pcs_add_command .= "op monitor interval=\"60\"";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command,
}});
# Does this device exist already?
if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name})
{
foreach my $argument (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name}{argument}})
{
next if $argument eq "delay";
my $old_entry = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name}{argument}{$argument}{value};
my $new_entry = exists $old_switches->{$argument} ? $old_switches->{$argument} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:argument' => $argument,
's2:old_entry' => $old_entry,
's3:new_entry' => $new_entry,
}});
if ($old_entry ne $new_entry)
{
# Changed, delete and recreate.
$delete_old = 1;
$create_entry = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
delete_old => $delete_old,
create_entry => $create_entry,
}});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0121,!!device!".$stonith_name."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0121", variables => { device => $stonith_name }});
last;
}
# Delete the old switch.
delete $old_switches->{$argument};
}
# Are there any old switches left?
my $old_switch_count = keys %{$old_switches};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_switch_count => $old_switch_count }});
if ((not $delete_old) && ($old_switch_count))
{
# Delete and recreate.
$delete_old = 1;
$create_entry = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
delete_old => $delete_old,
create_entry => $create_entry,
}});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0121,!!device!".$stonith_name."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0121", variables => { device => $stonith_name }});
}
}
else
{
# No existing entry, add a new one.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0122,!!device!".$stonith_name."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0122", variables => { device => $stonith_name }});
$create_entry = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_entry => $create_entry }});
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
delete_old => $delete_old,
create_entry => $create_entry,
}});
if ($delete_old)
{
# Delete
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0119,!!device!".$stonith_name."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0119", variables => { device => $stonith_name }});
my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith delete ".$stonith_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
if ($return_code)
{
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
shell_call => $shell_call,
output => $output,
return_code => $return_code,
}});
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
sleep 2;
$anvil->nice_exit({exit_code => 6});
}
$something_changed->{$node_name} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
}
if ($create_entry)
{
# Create.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0120,!!device!".$stonith_name."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0120", variables => { device => $stonith_name }});
my $shell_call = $pcs_add_command;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
if ($return_code)
{
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
shell_call => $shell_call,
output => $output,
return_code => $return_code,
}});
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
sleep 2;
$anvil->nice_exit({exit_code => 6});
}
$something_changed->{$node_name} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
}
}
### If we had a fence_ipmilan entry, add a 'fence_delay' entry, if needed.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
node => $node,
node1_use_delay => $node1_use_delay,
node2_use_delay => $node2_use_delay,
}});
if ((($node eq "node1") && ($node1_use_delay)) or
(($node eq "node2") && ($node2_use_delay)))
{
my $stonith_name = "delay_".$node;
push @{$fence_order->{$node_name}}, "fence_delay";
$fence_devices->{$node_name}{fence_delay} = [$stonith_name];
# Add the fence delay if it doesn't exist yet.
if (not exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name})
{
my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith create ".$stonith_name." fence_delay pcmk_host_list=\"".$node_name."\" wait=\"60\" op monitor interval=\"60\"";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
if ($return_code)
{
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
shell_call => $shell_call,
output => $output,
return_code => $return_code,
}});
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
sleep 2;
$anvil->nice_exit({exit_code => 6});
}
}
}
}
# Setup fence levels.
foreach my $node_name (sort {$a cmp $b} keys %{$fence_order})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
if ($something_changed->{$node_name})
{
# Update our view of the cluster.
my $problem = $anvil->Cluster->parse_cib({debug => 2});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
# Delete any existing fence levels
if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name})
{
foreach my $index (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{order}})
{
# pcs stonith level delete <index> <target>
my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith level delete ".$index." ".$node_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
if ($return_code)
{
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
shell_call => $shell_call,
output => $output,
return_code => $return_code,
}});
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
sleep 2;
$anvil->nice_exit({exit_code => 6});
}
}
}
# Create the new fence levels
my $i = 1;
foreach my $fence_agent (@{$fence_order->{$node_name}})
{
my $devices = "";
foreach my $device (sort {$a cmp $b} @{$fence_devices->{$node_name}{$fence_agent}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { device => $device }});
$devices .= $device.",";
}
$devices =~ s/,$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { devices => $devices }});
my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith level add ".$i." ".$node_name." ".$devices;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
if ($return_code)
{
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
shell_call => $shell_call,
output => $output,
return_code => $return_code,
}});
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
sleep 2;
$anvil->nice_exit({exit_code => 6});
}
$i++;
}
}
}
}
# Make sure logind is update to handle fencing properly
# see - https://access.redhat.com/solutions/1578823
$anvil->Cluster->configure_logind({debug => 2});
# Enable fencing and set the retry to INFINITY, if needed.
$anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'} = "" if not defined $anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"cib::parsed::data::stonith::max-attempts" => $anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'},
}});
if ($anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'} ne "INFINITY")
{
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0124");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0124"});
my $shell_call = $anvil->data->{path}{exe}{pcs}." property set stonith-max-attempts=INFINITY";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
if ($return_code)
{
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
shell_call => $shell_call,
output => $output,
return_code => $return_code,
}});
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
sleep 2;
$anvil->nice_exit({exit_code => 6});
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"cib::parsed::data::stonith::enabled" => $anvil->data->{cib}{parsed}{data}{stonith}{enabled},
}});
if (not $anvil->data->{cib}{parsed}{data}{stonith}{enabled})
{
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0125");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0125"});
my $shell_call = $anvil->data->{path}{exe}{pcs}." property set stonith-enabled=true";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
if ($return_code)
{
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
shell_call => $shell_call,
output => $output,
return_code => $return_code,
}});
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
sleep 2;
$anvil->nice_exit({exit_code => 6});
}
}
# Update (if needed) corosync.conf to use the BCN1 and SN1 as knet networks.
if ($machine eq "node1")
{
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0344");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0344"});
check_corosync($anvil);
}
# Record that we're done configuring pacemaker.
$anvil->Database->insert_or_update_variables({
variable_name => "system::pacemaker::configured",
variable_value => 1,
variable_default => "",
variable_description => "striker_0295",
variable_section => "system",
variable_source_uuid => $anvil->data->{sys}{host_uuid},
variable_source_table => "hosts",
});
return(0);
}
sub check_corosync
{
my ($anvil) = @_;
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0345");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0345"});
my $waiting = 1;
my $anvil_uuid = $anvil->data->{sys}{anvil_uuid};
my $new_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
while ($waiting)
{
my $problem = $anvil->Cluster->parse_cib({debug => 3});
my $peer_ready = $anvil->data->{cib}{parsed}{peer}{ready};
my $peer_name = $anvil->data->{cib}{parsed}{peer}{name};
my $peer_short_name = $peer_name;
$peer_short_name =~ s/\..*$//;
my $peer_bcn_name = $peer_short_name.".bcn1";
my $peer_sn_name = $peer_short_name.".sn1";
my $local_ready = $anvil->data->{cib}{parsed}{'local'}{ready};
my $local_name = $anvil->data->{cib}{parsed}{'local'}{name};
my $local_short_name = $anvil->Get->short_host_name;
my $local_bcn_name = $local_short_name.".bcn1";
my $local_sn_name = $local_short_name.".sn1";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
problem => $problem,
peer_ready => $peer_ready,
peer_name => $peer_name,
peer_short_name => $peer_short_name,
peer_bcn_name => $peer_bcn_name,
peer_sn_name => $peer_sn_name,
local_ready => $local_ready,
local_name => $local_name,
local_short_name => $local_short_name,
local_bcn_name => $local_bcn_name,
local_sn_name => $local_sn_name,
}});
if ((not $problem) && ($peer_ready) && ($local_ready))
{
update_progress($anvil, $anvil->data->{job}{progress}, "job_0346");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0346"});
# Verify we can ping the peer on the BCN and SN.
my $bcn_access = $anvil->Remote->test_access({
target => $peer_bcn_name,
password => $new_password,
});
my $sn_access = $anvil->Remote->test_access({
target => $peer_sn_name,
password => $new_password,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
bcn_access => $bcn_access,
sn_access => $sn_access,
}});
if (($bcn_access) && ($sn_access))
{
# We're ready!
update_progress($anvil, ($anvil->data->{job}{progress}+1), "job_0347");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0347"});
$waiting = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
my $in_totem = 0;
my $token_seen = 0;
my $in_nodelist = 0;
my $in_node = 0;
my $ring0_addr = "";
my $ring1_addr = "";
my $in_node_name = "";
my $nodelist_body = "";
my $old_corosync_conf = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'corosync.conf'}});
my $new_corosync_conf = "";
foreach my $line (split/\n/, $old_corosync_conf)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
if ($line =~ /totem \{/)
{
$in_totem = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_totem => $in_totem }});
$new_corosync_conf .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
next;
}
if ($line =~ /nodelist \{/)
{
$in_nodelist = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
$new_corosync_conf .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
next;
}
if ($in_nodelist)
{
if ($line =~ /node \{/)
{
$in_node = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_node => $in_node }});
$new_corosync_conf .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
next;
}
if ($in_node)
{
if ($line =~ /name:(.*)$/)
{
$in_node_name = $anvil->Words->clean_spaces({string => $1});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_node_name => $in_node_name }});
$nodelist_body .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }});
next;
}
elsif ($line =~ /ring0_addr:(.*)$/)
{
$ring0_addr = $anvil->Words->clean_spaces({string => $1});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ring0_addr => $ring0_addr }});
$nodelist_body .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }});
next;
}
elsif ($line =~ /ring1_addr:(.*)$/)
{
$ring1_addr = $anvil->Words->clean_spaces({string => $1});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ring1_addr => $ring1_addr }});
$nodelist_body .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }});
next;
}
elsif ($line =~ /}/)
{
$in_node = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_node => $in_node }});
# Is this the local or peer node?
my $test_ring0_addr = $peer_bcn_name;
my $test_ring1_addr = $peer_sn_name;
if (($in_node_name eq $anvil->Get->host_name) or ($in_node_name eq $anvil->Get->short_host_name))
{
# It's us
$test_ring0_addr = $local_bcn_name;
$test_ring1_addr = $local_sn_name;
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:ring0_addr' => $ring0_addr,
's2:test_ring0_addr' => $test_ring0_addr,
's3:ring1_addr' => $ring1_addr,
's4:test_ring1_addr' => $test_ring1_addr,
}});
if ((not $ring1_addr) or ($ring0_addr ne $test_ring0_addr) or ($ring1_addr ne $test_ring1_addr))
{
# Rewrite the nodelist body.
my $new_nodelist_body = " ring0_addr: ".$test_ring0_addr."\n";
$new_nodelist_body .= " ring1_addr: ".$test_ring1_addr."\n";
foreach my $nodelist_line (split/\n/, $nodelist_body)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_line => $nodelist_line }});
next if $nodelist_line =~ /ring\d_addr/;
$new_nodelist_body .= $nodelist_line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_nodelist_body => $new_nodelist_body }});
}
$nodelist_body = $new_nodelist_body;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_nodelist_body => $new_nodelist_body }});
}
$new_corosync_conf .= $nodelist_body;
$new_corosync_conf .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
$ring0_addr = "";
$ring1_addr = "";
$in_node_name = "";
$nodelist_body = "";
next;
}
else
{
# Normal line, stash it.
$nodelist_body .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }});
next;
}
}
elsif ($line =~ /}/)
{
$in_nodelist = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_nodelist => $in_nodelist }});
$new_corosync_conf .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
next;
}
}
if ($in_totem)
{
if ($line =~ /}/)
{
if (not $token_seen)
{
$new_corosync_conf .= " token: 10000\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
}
$new_corosync_conf .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
$in_totem = 0;
$token_seen = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
in_totem => $in_totem,
token_seen => $token_seen,
}});
next;
}
if ($line =~ /token:/)
{
$token_seen = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
}
$new_corosync_conf .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
}
else
{
# Normal line
$new_corosync_conf .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
}
}
# Take the last new lines of the file bodies.
$old_corosync_conf =~ s/\n$//g;
$new_corosync_conf =~ s/\n$//g;
my $difference = diff \$old_corosync_conf, \$new_corosync_conf, { STYLE => 'Unified' };
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
if ($difference)
{
# Update the corosync.conf, sync it and reload corosync.
update_progress($anvil, ($anvil->data->{job}{progress}+1), "log_0643,!!difference!".$difference."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0643", variables => { difference => $difference }});
$anvil->Storage->write_file({
file => $anvil->data->{path}{configs}{'corosync.conf'},
body => $new_corosync_conf,
overwrite => 1,
backup => 1,
});
# Sync
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0644"});
my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster sync";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
# RC 0 is OK
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
update_progress($anvil, ($anvil->data->{job}{progress}+1), "job_0348,!!return_code!".$return_code."!!,!!output!".$output."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0348", variables => {
output => $output,
return_code => $return_code,
}});
# Reload
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0645"});
$shell_call = $anvil->data->{path}{exe}{pcs}." cluster reload corosync";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
# RC 0 is OK
($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
update_progress($anvil, ($anvil->data->{job}{progress}+1), "job_0349,!!return_code!".$return_code."!!,!!output!".$output."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0349", variables => {
output => $output,
return_code => $return_code,
}});
}
}
}
if ($waiting)
{
sleep 5;
}
}
return(0);
}
# Check if we need to change any IPs or our hostname.
sub check_local_network
{
my ($anvil) = @_;
# What host name and IP(s) should I have?
my $local_host = $anvil->Get->short_host_name();
my $machine = $anvil->data->{sys}{machine};
my $manifest_uuid = $anvil->data->{sys}{manifest_uuid};
my $domain = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{domain};
my $old_host_name = $anvil->Get->host_name;
my $new_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{name};
if ($domain)
{
$new_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{name}.".".$domain;
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's2:domain' => $domain,
's3:old_host_name' => $old_host_name,
's4:new_host_name' => $new_host_name,
}});
$anvil->data->{sys}{host_name} = $new_host_name;
# If the hostname isn't the same, change it.
if ($old_host_name ne $new_host_name)
{
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0061,!!host_name!".$new_host_name."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0061", variables => { host_name => $new_host_name }});
my ($now_host_name) = $anvil->System->host_name({
debug => 2,
set => $new_host_name,
});
if ($now_host_name eq $new_host_name)
{
# Success!
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0045");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0045"});
}
else
{
# Failed
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0063", variables => {
host_name => $new_host_name,
current_host_name => $now_host_name,
}});
update_progress($anvil, 0, "job_0063,!!host_name!".$new_host_name."!!,!!current_host_name!".$now_host_name."!!");
sleep 2;
$anvil->nice_exit({exit_code => 3});
}
}
else
{
# No need to change
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0077,!!host_name!".$new_host_name."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0077", variables => { host_name => $new_host_name }});
}
# Read the local network manager data.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0080");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0080"});
$anvil->Network->read_nmcli({debug => 2});
# Now check IP addresses.
$anvil->Network->get_ips({debug => 2});
# This will be set when the first IFN with a gateway is set.
my $default_gateway_interface = "";
foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
{
if ($anvil->data->{network}{$local_host}{interface}{$in_iface}{default_gateway})
{
$default_gateway_interface = $in_iface;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { default_gateway_interface => $default_gateway_interface }});
last;
}
}
my $dns = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{dns};
my $mtu = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{mtu};
my $ntp = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{ntp};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
dns => $dns,
mtu => $mtu,
ntp => $ntp,
}});
if (not $dns)
{
$dns = "8.8.8.8,8.8.4.4";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dns => $dns }});
}
if (not $mtu)
{
$mtu = "1500";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mtu => $mtu }});
}
### TODO: sorting the array seems inconsistent, so sorting in a hash
# The DNS are comma-separated lists, that may or may not have spaces and may or may not be in
# alphabetical order. To properly compare, we'll rebuild the CSV string of the current and desired
# DNS settings.
my @dns_array = split/,/, $dns;
my $wanted_dns_hash = {};
foreach my $this_dns (sort {$a cmp $b} @dns_array)
{
$this_dns = $anvil->Words->clean_spaces({ string => $this_dns });
$wanted_dns_hash->{$this_dns} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"wanted_dns_hash->${this_dns}" => $wanted_dns_hash->{$this_dns},
}});
}
my $cleaned_wanted_dns = "";
foreach my $wanted_dns (sort {$a cmp $b} keys %{$wanted_dns_hash})
{
$cleaned_wanted_dns .= $wanted_dns.",";
}
$cleaned_wanted_dns =~ s/,$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cleaned_wanted_dns => $cleaned_wanted_dns }});
# If any network interface is changed, we'll write out the config file then, when done, disconnect
# from the database, restart networking and then reconnect before moving on.
my $restart_interfaces = [];
foreach my $network (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}})
{
my $ip = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}{$network}{ip};
my $subnet = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network}{subnet};
my $cidr = $anvil->Convert->cidr({subnet_mask => $subnet});
my $gateway = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network}{gateway};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:network' => $network,
's2:ip' => $ip,
's3:subnet' => $subnet,
's4:cidr' => $cidr,
's5:gateway' => $gateway,
}});
foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
{
# Only one interface will start with the network name and have an IP address.
next if $in_iface !~ /^${network}_/;
next if not $anvil->data->{network}{$local_host}{interface}{$in_iface}{ip};
my $current_ip = $anvil->data->{network}{$local_host}{interface}{$in_iface}{ip};
my $current_subnet = $anvil->data->{network}{$local_host}{interface}{$in_iface}{subnet_mask};
my $current_gateway = $anvil->data->{network}{$local_host}{interface}{$in_iface}{gateway};
my $current_dns = $anvil->data->{network}{$local_host}{interface}{$in_iface}{dns};
my $current_mtu = $anvil->data->{network}{$local_host}{interface}{$in_iface}{mtu};
my $mac_address = $anvil->data->{network}{$local_host}{interface}{$in_iface}{mac_address};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:in_iface' => $in_iface,
's2:current_ip' => $current_ip,
's4:current_subnet' => $current_subnet,
's5:current_gateway' => $current_gateway,
's6:current_dns' => $current_dns,
's7:current_mtu' => $current_mtu,
}});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0081,!!name!".$in_iface."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0081", variables => { name => $in_iface }});
if ((not $default_gateway_interface) && ($in_iface =~ /^ifn/) && ($gateway))
{
# No existing default gateway, but this is the first IFN we've seen with a
# gateway defined, so we'll use this one.
$default_gateway_interface = $in_iface;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_gateway_interface => $default_gateway_interface }});
}
my $change = 0;
my $current_dns_hash = {};
my @current_dns_array = split/,/, $current_dns;
foreach my $this_current_dns (sort {$a cmp $b} @current_dns_array)
{
$this_current_dns = $anvil->Words->clean_spaces({ string => $this_current_dns });
$current_dns_hash->{$this_current_dns} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"current_dns_hash->::${this_current_dns}" => $current_dns_hash->{$this_current_dns},
}});
}
my $cleaned_current_dns = "";
foreach my $current_dns (sort {$a cmp $b} keys %{$current_dns_hash})
{
$cleaned_current_dns .= $current_dns.",";
}
$cleaned_current_dns =~ s/,$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
cleaned_wanted_dns => $cleaned_wanted_dns,
cleaned_current_dns => $cleaned_current_dns,
}});
if (($current_ip ne $ip) or ($current_subnet ne $subnet))
{
# IP / subnet changed.
#print "IP: .... [".$current_ip."] -> [".$ip."]\n";
#print "Subnet: [".$current_subnet."] -> [".$subnet."]\n";
$change = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
}
# gateway?
if ($current_gateway ne $gateway)
{
#print "Gateway: [".$current_gateway."] -> [".$gateway."]\n";
$change = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
}
# Check DNS only if this is the default gateway interface.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
in_iface => $in_iface,
default_gateway_interface => $default_gateway_interface,
cleaned_current_dns => $cleaned_current_dns,
}});
if ($in_iface eq $default_gateway_interface)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
cleaned_current_dns => $cleaned_current_dns,
cleaned_wanted_dns => $cleaned_wanted_dns,
}});
if ($cleaned_wanted_dns ne $cleaned_current_dns)
{
$change = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
}
}
elsif ($cleaned_current_dns)
{
# Remove the old DNS entries.
$change = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
}
if (not $change)
{
# No change
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0082");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0082"});
}
else
{
# Update the config.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0078,!!interface!".$in_iface."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0078", variables => { interface => $in_iface }});
my $interface_uuid = $anvil->data->{nmcli}{$local_host}{device_to_uuid}{$in_iface};
my $filename = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{filename};
my $interface_name = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:in_iface' => $in_iface,
's2:interface_name' => $interface_name,
's3:filename' => $filename,
's4:interface_uuid' => $interface_uuid,
}});
# Record the name to restart
push @{$restart_interfaces}, $interface_name;
# Read in the file. We'll update the date stamp and change the variables as
# needed and write it back out.
my $ip_seen = 0;
my $subnet_seen = 0;
my $gateway_seen = 0;
my $dns_seen = 0;
my $mtu_seen = 0;
my $boot_proto_seen = 0;
my $defroute_seen = 0;
my $say_default = $default_gateway_interface eq $in_iface ? "YES" : "NO";
my $new_config = "";
my $old_config = $anvil->Storage->read_file({file => $filename});
foreach my $line (split/\n/, $old_config)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
if ($line =~ /^BOOTPROTO="(.*?)"/)
{
my $boot_protocol = $1;
$boot_proto_seen = 1;
$new_config .= "BOOTPROTO=\"none\"\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:boot_proto_seen' => $boot_proto_seen,
's2:boot_protocol' => $boot_protocol,
's3:new_config' => $new_config,
}});
if ($boot_protocol eq "dhcp")
{
# Inject the network config.
if (not $ip_seen)
{
$ip_seen = 1;
$new_config .= "IPADDR=\"".$ip."\"\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:ip_seen' => $boot_proto_seen,
's2:new_config' => $new_config,
}});
}
if (not $subnet_seen)
{
if ($cidr)
{
$new_config .= "PREFIX=\"".$cidr."\"\n";
}
else
{
$new_config .= "NETMASK=\"".$subnet."\"\n";
}
$subnet_seen = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:subnet_seen' => $subnet_seen,
's2:new_config' => $new_config,
}});
}
if (($gateway) && (not $gateway_seen))
{
$gateway_seen = 1;
$new_config .= "GATEWAY=\"".$gateway."\"\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:gateway_seen' => $gateway_seen,
's2:new_config' => $new_config,
}});
}
if (not $defroute_seen)
{
$defroute_seen = 1;
$new_config .= "DEFROUTE=\"".$say_default."\"\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:defroute_seen' => $defroute_seen,
's2:new_config' => $new_config,
}});
}
if (($default_gateway_interface eq $in_iface) && ($cleaned_wanted_dns))
{
if (not $dns_seen)
{
my $i = 1;
foreach my $this_dns (split/,/, $cleaned_wanted_dns)
{
$new_config .= "DNS".$i."=\"".$this_dns."\"\n";
$i++;
}
$dns_seen = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:dns_seen' => $dns_seen,
's2:new_config' => $new_config,
}});
}
}
}
}
elsif ($line =~ /^IPADDR=".*?"/)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ip_seen => $ip_seen }});
if (not $ip_seen)
{
$ip_seen = 1;
$new_config .= "IPADDR=\"".$ip."\"\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:ip_seen' => $boot_proto_seen,
's2:new_config' => $new_config,
}});
}
}
elsif (($line =~ /^PREFIX=".*?"/) or ($line =~ /^NETMASK=".*?"/))
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { subnet_seen => $subnet_seen }});
if (not $subnet_seen)
{
$subnet_seen = 1;
if ($cidr)
{
$new_config .= "PREFIX=\"".$cidr."\"\n";
}
else
{
$new_config .= "NETMASK=\"".$subnet."\"\n";
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:subnet_seen' => $subnet_seen,
's2:new_config' => $new_config,
}});
}
}
elsif ($line =~ /^GATEWAY=".*"/)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { gateway_seen => $gateway_seen }});
if ($in_iface ne $default_gateway_interface)
{
# This interface shouldn't have DNS.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0111", variables => {
interface => $in_iface,
gateway_line => $line,
}});
next;
}
if (not $gateway_seen)
{
$gateway_seen = 1;
$new_config .= "GATEWAY=\"".$gateway."\"\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:gateway_seen' => $gateway_seen,
's2:new_config' => $new_config,
}});
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { gateway_seen => $gateway_seen }});
}
elsif ($line =~ /^DNS\d+=".*"/)
{
# The first time we see a DNS entry, we inject the DNS we've
# got and then ignore the rest.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dns_seen => $dns_seen }});
if ($in_iface ne $default_gateway_interface)
{
# This interface shouldn't have DNS.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0110", variables => {
interface => $in_iface,
dns_line => $line,
}});
next;
}
if (not $dns_seen)
{
# If there's no DNS, this will do nothing.
my $i = 1;
foreach my $this_dns (split/,/, $cleaned_wanted_dns)
{
$new_config .= "DNS".$i."=\"".$this_dns."\"\n";
$i++;
}
$dns_seen = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:dns_seen' => $dns_seen,
's2:new_config' => $new_config,
}});
}
}
elsif ($line =~ /^MTU=".*"/)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mtu_seen => $mtu_seen }});
if (not $mtu_seen)
{
$mtu_seen = 1;
$new_config .= "MTU=\"".$mtu."\"\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:mtu_seen' => $mtu_seen,
's2:new_config' => $new_config,
}});
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mtu_seen => $mtu_seen }});
}
elsif ($line =~ /^DEFROUTE=".*"/)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }});
if (not $defroute_seen)
{
$defroute_seen = 1;
$new_config .= "DEFROUTE=\"".$say_default."\"\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:defroute_seen' => $defroute_seen,
's2:new_config' => $new_config,
}});
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }});
}
else
{
$new_config .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { defroute_seen => $defroute_seen }});
}
}
# Add any values I've not yet seen.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
ip_seen => $ip_seen,
subnet_seen => $subnet_seen,
gateway_seen => $gateway_seen,
dns_seen => $dns_seen,
mtu_seen => $mtu_seen,
defroute_seen => $defroute_seen,
}});
if (not $ip_seen)
{
$new_config .= "IPADDR=\"".$ip."\"\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
}
if (not $subnet_seen)
{
if ($cidr)
{
$new_config .= "PREFIX=\"".$cidr."\"\n";
}
else
{
$new_config .= "NETMASK=\"".$subnet."\"\n";
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
}
if ($in_iface eq $default_gateway_interface)
{
if (not $gateway_seen)
{
$new_config .= "GATEWAY=\"".$gateway."\"\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
}
if (not $dns_seen)
{
# If there's no DNS, this will do nothing.
my $i = 1;
foreach my $this_dns (split/,/, $cleaned_wanted_dns)
{
$new_config .= "DNS".$i."=\"".$this_dns."\"\n";
$i++;
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
}
}
if (not $mtu_seen)
{
$new_config .= "MTU=\"".$mtu."\"\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
}
if (not $defroute_seen)
{
$new_config .= "DEFROUTE=\"".$say_default."\"\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
}
# Write out the new file.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0083,!!file!".$filename."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0083", variables => { file => $filename }});
$anvil->Storage->write_file({
debug => 2,
file => $filename,
body => $new_config,
user => "root",
group => "root",
mode => "0644",
overwrite => 1,
});
}
last;
}
}
# If there are any entries in '$restart_interfaces', restart
my $restart_interface_count = @{$restart_interfaces};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { restart_interface_count => $restart_interface_count }});
if ($restart_interface_count)
{
# Disconnect from the database, as we're about to tear down our connection.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0079"});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0079");
$anvil->Database->disconnect();
# Tell nmcli to re-read the config files.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0463"});
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection reload"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
output => $output,
return_code => $return_code,
}});
foreach my $interface_name (sort {$a cmp $b} @{$restart_interfaces})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0462", variables => { interface => $interface_name }});
$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection down \"".$interface_name."\""});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0464", variables => { interface => $interface_name }});
$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection up \"".$interface_name."\""});
}
# Wait for a DB connection. We'll wait up to 130 seconds (updelay is 120 seconds, plus a small buffer).
my $wait_until = time + 130;
until ($anvil->data->{sys}{database}{connections})
{
$anvil->refresh();
$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
if (time > $wait_until)
{
# Failed to reconnect, exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0107"});
$anvil->nice_exit({exit_code => 4});
}
# No databases, sleep and then try again.
sleep 2;
}
# reload the job data.
load_job($anvil);
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0084"});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0084");
}
# Remove virbr0 if it exists.
if (exists $anvil->data->{network}{$local_host}{interface}{virbr0})
{
# Remove the NAT'ed bridge
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0085"});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0085");
$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-destroy default"});
$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-undefine default "});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0034"});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0034");
}
# Update network view
$anvil->Network->read_nmcli({debug => 3});
$anvil->Network->get_ips({debug => 3});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0086"});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0086");
# Update MTUs (running interface and config) if needed.
foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
{
# Only one interface will start with the network name and have an IP address.
my $current_mtu = $anvil->data->{network}{$local_host}{interface}{$in_iface}{mtu};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:in_iface' => $in_iface,
's2:current_mtu' => $current_mtu,
}});
if (($mtu) && ($current_mtu eq $mtu))
{
# It's fine
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0087,!!interface!".$in_iface."!!,!!mtu!".$mtu."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0087", variables => {
interface => $in_iface,
mtu => $mtu,
}});
}
else
{
# Change the MTU both on the running interface and in the config file.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0088,!!interface!".$in_iface."!!,!!old_mtu!".$current_mtu."!!,!!mtu!".$mtu."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0088", variables => {
interface => $in_iface,
old_mtu => $current_mtu,
mtu => $mtu,
}});
# Change the live MTU.
$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{ip}." link set ".$in_iface." mtu ".$mtu});
# Now update the config file.
my $interface_uuid = $anvil->data->{nmcli}{$local_host}{device_to_uuid}{$in_iface};
my $filename = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{filename};
my $interface_name = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:interface_name' => $interface_name,
's2:filename' => $filename,
's3:interface_uuid' => $interface_uuid,
}});
my $mtu_seen = 0;
my $new_config = "";
my $old_config = $anvil->Storage->read_file({file => $filename});
foreach my $line (split/\n/, $old_config)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
if ($line =~ /^MTU=".*?"/)
{
$mtu_seen = 1;
$new_config .= "MTU=\"".$mtu."\"\n";
}
else
{
$new_config .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { new_config => $new_config }});
}
}
if (not $mtu_seen)
{
# Inject the MTU variable
$new_config .= "MTU=\"".$mtu."\"\n";
}
# Write out the new file.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0083,!!file!".$filename."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0083", variables => { file => $filename }});
$anvil->Storage->write_file({
debug => 2,
file => $filename,
body => $new_config,
user => "root",
group => "root",
mode => "0644",
overwrite => 1,
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0034"});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0034");
}
}
# Update NTP if set and needed.
if ($ntp)
{
# Break up the NTP servers into a list, we'll set to '1' the ones we find.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0089");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0089"});
my $new_ntp = {};
foreach my $ntp_server (split/,/, $ntp)
{
$ntp_server = $anvil->Words->clean_spaces({ string => $ntp_server });
$new_ntp->{$ntp_server} = 0;
}
# Call chrony to see what servers are setup already.
my $change = 0;
my $new_config = "";
my $old_config = $anvil->Storage->read_file({file => $anvil->data->{path}{data}{'chrony.conf'}});
foreach my $line (split/\n/, $old_config)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
$new_config .= $line."\n";
if ($line =~ /^Server (.*)$/)
{
# This is a source.
my $source = $1;
$new_ntp->{$source} = 1;
}
}
# Are any sources not enabled?
foreach my $source (sort {$a cmp $b} keys %{$new_ntp})
{
if (not $new_ntp->{$source})
{
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0090,!!server!".$source."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0090", variables => { server => $source }});
$change = 1;
$new_config .= "Server ".$source."\n";
}
}
if ($change)
{
# Write out the updated file.
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0055,!!file!".$anvil->data->{path}{data}{'chrony.conf'}."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0055", variables => { file => $anvil->data->{path}{data}{'chrony.conf'} }});
$anvil->Storage->write_file({
debug => 2,
file => $anvil->data->{path}{data}{'chrony.conf'},
body => $new_config,
user => "root",
group => "root",
mode => "0644",
overwrite => 1,
});
# Restart the daemon.
my $daemon = "chronyd";
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0091,!!daemon!".$daemon."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0091", variables => { daemon => $daemon }});
$anvil->System->restart_daemon({
debug => 3,
daemon => $daemon,
})
}
}
# Update the hosts file.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0112"});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0112");
$anvil->System->update_hosts({debug => 3});
# Configure SSH by adding ours and our peer's SSH keys to ~/.ssh/known_hosts
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0113"});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0113");
$anvil->System->check_ssh_keys({debug => 2});
# Setup IPMI, if needed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0114"});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0114");
$anvil->System->configure_ipmi({debug => 2, manifest_uuid => $manifest_uuid});
return(0);
}
# Load the job details.
sub load_job
{
my ($anvil) = @_;
# See if we can find the job details. This method checks for the 'job-uuid' switch if it was used.
$anvil->data->{switches}{'job-uuid'} = "" if not exists $anvil->data->{switches}{'job-uuid'};
$anvil->data->{jobs}{job_uuid} = "" if not exists $anvil->data->{jobs}{job_uuid};
$anvil->Job->get_job_details({debug => 3});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"jobs::job_uuid" => $anvil->data->{jobs}{job_uuid},
}});
if (not $anvil->data->{jobs}{job_uuid})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0130"});
sleep 1;
$anvil->nice_exit({exit_code => 7});
}
$anvil->data->{job}{progress} = 0;
update_progress($anvil, $anvil->data->{job}{progress}, "clear");
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0074,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0074", variables => { 'job-uuid' => $anvil->data->{switches}{'job-uuid'} }});
my ($machine, $manifest_uuid, $anvil_uuid) = ($anvil->data->{jobs}{job_data} =~ /as_machine=(.*?),manifest_uuid=(.*?),anvil_uuid=(.*?)$/);
$machine = "" if not defined $machine;
$manifest_uuid = "" if not defined $manifest_uuid;
$anvil_uuid = "" if not defined $anvil_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => {
's1:job_data' => $anvil->data->{jobs}{job_data},
's2:machine' => $machine,
's3:manifest_uuid' => $manifest_uuid,
's4:anvil_uuid' => $anvil_uuid,
}});
if ((not $machine) or
(not $manifest_uuid) or
(not $anvil_uuid))
{
# This occassionally is hit, but then works when tried again.
update_progress($anvil, 100, "error_0308,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!,!!raw!".$anvil->data->{jobs}{job_data}."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0308", variables => {
'job-uuid' => $anvil->data->{switches}{'job-uuid'},
raw => $anvil->data->{jobs}{job_data},
}});
sleep 2;
$anvil->nice_exit({exit_code => 5});
}
$anvil->data->{sys}{machine} = $machine;
$anvil->data->{sys}{manifest_uuid} = $manifest_uuid;
$anvil->data->{sys}{anvil_uuid} = $anvil_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"sys::machine" => $anvil->data->{sys}{machine},
"sys::manifest_uuid" => $anvil->data->{sys}{manifest_uuid},
"sys::anvil_uuid" => $anvil->data->{sys}{anvil_uuid},
}});
# Load in the host, manifest and anvil data.
$anvil->Database->get_hosts();
$anvil->Database->get_manifests();
$anvil->Database->get_anvils();
# Parse the manifest
if (not exists $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "warning_0046", variables => { uuid => $anvil->data->{sys}{manifest_uuid} }});
update_progress($anvil, 0, "warning_0046,!!uuid!".$anvil->data->{sys}{manifest_uuid}."!!");
sleep 10;
$anvil->nice_exit({exit_code => 2});
}
# Parse the manifest
my $problem = $anvil->Striker->load_manifest({manifest_uuid => $anvil->data->{sys}{manifest_uuid}});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
manifest_uuid => $anvil->data->{sys}{manifest_uuid},
problem => $problem,
}});
if ($problem)
{
# Something went wrong, fatally. Abort the job.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0076", variables => { uuid => $anvil->data->{sys}{manifest_uuid} }});
update_progress($anvil, 100, "job_0076,!!uuid!".$anvil->data->{sys}{manifest_uuid}."!!");
$anvil->nice_exit({exit_code => 2});
}
# Make sure we have a valid Anvil!
if (not exists $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid})
{
# Odd. Error out, the Anvil! might not be loaded yet.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "warning_0121", variables => { uuid => $anvil->data->{sys}{anvil_uuid} }});
update_progress($anvil, 0, "warning_0121,!!uuid!".$anvil->data->{sys}{anvil_uuid}."!!");
sleep 10;
$anvil->nice_exit({exit_code => 5});
}
# Load the anvil
$anvil->data->{sys}{node1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
$anvil->data->{sys}{node2_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"sys::node1_host_uuid" => $anvil->data->{sys}{node1_host_uuid},
"sys::node2_host_uuid" => $anvil->data->{sys}{node2_host_uuid},
}});
update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0075,!!machine!".$anvil->data->{sys}{machine}."!!,!!manifest_uuid!".$anvil->data->{sys}{manifest_uuid}."!!");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0075", variables => {
machine => $anvil->data->{sys}{machine},
manifest_uuid => $anvil->data->{sys}{manifest_uuid},
}});
return(0);
}
# If this is being called as a job, this will allow the progress to be updated.
sub update_progress
{
my ($anvil, $progress, $message) = @_;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:progress' => $progress,
's2:message' => $message,
}});
$progress = 98 if $progress > 100;
if (not $anvil->data->{switches}{'job-uuid'})
{
return(0);
}
$anvil->Job->update_progress({
debug => 3,
progress => $progress,
message => $message,
job_uuid => $anvil->data->{switches}{'job-uuid'},
});
return(0);
}
sub wait_for_subnodes
{
my ($anvil) = @_;
my $anvil_uuid = $anvil->data->{sys}{anvil_uuid};
my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
anvil_uuid => $anvil_uuid,
node1_host_uuid => $node1_host_uuid,
node2_host_uuid => $node2_host_uuid,
}});
my $waiting = 1;
update_progress($anvil, $anvil->data->{job}{progress}, "job_0477");
while($waiting)
{
my $ready = 1;
foreach my $host_uuid ($node1_host_uuid, $node2_host_uuid)
{
my $host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
host_uuid => $host_uuid,
host_name => $host_name,
}});
my ($maintenance_mode, $variable_uuid, $modified_date) = $anvil->Database->read_variable({
variable_name => "maintenance_mode",
variable_source_table => "hosts",
variable_source_uuid => $host_uuid,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { maintenance_mode => $maintenance_mode }});
(my $configured, $variable_uuid, $modified_date) = $anvil->Database->read_variable({
variable_name => "system::configured",
variable_source_uuid => $host_uuid,
variable_source_table => "hosts",
});
$maintenance_mode = 1 if not defined $maintenance_mode;
$configured = 0 if not defined $configured;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { configured => $configured }});
# Is anvil-configure-host running?
my $job_running = 0;
$anvil->Database->get_jobs({job_host_uuid => $host_uuid});
foreach my $job_uuid (sort {$a cmp $b} keys %{$anvil->data->{jobs}{running}})
{
my $job_command = $anvil->data->{jobs}{running}{$job_uuid}{job_command};
my $job_data = $anvil->data->{jobs}{running}{$job_uuid}{job_data};
my $job_progress = $anvil->data->{jobs}{running}{$job_uuid}{job_progress};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:job_uuid' => $job_uuid,
's2:job_command' => $job_command,
's3:job_data' => $job_data,
's4:job_progress' => $job_progress,
}});
next if $job_progress == 100;
if ($job_command =~ /anvil-configure-host/)
{
$job_running = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_running => $job_running }});
}
}
if (($maintenance_mode) or ($job_running) or (not $configured))
{
$ready = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ready => $ready }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0475", variables => {
subnode => $host_name,
configured => $configured,
maintenance_mode => $maintenance_mode,
job_running => $job_running,
}});
}
}
if ($ready)
{
$waiting = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
}
else
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0476"});
sleep 5;
}
}
return(0);
}