diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index dd3aacb0..89b09c36 100644 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -1017,6 +1017,7 @@ sub _set_paths # Executables $anvil->data->{path} = { configs => { + 'alteeve-el8.repo' => "/etc/yum.repos.d/alteeve-el8.repo", 'anvil.conf' => "/etc/anvil/anvil.conf", 'anvil.version' => "/etc/anvil/anvil.version", 'autoindex.conf' => "/etc/httpd/conf.d/autoindex.conf", @@ -1247,6 +1248,7 @@ sub _set_paths urls => { skins => "/skins", oui_file => "http://standards.ieee.org/develop/regauth/oui/oui.txt", + alteeve_repo => "https://www.alteeve.com/an-repo/el8/alteeve-el8-repo-latest.noarch.rpm", }, words => { 'words.xml' => "/usr/share/anvil/words.xml", diff --git a/Anvil/Tools/Log.pm b/Anvil/Tools/Log.pm index 00e1ec3f..40deb729 100644 --- a/Anvil/Tools/Log.pm +++ b/Anvil/Tools/Log.pm @@ -357,6 +357,7 @@ sub entry print $THIS_FILE." ".__LINE__."; priority string: [".$priority_string."]\n" if $test; # Log the file and line, if passed. + my $job_uuid = ""; my $string = ""; my $print_string = ""; if ($anvil->data->{sys}{'log'}{date}) @@ -365,19 +366,25 @@ sub entry $string .= $anvil->Get->date_and_time({debug => 99}).":"; print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test; } + if ((exists $anvil->data->{switches}{'job-uuid'}) && ($anvil->Validate->uuid({uuid => $anvil->data->{switches}{'job-uuid'}}))) + { + $job_uuid = $anvil->data->{switches}{'job-uuid'}; + $job_uuid =~ s/^(\w+?)-.*$/$1/; + $string .= "[".$job_uuid."]:"; + } if (($source) && ($line)) { - $string .= "$source:$line; "; + $string .= $source.":".$line."; "; print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test; } elsif ($source) { - $string .= "$source; "; + $string .= $source."; "; print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test; } elsif ($line) { - $string .= "$line; "; + $string .= $line."; "; print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test; } print $THIS_FILE." ".__LINE__."; loop::count: [".$anvil->data->{loop}{count}."] " if $test; diff --git a/Anvil/Tools/Remote.pm b/Anvil/Tools/Remote.pm index 06cd674d..b1f1e3f6 100644 --- a/Anvil/Tools/Remote.pm +++ b/Anvil/Tools/Remote.pm @@ -130,7 +130,7 @@ sub add_target_to_known_hosts }}); # Get the local user's home - my $users_home = $anvil->Get->users_home({debug => $debug, user => $user}); + my $users_home = $anvil->Get->users_home({debug => ($debug + 1), user => $user}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => 0, list => { users_home => $users_home }}); if (not $users_home) { @@ -148,6 +148,7 @@ sub add_target_to_known_hosts { # Yup, see if the target is there already, $known_machine = $anvil->Remote->_check_known_hosts_for_target({ + debug => ($debug + 1), target => $target, port => $port, known_hosts => $known_hosts, @@ -162,7 +163,7 @@ sub add_target_to_known_hosts { # We don't know about this machine yet, so scan it. my $added = $anvil->Remote->_call_ssh_keyscan({ - debug => 2, + debug => ($debug + 1), target => $target, port => $port, user => $user, @@ -331,7 +332,7 @@ sub call { # No shell call $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Remote->call()", parameter => "shell_call" }}); - return("!!error!!"); + return("!!error!!", "!!error!!", 9999); } if (not $target) { @@ -343,12 +344,12 @@ sub call secure => $secure, shell_call => (not $secure) ? $shell_call : $anvil->Log->is_secure($shell_call), }}); - return("!!error!!"); + return("!!error!!", "!!error!!", 9999); } if (not $remote_user) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Remote->call()", parameter => "remote_user" }}); - return("!!error!!"); + return("!!error!!", "!!error!!", 9999); } if (($timeout) && ($timeout =~ /\D/)) { @@ -408,7 +409,7 @@ sub call if ((not defined $port) or (($port !~ /^\d+$/) or ($port < 0) or ($port > 65536))) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0058", variables => { port => $port }}); - return("!!error!!"); + return("!!error!!", "!!error!!", 9999); } # If the target is a host name, convert it to an IP. @@ -963,7 +964,7 @@ sub test_access return_code => $return_code, }}); - if ($output) + if ($output eq "1") { $access = 1; } @@ -1063,6 +1064,7 @@ sub _call_ssh_keyscan # Verify that it's now there. my $known_machine = $anvil->Remote->_check_known_hosts_for_target({ + debug => $debug, target => $target, port => $port, known_hosts => $known_hosts, @@ -1139,7 +1141,8 @@ sub _check_known_hosts_for_target { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => 0, list => { line => $line }}); - if (($line =~ /$target ssh-rsa /) or ($line =~ /\[$target\]:$port ssh-rsa /)) + # This is wider scope now to catch hosts using other hashes than 'ssh-rsa' + if (($line =~ /$target /) or ($line =~ /\[$target\]:$port /)) { # We already know this machine (or rather, we already have a fingerprint for # this machine). diff --git a/share/words.xml b/share/words.xml index 06c30f08..45c555e1 100644 --- a/share/words.xml +++ b/share/words.xml @@ -722,6 +722,14 @@ It should be provisioned in the next minute or two. The job UUID: [#!variable!job_uuid!#] is at: [#!variable!progress!#%]. Not all jobs are done yet, will check again in a bit. All jobs are complete! Baring problems, the Anvil! system(s) should now be ready to use. + The peer Striker: [#!variable!number!#] with host name: [#!variable!peer_host_name!#] is already peered with us. + Configuring the network of all machines now. + Created a job for: [#!variable!host_name!#] to configure it's network under job UUID: [#!variable!job_uuid!#]. + All machines should be configuring their network now. Waiting for all to become accessible over BCN 1. + The machine: [#!variable!host_name!#] is not yet accessible at: [#!variable!ip_address!#]. + One or more machines are not yet accessible on the first BCN. Will check again in a moment. + All machines are now available on the first BCN! + One of the Striker dashboards has not yet updated network information in the database. We need this to know which IP to tell the peer to use to connect to us. We'll wait a moment and check again. Starting: [#!variable!program!#]. diff --git a/tools/anvil-configure-host b/tools/anvil-configure-host index b05996d9..89f6cd92 100755 --- a/tools/anvil-configure-host +++ b/tools/anvil-configure-host @@ -11,7 +11,9 @@ # 5 = Failed to write the temp file with the new password needed to call anvil-change-password. # 6 = The job-uuid was not found. # -# TODO: Add MTU support +# TODO: +# - Add MTU support +# - Check to see if this is a cluster node and/or running VMs, and if so, refuse to run. # use strict; diff --git a/tools/anvil-join-anvil b/tools/anvil-join-anvil index ebf40c12..cc5ad56e 100755 --- a/tools/anvil-join-anvil +++ b/tools/anvil-join-anvil @@ -13,7 +13,7 @@ # 7 = No job was found to run. # # TODO: -# - +# - Check to see if this is a cluster node and/or running VMs, and if so, refuse to run. # use strict; @@ -396,6 +396,7 @@ sub configure_pacemaker my $both_online = 0; until ($both_online) { + ### TODO: If we're waiting more that five minutes, call 'pcs cluster start --all' again. my $problem = $anvil->Cluster->parse_cib({debug => 3}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }}); if (not $problem) @@ -420,15 +421,15 @@ sub configure_pacemaker # Not online yet, wait a bit. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0105", variables => { node1_name => $node1_host_name, - node1_ready => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm}, - node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd}, - node1_crmd => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{'join'}, - node1_join => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{ready}, + node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm}, + node1_crmd => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd}, + node1_join => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{'join'}, + node1_ready => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{ready}, node2_name => $node2_host_name, - node2_ready => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{in_ccm}, - node2_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{crmd}, - node2_crmd => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{'join'}, - node2_join => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{ready}, + node2_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{in_ccm}, + node2_crmd => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{crmd}, + node2_join => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{'join'}, + node2_ready => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{ready}, }}); } } @@ -1484,8 +1485,8 @@ sub check_local_network $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { mtu_seen => $mtu_seen }}); if (not $mtu_seen) { - $mtu_seen = 1; - $new_config .= "MTU=\"".$mtu."\"\n"; + $mtu_seen = 1; + $new_config .= "MTU=\"".$mtu."\"\n"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 's1:mtu_seen' => $mtu_seen, 's2:new_config' => $new_config, diff --git a/tools/striker-auto-initialize-all b/tools/striker-auto-initialize-all index 5810e33c..c8f8c00e 100755 --- a/tools/striker-auto-initialize-all +++ b/tools/striker-auto-initialize-all @@ -111,21 +111,314 @@ sub striker_stage2 # Initialize nodes and DR. initialize_machines($anvil); + # Configure the network on the machines. + configure_machine_networks($anvil); + # Run the manifest(s). run_manifests($anvil); return(0); } +# This makes sure nodes and DR hosts have their networks configured. +sub configure_machine_networks +{ + my ($anvil) = @_; + + update_progress($anvil, 70, "job_0265"); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0265"}); + + my $job_uuids = []; + foreach my $anvil_number (sort {$a cmp $b} keys %{$anvil->data->{anvil}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_number => $anvil_number }}); + foreach my $machine_type ("node", "dr") + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { machine_type => $machine_type }}); + foreach my $machine_number (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}}) + { + my $key = $machine_type.$machine_number; + my $machine_host_uuid = $anvil->data->{process}{anvil}{$anvil_number}{$key}{host_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:key' => $key, + 's2:machine_host_uuid' => $machine_host_uuid, + }}); + + my $padded_sequence = $anvil_number; + if (length($padded_sequence) == 1) + { + $padded_sequence = sprintf("%02d", $padded_sequence); + } + my $machine_suffix = $machine_type eq "node" ? "n".sprintf("%02d", $machine_number) : "dr".sprintf("%02d", $machine_number); + my $name_prefix = "a".$padded_sequence; + my $host_name = $anvil->data->{base}{prefix}."-".$name_prefix.$machine_suffix.".".$anvil->data->{base}{domain}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }}); + $anvil->Database->insert_or_update_variables({ + variable_name => "form::config_step2::host_name::value", + variable_value => $host_name, + variable_default => "", + variable_description => "striker_0159", + variable_section => "config_step2", + variable_source_uuid => $machine_host_uuid, + variable_source_table => "hosts", + }); + + $anvil->data->{process}{anvil}{$anvil_number}{$key}{host_name} = $host_name; + + foreach my $network (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}}) + { + # Record the network count. + my $network_count = keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}}; + my $network_key = $network."_count"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:network' => $network, + 's2:network_key' => $network_key, + 's3:network_count' => $network_count, + }}); + + # Store the network counts. + $anvil->Database->insert_or_update_variables({ + variable_name => "form::config_step1::".$network_key."::value", + variable_value => $network_count, + variable_default => "", + variable_description => "striker_0163", + variable_section => "config_step1", + variable_source_uuid => $machine_host_uuid, + variable_source_table => "hosts", + }); + + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network => $network }}); + foreach my $network_number (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}}) + { + my $network_name = $network.$network_number; + my $ip_address = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{ip}; + my $subnet_mask = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{subnet_mask}; + my $link1_mac = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{'link'}{1}{mac}; + my $link2_mac = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{'link'}{2}{mac}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:network_name' => $network_name, + 's2:ip_address' => $ip_address, + 's3:subnet_mask' => $subnet_mask, + 's4:link1_mac' => $link1_mac, + 's5:link2_mac' => $link2_mac, + }}); + + my $bridge_key = $network_name."_create_bridge"; + my $ip_key = $network_name."_ip"; + my $subnet_key = $network_name."_subnet_mask"; + my $link1_key = $network_name."_link1_mac_to_set"; + my $link2_key = $network_name."_link2_mac_to_set"; + my $say_bridge = "#!string!unit_0002!#"; + my $create_bridge = 0; + if ($network ne "sn") + { + # We're making a bridge. + $say_bridge = "#!string!unit_0001!#"; + $create_bridge = 1; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:bridge_key' => $bridge_key, + 's2:create_bridge' => $create_bridge, + 's3:say_bridge' => $say_bridge, + 's4:ip_key' => $ip_key, + 's5:subnet_key' => $subnet_key, + 's6:link1_mac' => $link1_mac, + 's7:link2_mac' => $link2_mac, + }}); + + # Store the link info + $anvil->Database->insert_or_update_variables({ + variable_name => "form::config_step2::".$link1_key."::value", + variable_value => $link1_mac, + variable_default => "", + variable_description => "striker_0156", + variable_section => "config_step2", + variable_source_uuid => $machine_host_uuid, + variable_source_table => "hosts", + }); + $anvil->Database->insert_or_update_variables({ + variable_name => "form::config_step2::".$link2_key."::value", + variable_value => $link2_mac, + variable_default => "", + variable_description => "striker_0157", + variable_section => "config_step2", + variable_source_uuid => $machine_host_uuid, + variable_source_table => "hosts", + }); + + # Store the bridge variables + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_bridge => $create_bridge }}); + $anvil->Database->insert_or_update_variables({ + variable_name => "form::config_step2::".$bridge_key."::value", + variable_value => $create_bridge, + variable_default => "", + variable_description => "striker_0158", + variable_section => "config_step2", + variable_source_uuid => $machine_host_uuid, + variable_source_table => "hosts", + }); + + # Store the IP variables + $anvil->Database->insert_or_update_variables({ + variable_name => "form::config_step2::".$ip_key."::value", + variable_value => $ip_address, + variable_default => "", + variable_description => "striker_0153,!!say_network!".$network_name."!!", + variable_section => "config_step2", + variable_source_uuid => $machine_host_uuid, + variable_source_table => "hosts", + }); + + # Store the subnet mask variables + $anvil->Database->insert_or_update_variables({ + variable_name => "form::config_step2::".$subnet_key."::value", + variable_value => $subnet_mask, + variable_default => "", + variable_description => "striker_0154,!!say_network!".$network_name."!!", + variable_section => "config_step2", + variable_source_uuid => $machine_host_uuid, + variable_source_table => "hosts", + }); + + if ($network_name eq $anvil->data->{base}{gateway_network}) + { + # This is the gateway network, store the gateway and DNS + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + network_name => $network_name, + 'base::gateway_network' => $anvil->data->{base}{gateway_network}, + }}); + $anvil->Database->insert_or_update_variables({ + variable_name => "form::config_step2::gateway_interface::value", + variable_value => $anvil->data->{base}{gateway_network}, + variable_default => "", + variable_description => "striker_0155", + variable_section => "config_step2", + variable_source_uuid => $machine_host_uuid, + variable_source_table => "hosts", + }); + + # Gateway IP + $anvil->Database->insert_or_update_variables({ + variable_name => "form::config_step2::gateway::value", + variable_value => $anvil->data->{base}{gateway}, + variable_default => "", + variable_description => "striker_0036", + variable_section => "config_step2", + variable_source_uuid => $machine_host_uuid, + variable_source_table => "hosts", + }); + + $anvil->Database->insert_or_update_variables({ + variable_name => "form::config_step2::dns::value", + variable_value => $anvil->data->{base}{dns}, + variable_default => "", + variable_description => "striker_0038", + variable_section => "config_step2", + variable_source_uuid => $machine_host_uuid, + variable_source_table => "hosts", + }); + } + } + } + + my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ + job_host_uuid => $machine_host_uuid, + job_command => $anvil->data->{path}{exe}{'anvil-configure-host'}, + job_data => "form::config_step2", + job_name => "configure::network", + job_title => "job_0001", + job_description => "job_0071", + job_progress => 0, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { job_uuid => $job_uuid }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0266", variables => { + host_name => $host_name, + job_uuid => $job_uuid, + }}); + } + } + } + + # Now wait until all the machines are accessible by the BCN1 IP. + update_progress($anvil, 75, "job_0267"); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0267"}); + + my $waiting = 1; + while ($waiting) + { + $waiting = 0; + foreach my $anvil_number (sort {$a cmp $b} keys %{$anvil->data->{anvil}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_number => $anvil_number }}); + foreach my $machine_type ("node", "dr") + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { machine_type => $machine_type }}); + foreach my $machine_number (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}}) + { + my $key = $machine_type.$machine_number; + my $host_uuid = $anvil->data->{process}{anvil}{$anvil_number}{$key}{host_uuid}; + my $host_name = $anvil->data->{process}{anvil}{$anvil_number}{$key}{host_name}; + my $bcn1_ip = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{bcn}{1}{ip}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + key => $key, + host_uuid => $host_uuid, + host_name => $host_name, + bcn1_ip => $bcn1_ip, + }}); + + my $access = $anvil->Remote->test_access({ + target => $bcn1_ip, + password => $anvil->data->{base}{password}{startup}, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }}); + + if (not $access) + { + # Try again with the new password, in case it's already + # updated. + my $access = $anvil->Remote->test_access({ + target => $bcn1_ip, + password => $anvil->data->{base}{password}{desired}, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }}); + + if (not $access) + { + # Still nothing, keep waiting. + $waiting = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0268", variables => { + host_name => $host_name, + ip_address => $bcn1_ip, + }}); + } + } + } + } + } + + if ($waiting) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0269"}); + sleep 10 + } + } + + update_progress($anvil, 80, "job_0270"); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0270"}); + + return(0); +} + # By this point, all machines should be initialized and online. sub run_manifests { my ($anvil) = @_; - update_progress($anvil, 70, "job_0257"); + update_progress($anvil, 85, "job_0257"); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0257"}); - # All machines are up! + # Run the manifest now! my $job_uuids = []; foreach my $anvil_number (sort {$a cmp $b} keys %{$anvil->data->{anvil}}) { @@ -136,12 +429,13 @@ sub run_manifests my $node2_host_uuid = $anvil->data->{process}{anvil}{$anvil_number}{node2}{host_uuid}; my $dr1_host_uuid = exists $anvil->data->{process}{anvil}{$anvil_number}{dr1}{host_uuid} ? $anvil->data->{process}{anvil}{$anvil_number}{dr1}{host_uuid} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - 's1:anvil_name' => $anvil_name, - 's2:manifest_name' => $manifest_name, - 's3:manifest_uuid' => $manifest_uuid, - 's4:node1_host_uuid' => $node1_host_uuid, - 's5:node2_host_uuid' => $node2_host_uuid, - 's6:dr1_host_uuid' => $dr1_host_uuid, + 's1:anvil_number' => $anvil_number, + 's2:anvil_name' => $anvil_name, + 's3:manifest_name' => $manifest_name, + 's4:manifest_uuid' => $manifest_uuid, + 's5:node1_host_uuid' => $node1_host_uuid, + 's6:node2_host_uuid' => $node2_host_uuid, + 's7:dr1_host_uuid' => $dr1_host_uuid, }}); my ($anvil_uuid) = $anvil->Database->insert_or_update_anvils({ @@ -196,7 +490,7 @@ sub run_manifests }}); push @{$job_uuids}, $node2_job_uuid; - if ($anvil->data->{cgi}{dr1_host}{value}) + if ($dr1_host_uuid) { my ($dr1_job_uuid) = $anvil->Database->insert_or_update_jobs({ job_host_uuid => $dr1_host_uuid, @@ -218,7 +512,7 @@ sub run_manifests } } - update_progress($anvil, 80, "job_0257"); + update_progress($anvil, 90, "job_0257"); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0257"}); # Wait for jobs to complete. @@ -230,7 +524,7 @@ sub run_manifests { my $return = $anvil->Database->get_job_details({job_uuid => $job_uuid}); my $job_progress = $return->{job_progress}; - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0259", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0261", variables => { job_uuid => $job_uuid, progress => $job_progress, }}); @@ -277,7 +571,7 @@ sub initialize_machines $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { machine_type => $machine_type }}); foreach my $machine_number (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}}) { - my $startup_ip = $anvil->data->{anvil}{$anvil_number}{$machine_type}{start_ip}; + my $startup_ip = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{startup_ip}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { machine_number => $machine_number, startup_ip => $startup_ip, @@ -288,15 +582,15 @@ sub initialize_machines my $machine_ips = []; push @{$machine_ips}, $startup_ip; - foreach my $network (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}{network}}) + foreach my $network (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}}) { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network => $network }}); - foreach my $network_number (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}{network}{$network}}) + foreach my $network_number (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}}) { my $ip_address = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{ip}; my $subnet_mask = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{subnet_mask}; my $link1_mac = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{'link'}{1}{mac}; - my $link2_mac = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{'link'}{1}{mac}; + my $link2_mac = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{'link'}{2}{mac}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:ip_address' => $ip_address, 's2:subnet_mask' => $subnet_mask, @@ -305,9 +599,10 @@ sub initialize_machines }}); # No sense pushing SN IPs - if ($network != /^sn/) + if ($network !~ /^sn/) { push @{$machine_ips}, $ip_address; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ip_count => @{$machine_ips} }}); } if (not $machine_host_uuid) @@ -336,7 +631,7 @@ sub initialize_machines { # Which IP should be use to initialize? my $use_ip = ""; - my $use_password = $anvil->data->{base}{password}{current}; + my $use_password = $anvil->data->{base}{password}{startup}; until ($use_ip) { foreach my $ip_address (@{$machine_ips}) @@ -402,7 +697,7 @@ sub initialize_machines my $say_host_name = $anvil->data->{base}{prefix}."-".$name_prefix.$machine_suffix.".".$anvil->data->{base}{domain}; # Store the peer's password as the job data - my $job_data = "password=".$anvil->data->{base}{password}{desired}."\n"; + my $job_data = "password=".$anvil->data->{base}{password}{startup}."\n"; $job_data .= "rh_password=".$anvil->data->{base}{rhn}{password}."\n"; $job_data .= "rh_user=".$anvil->data->{base}{rhn}{user}."\n"; $job_data .= "host_ip_address=".$use_ip."\n"; @@ -454,15 +749,16 @@ sub initialize_machines $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { machine_type => $machine_type }}); foreach my $machine_number (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}}) { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { machine_number => $machine_number }}); my $machine_host_uuid = ""; - foreach my $network (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}{network}}) + foreach my $network (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}}) { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network => $network }}); - foreach my $network_number (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}{network}{$network}}) + foreach my $network_number (sort {$a cmp $b} keys %{$anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}}) { next if $machine_host_uuid; my $link1_mac = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{'link'}{1}{mac}; - my $link2_mac = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{'link'}{1}{mac}; + my $link2_mac = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{'link'}{2}{mac}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:link1_mac' => $link1_mac, 's2:link2_mac' => $link2_mac, @@ -619,7 +915,7 @@ sub create_manifest my $ip_address = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{ip}; my $subnet_mask = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{subnet_mask}; my $link1_mac = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{'link'}{1}{mac}; - my $link2_mac = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{'link'}{1}{mac}; + my $link2_mac = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{'link'}{2}{mac}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:ip_address' => $ip_address, 's2:subnet_mask' => $subnet_mask, @@ -646,7 +942,7 @@ sub create_manifest my $ip_key = $machine."_".$network_name."_ip"; $anvil->data->{cgi}{$network_key}{value} = $anvil->Network->get_network({ip => $ip_address, subnet_mask => $subnet_mask}); $anvil->data->{cgi}{$subnet_key}{value} = $subnet_mask; - $anvil->data->{cgi}{$gateway_key}{value} = $anvil->data->{base}{gateway_network} eq $network_name ? $anvil->data->{base}{gateway_network} : ""; + $anvil->data->{cgi}{$gateway_key}{value} = $anvil->data->{base}{gateway_network} eq $network_name ? $anvil->data->{base}{gateway} : ""; $anvil->data->{cgi}{$ip_key}{value} = $ip_address; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "cgi::${network_key}::value" => $anvil->data->{cgi}{$network_key}{value}, @@ -658,7 +954,7 @@ sub create_manifest if ($network_name eq "bcn1") { my $ipmi_ip_key = $machine."_ipmi_ip"; - $anvil->data->{cgi}{$ipmi_ip_key}{value} = $anvil->data->{$machine_type}{$machine_number}{network}{$network}{$network_number}{ipmi_ip}; + $anvil->data->{cgi}{$ipmi_ip_key}{value} = $anvil->data->{anvil}{$anvil_number}{$machine_type}{$machine_number}{network}{$network}{$network_number}{ipmi_ip}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "cgi::${ipmi_ip_key}::value" => $anvil->data->{cgi}{$ipmi_ip_key}{value}, }}); @@ -930,7 +1226,15 @@ sub merge_peer_striker } } - if (not $peer_host_uuid) + if ($peer_host_uuid) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0264", variables => { + number => $striker_number, + peer_host_name => $anvil->Database->get_host_from_uuid({host_uuid => $peer_host_uuid}), + }}); + next; + } + else { ### Add the peer. # First, wait for access. @@ -988,39 +1292,46 @@ fi; # Find the IP we used to reach the peer. my $our_ip = ""; - $anvil->Network->load_ips({ - host => "local", - host_uuid => $anvil->Get->host_uuid(), - }); - foreach my $interface_name (sort {$a cmp $b} keys %{$anvil->data->{network}{'local'}{interface}}) + until($our_ip) { - my $local_ip = $anvil->data->{network}{'local'}{interface}{$interface_name}{ip}; - my $local_subnet_mask = $anvil->data->{network}{'local'}{interface}{$interface_name}{subnet_mask}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - interface_name => $interface_name, - local_ip => $local_ip, - local_subnet_mask => $local_subnet_mask, - }}); - next if $local_subnet_mask ne $subnet_mask; + $anvil->Network->load_ips({ + host => "local", + host_uuid => $anvil->Get->host_uuid(), + }); + foreach my $interface_name (sort {$a cmp $b} keys %{$anvil->data->{network}{'local'}{interface}}) + { + my $local_ip = $anvil->data->{network}{'local'}{interface}{$interface_name}{ip}; + my $local_subnet_mask = $anvil->data->{network}{'local'}{interface}{$interface_name}{subnet_mask}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + interface_name => $interface_name, + local_ip => $local_ip, + local_subnet_mask => $local_subnet_mask, + }}); + next if $local_subnet_mask ne $subnet_mask; + + # See if this IP is in the same subnet. + my $first = NetAddr::IP->new($local_ip."/".$local_subnet_mask); + my $second = NetAddr::IP->new($ip."/".$subnet_mask); + if ($second->within($first)) + { + # Found the IP to tell the peer to use + $our_ip = $local_ip; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { our_ip => $our_ip }}); + last; + } + } - # See if this IP is in the same subnet. - my $first = NetAddr::IP->new($local_ip."/".$local_subnet_mask); - my $second = NetAddr::IP->new($ip."/".$subnet_mask); - if ($second->within($first)) + if (not $our_ip) { - # Found the IP to tell the peer to use - $our_ip = $local_ip; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { our_ip => $our_ip }}); - last; + # One of the machines hasn't updated IP information yet. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0271", variables => { + number => $striker_number, + ip => $ip, + }}); + sleep 10; } } - if (not $our_ip) - { - # wtf; - die; - } - # Register a job and then wait for it to show up in our database. my $job_data = "password=".$anvil->data->{base}{password}{desired}."\n"; $job_data .= "peer_job_command=".$anvil->data->{path}{exe}{'striker-manage-peers'}." --add --host-uuid ".$anvil->Get->host_uuid." --host ".$our_ip." --port 5432 --ping 1"; diff --git a/tools/striker-auto-initialize-all.example b/tools/striker-auto-initialize-all.example index a997faae..d130da34 100644 --- a/tools/striker-auto-initialize-all.example +++ b/tools/striker-auto-initialize-all.example @@ -32,7 +32,7 @@ base::gateway = 192.168.122.1 base::gateway_network = ifn1 # This is the startup password for newly built nodes and DR hosts we'll be integrating. -base::password::current = Initial1 +base::password::startup = Initial1 # This is the password to set for all systems. base::password::desired = super secret password @@ -77,43 +77,43 @@ striker::2::network::bcn::1::link::2::mac = 52:54:00:30:f9:db ups::1::name = an-ups01 ups::1::agent = scan-apc-ups ups::1::ip_address = 10.201.3.1 -ups::2::name = an-ups01 +ups::2::name = an-ups02 ups::2::agent = scan-apc-ups -ups::2::ip_address = 10.201.3.1 +ups::2::ip_address = 10.201.3.2 ### Fence device(s) # NOTE: Do NOT specify fence_ipmilan! It is detected / configured automatically. # The fence_arguments only needs to contain variable="value" pairs that don't have a useful default. fence::1::name = gravitar -fence::1::agent = virsh +fence::1::agent = fence_virsh fence::1::arguments = action="reboot" ip="192.168.122.1" password_script="/root/gravitar_password" username="root" # Example PDUs #fence::2::name = an-pdu01 -#fence::2::agent = apc_snmp +#fence::2::agent = fence_apc_snmp #fence::2::arguments = action="reboot" ip="10.201.2.1" #fence::3::name = an-pdu02 -#fence::3::agent = apc_snmp +#fence::3::agent = fence_apc_snmp #fence::3::arguments = action="reboot" ip="10.201.2.2" # Anvil description anvil::1::description = "Test Anvil! running on gravitar" ### Nodes -anvil::1::node::1::start_ip = 192.168.122.235 -anvil::1::node::1::network::ifn::1::ip = 192.168.122.11 -anvil::1::node::1::network::ifn::1::subnet_mask = 255.255.255.0 -anvil::1::node::1::network::ifn::1::link::1::mac = 52:54:00:17:d6:0b -anvil::1::node::1::network::ifn::1::link::2::mac = 52:54:00:c4:87:b6 +anvil::1::node::1::startup_ip = 192.168.122.235 anvil::1::node::1::network::bcn::1::ip = 10.201.10.1 anvil::1::node::1::network::bcn::1::ipmi_ip = 10.201.11.1 anvil::1::node::1::network::bcn::1::subnet_mask = 255.255.0.0 -anvil::1::node::1::network::bcn::1::link::1::mac = 52:54:00:d2:6b:0b -anvil::1::node::1::network::bcn::1::link::2::mac = 52:54:00:01:11:0e +anvil::1::node::1::network::bcn::1::link::1::mac = 52:54:00:7e:b8:9e +anvil::1::node::1::network::bcn::1::link::2::mac = 52:54:00:23:e8:46 anvil::1::node::1::network::sn::1::ip = 10.101.4.1 anvil::1::node::1::network::sn::1::subnet_mask = 255.255.0.0 -anvil::1::node::1::network::sn::1::link::1::mac = 52:54:00:7e:b8:9e -anvil::1::node::1::network::sn::1::link::2::mac = 52:54:00:23:e8:46 +anvil::1::node::1::network::sn::1::link::1::mac = 52:54:00:d2:6b:0b +anvil::1::node::1::network::sn::1::link::2::mac = 52:54:00:01:11:0e +anvil::1::node::1::network::ifn::1::ip = 192.168.122.11 +anvil::1::node::1::network::ifn::1::subnet_mask = 255.255.255.0 +anvil::1::node::1::network::ifn::1::link::1::mac = 52:54:00:17:d6:0b +anvil::1::node::1::network::ifn::1::link::2::mac = 52:54:00:c4:87:b6 # Fence (IPMI configured automatically), same fence types grouped automatically anvil::1::node::1::fence::1::name = gravitar anvil::1::node::1::fence::1::port = an-a01n01 @@ -125,11 +125,7 @@ anvil::1::node::1::fence::1::port = an-a01n01 #anvil::1::node::1::ups::1::name = an-ups01 #anvil::1::node::1::ups::2::name = an-ups02 -anvil::1::node::2::start_ip = 192.168.122.185 -anvil::1::node::2::network::ifn::1::ip = 192.168.122.12 -anvil::1::node::2::network::ifn::1::subnet_mask = 255.255.255.0 -anvil::1::node::2::network::ifn::1::link::1::mac = 52:54:00:09:b7:90 -anvil::1::node::2::network::ifn::1::link::2::mac = 52:54:00:78:a3:41 +anvil::1::node::2::startup_ip = 192.168.122.185 anvil::1::node::2::network::bcn::1::ip = 10.201.10.2 anvil::1::node::2::network::bcn::1::ipmi_ip = 10.201.11.2 anvil::1::node::2::network::bcn::1::subnet_mask = 255.255.0.0 @@ -139,16 +135,16 @@ anvil::1::node::2::network::sn::1::ip = 10.101.4.2 anvil::1::node::2::network::sn::1::subnet_mask = 255.255.0.0 anvil::1::node::2::network::sn::1::link::1::mac = 52:54:00:79:1c:ce anvil::1::node::2::network::sn::1::link::2::mac = 52:54:00:5d:d3:6d +anvil::1::node::2::network::ifn::1::ip = 192.168.122.12 +anvil::1::node::2::network::ifn::1::subnet_mask = 255.255.255.0 +anvil::1::node::2::network::ifn::1::link::1::mac = 52:54:00:09:b7:90 +anvil::1::node::2::network::ifn::1::link::2::mac = 52:54:00:78:a3:41 # Set for UPSes powering the node, if any anvil::1::node::2::ups::1::name = an-ups01 anvil::1::node::2::ups::2::name = an-ups02 ### DR host (optional) -anvil::1::dr::1::start_ip = 192.168.122.153 -anvil::1::dr::1::network::ifn::1::ip = 192.168.122.10 -anvil::1::dr::1::network::ifn::1::subnet_mask = 255.255.255.0 -anvil::1::dr::1::network::ifn::1::link::1::mac = 52:54:00:6d:05:5e -anvil::1::dr::1::network::ifn::1::link::2::mac = 52:54:00:7e:87:ec +anvil::1::dr::1::startup_ip = 192.168.122.153 anvil::1::dr::1::network::bcn::1::ip = 10.201.10.3 anvil::1::dr::1::network::bcn::1::ipmi_ip = 10.201.11.3 anvil::1::dr::1::network::bcn::1::subnet_mask = 255.255.0.0 @@ -158,3 +154,7 @@ anvil::1::dr::1::network::sn::1::ip = 10.101.4.3 anvil::1::dr::1::network::sn::1::subnet_mask = 255.255.0.0 anvil::1::dr::1::network::sn::1::link::1::mac = 52:54:00:d6:f8:8d anvil::1::dr::1::network::sn::1::link::2::mac = 52:54:00:51:7b:b6 +anvil::1::dr::1::network::ifn::1::ip = 192.168.122.13 +anvil::1::dr::1::network::ifn::1::subnet_mask = 255.255.255.0 +anvil::1::dr::1::network::ifn::1::link::1::mac = 52:54:00:6d:05:5e +anvil::1::dr::1::network::ifn::1::link::2::mac = 52:54:00:7e:87:ec diff --git a/tools/striker-initialize-host b/tools/striker-initialize-host index 1a8ac597..9884bdb7 100755 --- a/tools/striker-initialize-host +++ b/tools/striker-initialize-host @@ -342,7 +342,7 @@ sub add_repos # Add the local repo. my $repo = $anvil->Striker->get_local_repo({debug => 3}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { repo => $repo }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { repo => $repo }}); if ($repo) { @@ -353,7 +353,7 @@ sub add_repos $repo EOF "; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); my ($output, $error, $return_code) = $anvil->Remote->call({ debug => 3, shell_call => $shell_call, @@ -667,6 +667,38 @@ EOF return_code => $return_code, }}); + # Install the Alteeve repo, if possible. There may be no Internet access, so it's OK if this fails. + if (not -e $anvil->data->{path}{config}{'alteeve-el8.repo'}) + { + my ($alteeve_access) = $anvil->Network->check_internet({ + debug => 2, + domains => ["alteeve.com"], + password => $anvil->data->{data}{password}, + port => $anvil->data->{data}{ssh_port}, + target => $anvil->data->{data}{host_ip_address}, + remote_user => "root", + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { alteeve_access => $alteeve_access }}); + if ($alteeve_access) + { + $shell_call = $anvil->data->{path}{exe}{'dnf'}." -y install ".$anvil->data->{path}{urls}{alteeve_repo}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); + ($output, $error, $return_code) = $anvil->Remote->call({ + debug => 3, + shell_call => $shell_call, + password => $anvil->data->{data}{password}, + port => $anvil->data->{data}{ssh_port}, + target => $anvil->data->{data}{host_ip_address}, + remote_user => "root", + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + error => $error, + return_code => $return_code, + }}); + } + } + # Install the anvil package now. my $package = $anvil->data->{data}{type} eq "dr" ? "anvil-dr" : "anvil-node"; $anvil->data->{job}{progress} += 5;