diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 516602c9..1d9bc06b 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -15214,7 +15214,7 @@ sub _archive_table # We don't archive the OUI table, it generally has more entries than needed to trigger the archive, but it's needed. if (($table eq "oui") or ($table eq "states")) { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0459", variables => { table => $table }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, priority => "err", key => "log_0459", variables => { table => $table }}); return(0); } diff --git a/Anvil/Tools/Get.pm b/Anvil/Tools/Get.pm index 4e7c665d..1eaf322d 100644 --- a/Anvil/Tools/Get.pm +++ b/Anvil/Tools/Get.pm @@ -1809,22 +1809,31 @@ sub os_type ### NOTE: Examples; # Red Hat Enterprise Linux release 8.0 Beta (Ootpa) # CentOS Stream release 8 - - ### NOTE: This can be called before 'rsync' is called, so we use 'cat' - # Read in the /etc/redhat-release file - my ($release, $error, $return_code) = $anvil->Remote->call({ - debug => $debug, - shell_call => $anvil->data->{path}{exe}{cat}." ".$anvil->data->{path}{data}{'redhat-release'}, - port => $port, - password => $password, - remote_user => $remote_user, - target => $target, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - release => $release, - error => $error, - return_code => $return_code, - }}); + my $release = ""; + if ($target) + { + ### NOTE: This can be called before 'rsync' is called, so we use 'cat' + # Read in the /etc/redhat-release file + ($release, my $error, my $return_code) = $anvil->Remote->call({ + debug => $debug, + shell_call => $anvil->data->{path}{exe}{cat}." ".$anvil->data->{path}{data}{'redhat-release'}, + port => $port, + password => $password, + remote_user => $remote_user, + target => $target, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + release => $release, + error => $error, + return_code => $return_code, + }}); + } + else + { + # Local call. + $release = $anvil->Storage->read_file({debug => $debug, file => $anvil->data->{path}{data}{'redhat-release'}}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { release => $release }}); + } if ($release =~ /Red Hat Enterprise Linux .* (\d+)\./) { # RHEL, with the major version number appended @@ -1843,7 +1852,7 @@ sub os_type $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { os_type => $os_type }}); } - (my $output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{uname}." --hardware-platform"}); + my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{uname}." --hardware-platform"}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output, return_code => $return_code }}); if ($output) { diff --git a/Anvil/Tools/Remote.pm b/Anvil/Tools/Remote.pm index 47f25d49..f3d84fce 100644 --- a/Anvil/Tools/Remote.pm +++ b/Anvil/Tools/Remote.pm @@ -328,8 +328,14 @@ sub call } if (not $target) { - # No target - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Remote->call()", parameter => "target" }}); + # No target, this should not happen... + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0239", variables => { + remote_user => $remote_user, + port => $port, + 'close' => $close, + secure => $secure, + shell_call => (not $secure) ? $shell_call : $anvil->Log->is_secure($shell_call), + }}); return("!!error!!"); } if (not $remote_user) diff --git a/share/words.xml b/share/words.xml index dc897fe8..c88ae052 100644 --- a/share/words.xml +++ b/share/words.xml @@ -324,6 +324,14 @@ Output (if any): There appears to be no resource data in the database for the host: [#!variable!host_name!#]. Has ScanCore run and, specifically, has 'scan-hardware' run yet? Unable to provide available resources for this Anvil! system. The resource name: [#!variable!resource_name!#] already exists, and 'force_unique' is set. This is likely a name conflict, returning '!!error!!'. This node is not yet fully in the cluster. Sleeping for a bit, then we'll exit. The job will try again shortly after. + call() was called without a target being set. Other values passed in that may help locate the source of this call: +- remote_user: [#!variable!remote_user!#] +- port: ...... [#!variable!port!#] +- close: ..... [#!variable!close!#] +- secure: .... [#!variable!secure!#] +- shell_call: [#!variable!shell_call!#] + ]]> + Usage: [#!variable!program!# --config /path/to/config]. @@ -1337,7 +1345,7 @@ The file: [#!variable!file!#] needs to be updated. The difference is: The host: [#!variable!host_name!#] was powered off for thermal reasons. All available thermal sensors read as OK now. Booting it back up now. The file: [#!variable!file_path!#] isn't on (or isn't the right size on) Striker: [#!variable!host_name!#]. Not using it to pull from. The job: [#!variable!job_uuid!#] was assigned to our Anvil! and this is the primary node. Assigning the job to this machine. - #!free!# + I was about to start: [#!variable!command!#], but I last tried to run this: [#!variable!last_start!#] seconds ago. We'll wait at least a minute before we try to run it again. The LV(s) behind the resource: [#!variable!resource!#] have had their DRBD metadata created successfully. The LV(s) behind the resource: [#!variable!resource!#] have been forced to primary to initialize the resource. Asked to validate that the server: [#!variable!server!#] is able to run. diff --git a/tools/anvil-daemon b/tools/anvil-daemon index 80a58adc..c3682e56 100755 --- a/tools/anvil-daemon +++ b/tools/anvil-daemon @@ -1026,9 +1026,27 @@ sub run_jobs # If the job is not running, start it. if ((not $job_picked_up_by) && ($job_progress ne "100") && (not $anvil->data->{switches}{'no-start'})) { - # Start the job, appending '--job-uuid' to the command. my $command = $job_command." --job-uuid ".$job_uuid; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0210", variables => { command => $command }}); + + # Have we started this job recently? + if (exists $anvil->data->{jobs}{$job_uuid}{started}) + { + my $last_start = time - $anvil->data->{jobs}{$job_uuid}{started}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { last_start => $last_start }}); + + if ($last_start < 60) + { + # Skip, Started too recently. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0578", variables => { + command => $command, + last_start => $last_start, + }}); + next; + } + } + + # Start the job, appending '--job-uuid' to the command. ($anvil->data->{jobs}{handles}{$job_uuid}, my $return_code) = $anvil->System->call({ background => 1, stdout_file => "/tmp/anvil.job.".$job_uuid.".stdout", @@ -1042,6 +1060,10 @@ sub run_jobs # Log the PID (the job should update the database). my $pid = $anvil->data->{jobs}{handles}{$job_uuid}->pid(); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { pid => $pid }}); + + # Record that we've tried to start this job, so that we don't try to restart it for any reason for at least a minute. + $anvil->data->{jobs}{$job_uuid}{started} = time; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'jobs::$job_uuid::started' => $anvil->data->{jobs}{$job_uuid}{started} }}); } } diff --git a/tools/striker-auto-initialize-all b/tools/striker-auto-initialize-all index c7a37d11..967064eb 100755 --- a/tools/striker-auto-initialize-all +++ b/tools/striker-auto-initialize-all @@ -23,34 +23,34 @@ if (($running_directory =~ /^\./) && ($ENV{PWD})) =cut Striker initialization; - variable_uuid | variable_name | variable_value | variable_default | variable_description | variable_section | variable_source_uuid | variable_source_table | modified_date ---------------------------------------+--------------------------------------------------+-------------------------+------------------+----------------------+------------------+--------------------------------------+-----------------------+------------------------------- - b397973f-e871-4140-baab-6d77611b4edd | form::config_step1::organization::value | Alteeve | | striker_0004 | config_step1 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:26:22.938934-05 - 55261634-0495-448f-a41f-24af7ed57911 | form::config_step1::prefix::value | di | | striker_0006 | config_step1 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:26:22.938934-05 - 192d3636-3e66-46cb-bf60-5e11c35220dc | form::config_step1::domain::value | digimer.ca | | striker_0008 | config_step1 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:26:22.938934-05 - e53cc9f3-6d8e-4622-8b56-7df8b90cdd24 | form::config_step1::ifn_count::value | 1 | | striker_0012 | config_step1 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:26:22.938934-05 - 823222e7-e55a-48e6-8b16-cf2046c880b4 | form::config_step1::sequence::value | 2 | | striker_0010 | config_step1 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:27:39.491016-05 - - 858c40b8-6612-4922-b824-1eb6ce3c74cc | form::config_step2::host_name::value | di-striker02.digimer.ca | | striker_0017 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 - 9a176810-403b-4180-b502-90a8529c43bd | form::config_step2::striker_user::value | admin | | striker_0032 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 - 8bdf6cc8-3799-4759-8e7d-7d9ebbc74d6a | form::config_step2::striker_password::value | super secret password | | striker_0034 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 - 0c22eba6-438a-4a7d-8a4b-c473e24da790 | form::config_step2::dns::value | 8.8.8.8, 8.8.4.4 | | striker_0038 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 - cd443969-30ee-45ca-a189-8546e945dd13 | form::config_step2::bcn1_ip::value | 10.201.4.2 | | striker_0024 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 - 8f618047-59e9-4d56-b842-ea6e43c6aac1 | form::config_step2::bcn1_subnet_mask::value | 255.255.0.0 | | striker_0025 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 - 7b6d2213-3852-42d2-9a3c-f31f2197c6b2 | form::config_step2::bcn1_link1_mac_to_set::value | 52:54:00:8b:d6:82 | | striker_0029 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 - a1137f58-22c3-4d12-bfbd-1f88c1cb0956 | form::config_step2::ifn1_subnet_mask::value | 255.255.255.0 | | striker_0025 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 - 52842c37-4f8b-4d77-9712-c911d4abaf11 | form::config_step2::ifn1_link1_mac_to_set::value | 52:54:00:c0:f2:7c | | striker_0029 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 - 42f33705-8fbb-48eb-82cf-e0ec2f2cfebe | form::config_step2::ifn1_ip::value | 192.168.122.12 | | striker_0024 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:30:16.688974-05 - 7110e4e9-6cc7-4a9b-a0c8-8e234ab3fbc9 | form::config_step2::gateway::value | 192.168.122.1 | | striker_0036 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:30:16.688974-05 - 7efc6c79-4806-478e-ae55-b9d09e2f23aa | form::config_step2::gateway_interface::value | ifn1 | | | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:30:16.688974-05 + +# Figure out if we're striker '1' or '2' by matching found MAC addresses against configured MAC addresses. +# Striker 1 will, after configuring itself, reboot and wait for access to Striker 2. Then it will bond the two and begin configuring nodes (and DR, if defined) + data source | variable_name | variable_value | variable_default | variable_description | variable_section | variable_source_uuid | variable_source_table | modified_date +------------------------------------------+--------------------------------------------------+-------------------------+------------------+----------------------+------------------+--------------------------------------+-----------------------+------------------------------- +base::organization_name | form::config_step1::organization::value | Alteeve | | striker_0004 | config_step1 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:26:22.938934-05 +base::prefix | form::config_step1::prefix::value | di | | striker_0006 | config_step1 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:26:22.938934-05 +base::domain | form::config_step1::domain::value | digimer.ca | | striker_0008 | config_step1 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:26:22.938934-05 +keys %{striker}{1}{network}{ifn} | form::config_step1::ifn_count::value | 1 | | striker_0012 | config_step1 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:26:22.938934-05 +# Match MAC for Striker sequence | form::config_step1::sequence::value | 2 | | striker_0010 | config_step1 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:27:39.491016-05 +-striker0. | form::config_step2::host_name::value | di-striker02.digimer.ca | | striker_0017 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 +# Hard code to 'admin' | form::config_step2::striker_user::value | admin | | striker_0032 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 +base::password::desired | form::config_step2::striker_password::value | super secret password | | striker_0034 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 +base::dns | form::config_step2::dns::value | 8.8.8.8, 8.8.4.4 | | striker_0038 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 +base::gateway | form::config_step2::gateway::value | 192.168.122.1 | | striker_0036 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:30:16.688974-05 +base::interface | form::config_step2::gateway_interface::value | ifn1 | | | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:30:16.688974-05 +striker::1::network::bcn::1::ip | form::config_step2::bcn1_ip::value | 10.201.4.2 | | striker_0024 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 +striker::1::network::bcn::1::subnet_mask | form::config_step2::bcn1_subnet_mask::value | 255.255.0.0 | | striker_0025 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 +striker::1::network::bcn::1::link::1::mac | form::config_step2::bcn1_link1_mac_to_set::value | 52:54:00:8b:d6:82 | | striker_0029 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 +striker::1::network::ifn::1::subnet_mask | form::config_step2::ifn1_subnet_mask::value | 255.255.255.0 | | striker_0025 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 +striker::1::network::ifn::1::link::1::mac | form::config_step2::ifn1_link1_mac_to_set::value | 52:54:00:c0:f2:7c | | striker_0029 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:29:59.317812-05 +striker::1::network::ifn::1::ip | form::config_step2::ifn1_ip::value | 192.168.122.12 | | striker_0024 | config_step2 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | hosts | 2021-02-16 20:30:16.688974-05 + anvil=# SELECT * FROM jobs WHERE job_uuid = '158e8384-eac7-4289-8f70-bc43eaf8b017'; - job_uuid | job_host_uuid | job_command | job_data | job_picked_up_by | job_picked_up_at | job_updated | job_name | job_progress | job_title | job_description | job_status | m -odified_date ---------------------------------------+--------------------------------------+--------------------------------+--------------------+------------------+------------------+-------------+--------------------+--------------+-----------+-----------------+------------+---------- ---------------------- - 158e8384-eac7-4289-8f70-bc43eaf8b017 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | /usr/sbin/anvil-configure-host | form::config_step2 | 0 | 0 | 1613525509 | configure::network | 0 | job_0001 | job_0002 | | 2021-02-1 -6 20:31:49.108908-05 + job_uuid | job_host_uuid | job_command | job_data | job_picked_up_by | job_picked_up_at | job_updated | job_name | job_progress | job_title | job_description | job_status | modified_date +--------------------------------------+--------------------------------------+--------------------------------+--------------------+------------------+------------------+-------------+--------------------+--------------+-----------+-----------------+------------+------------------------------- + 158e8384-eac7-4289-8f70-bc43eaf8b017 | a64c477b-b0a1-4985-9968-f4b46d75fb0c | /usr/sbin/anvil-configure-host | form::config_step2 | 0 | 0 | 1613525509 | configure::network | 0 | job_0001 | job_0002 | | 2021-02-16 20:31:49.108908-05 # At this point, the Striker should be coming up at the IP. Once both/all Strikers are up, update their anvil.conf to add each other's UUID database entry. @@ -74,6 +74,18 @@ anvil=# SELECT * FROM jobs ; | | | host_name=di-a02n01.alteeve.com+| | | | | | | | | | | | | | | | | | | | | (1 row) +anvil=# SELECT * FROM jobs WHERE job_progress != 100; + job_uuid | job_host_uuid | job_command | job_data | job_picked_up_by | job_picked_up_at | job_updated | job_name | job_progress | job_title | job_description | job_status | modified_date +--------------------------------------+--------------------------------------+-----------------------------------+----------------------------------+------------------+------------------+-------------+---------------------------------+--------------+-----------+-----------------+------------+------------------------------- + 6c21d7ca-1c86-4389-9821-0ae945529754 | 46c00674-fea2-44af-981d-2833d5c8270a | /usr/sbin/striker-initialize-host | password=Initial1 +| 0 | 0 | 1614047831 | initialize::dr::192.168.122.141 | 0 | job_0021 | job_0022 | | 2021-02-22 21:37:11.082112-05 + | | | rh_password= +| | | | | | | | | + | | | rh_user= +| | | | | | | | | + | | | host_ip_address=192.168.122.141 +| | | | | | | | | + | | | ssh_port=22 +| | | | | | | | | + | | | type=dr +| | | | | | | | | + | | | host_name=di-a02dr01.alteeve.com+| | | | | | | | | + | | | | | | | | | | | | +(1 row) =cut @@ -91,4 +103,19 @@ if (not $anvil->data->{sys}{database}{connections}) } $anvil->Get->switches; +# Read in the config file +if ((not $anvil->data->{switches}{config}) or (not -f $anvil->data->{switches}{config})) +{ + # Print the usage. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0240", variables => { program => $THIS_FILE }}); + $anvil->nice_exit({exit_code => 1}); +} + +# Load the config. +$anvil->Storage->read_config({file => $anvil->data->{switches}{config}}); + +print Dumper $anvil->data->{base}; +print Dumper $anvil->data->{striker}; + + $anvil->nice_exit({exit_code => 0}); diff --git a/tools/striker-auto-initialize-all.example b/tools/striker-auto-initialize-all.example index 1c9cec3e..7e92e2dd 100644 --- a/tools/striker-auto-initialize-all.example +++ b/tools/striker-auto-initialize-all.example @@ -6,7 +6,15 @@ base::password::desired = super secret password base::organization_name = Alteeve's Niche! base::prefix = an base::domain = alteeve.com +base::dns = 8.8.8.8,8.8.4.4 +base::gateway = 192.168.122.1 +base::interface = ifn1 +# Startup IP is used for peers to find us striker::1::network::startup_ip = 192.168.122.145 striker::1::network::ifn::1::ip = 192.168.122.11 +striker::1::network::ifn::1::subnet_mask = 255.255.255.0 striker::1::network::ifn::1::link::1::mac = aa:bb:cc:dd:ee:ff +striker::1::network::bcn::1::ip = 192.168.122.11 +striker::1::network::bcn::1::subnet_mask = 255.255.255.0 +striker::1::network::bcn::1::link::1::mac = aa:bb:cc:dd:ff:00