Merge pull request #659 from ClusterLabs/update-debugging

Cluster update and node rebuild
main
Digimer 7 months ago committed by GitHub
commit c9ee2cb4f8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 13
      Anvil/Tools.pm
  2. 167
      Anvil/Tools/Database.pm
  3. 18
      Anvil/Tools/Network.pm
  4. 145
      Anvil/Tools/System.pm
  5. 11
      selinux/anvil-subnode.te.in
  6. 12
      share/words.xml
  7. 30
      tools/anvil-configure-host
  8. 244
      tools/anvil-join-anvil

@ -194,9 +194,10 @@ sub new
$anvil->data->{ENV_VALUES}{START_TIME} = Time::HiRes::time;
# Set passed parameters if needed.
my $debug = 3;
my $debug = 2;
my $on_sig_int;
my $on_sig_term;
my $on_sig_alarm;
if (ref($parameter) eq "HASH")
{
# Local parameters...
@ -213,8 +214,9 @@ sub new
$debug = $parameter->{debug};
}
$on_sig_int = $parameter->{on_sig_int};
$on_sig_term = $parameter->{on_sig_term};
$on_sig_int = $parameter->{on_sig_int};
$on_sig_term = $parameter->{on_sig_term};
$on_sig_alarm = $parameter->{on_sig_alarm};
}
elsif ($parameter)
{
@ -234,6 +236,11 @@ sub new
$anvil->catch_sig({signal => "TERM"});
};
$SIG{ALRM} = sub {
$on_sig_alarm->({ debug => $debug }) if (ref($on_sig_alarm) eq "CODE");
$anvil->catch_sig({signal => "ALRM"});
};
# This sets the environment this program is running in.
if ($ENV{SERVER_NAME})

@ -56,6 +56,7 @@ my $THIS_FILE = "Database.pm";
# get_tables_from_schema
# get_power
# get_upses
# get_variables
# initialize
# insert_or_update_alert_overrides
# insert_or_update_anvils
@ -2338,7 +2339,11 @@ sub connect
This cleanly closes any open file handles to all connected databases and clears some internal database related variables.
This method takes no parameters.
Parameters;
=head3 cleanup (optional, default '1')
If set to C<< 1 >> (default), the disconnect will be cleaned up (marked inactive, clear locking, etc). If the DB handle was lost unexpectedly, this is not possible. Set this to C<< 0 >> to prevent this.
=cut
sub disconnect
@ -2349,6 +2354,11 @@ sub disconnect
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->disconnect()" }});
my $cleanup = defined $parameter->{cleanup} ? $parameter->{cleanup} : 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
cleanup => $cleanup,
}});
my $marked_inactive = 0;
foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{database}})
{
@ -2356,14 +2366,19 @@ sub disconnect
next if ((not $anvil->data->{cache}{database_handle}{$uuid}) or ($anvil->data->{cache}{database_handle}{$uuid} !~ /^DBI::db=HASH/));
# Clear locks and mark that we're done running.
if (not $marked_inactive)
if ((not $marked_inactive) && ($cleanup))
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0857", variables => { uuid => $uuid }});
$anvil->Database->mark_active({debug => $debug, set => 0});
$anvil->Database->locking({debug => $debug, release => 1});
$marked_inactive = 1;
}
$anvil->data->{cache}{database_handle}{$uuid}->disconnect;
if ($cleanup)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0858", variables => { uuid => $uuid }});
$anvil->data->{cache}{database_handle}{$uuid}->disconnect;
}
delete $anvil->data->{cache}{database_handle}{$uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
}
@ -7075,6 +7090,138 @@ WHERE
}
=head2 get_variables
This method loads the C<< variables >> table data into memory.
If the record does NOT have a C<< variable_source_table >>, the data will be stored in the hash;
* variables::variable_uuid::<variable_uuid>::global::variable_name = <variable_name>
* variables::variable_uuid::<variable_uuid>::global::variable_value = <variable_value>
* variables::variable_uuid::<variable_uuid>::global::variable_default = <variable_default>
* variables::variable_uuid::<variable_uuid>::global::variable_description = <variable_description> (this is a string key)
* variables::variable_uuid::<variable_uuid>::global::modified_date = <modified_date> (this is a plain text english date and time)
* variables::variable_uuid::<variable_uuid>::global::modified_date_unix = <modified_date_unix> (this is the unix time stamp)
If there is a source table, then the data is stored in the hash;
* variables::source_table::<source_table>::source_uuid::<source_uuid>::variable_uuid::<variable_uuid>::variable_name = <variable_name>
* variables::source_table::<source_table>::source_uuid::<source_uuid>::variable_uuid::<variable_uuid>::variable_value = <variable_value>
* variables::source_table::<source_table>::source_uuid::<source_uuid>::variable_uuid::<variable_uuid>::variable_default = <variable_default>
* variables::source_table::<source_table>::source_uuid::<source_uuid>::variable_uuid::<variable_uuid>::variable_description = <variable_description> (this is a string key)
* variables::source_table::<source_table>::source_uuid::<source_uuid>::variable_uuid::<variable_uuid>::modified_date = <modified_date> (this is a plain text english date and time)
* variables::source_table::<source_table>::source_uuid::<source_uuid>::variable_uuid::<variable_uuid>::modified_date_unix = <modified_date_unix> (this is the unix time stamp)
This method takes no parameters.
=cut
sub get_variables
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->get_variables()" }});
if (exists $anvil->data->{variables})
{
delete $anvil->data->{variables};
}
# Load the power data.
$anvil->Database->get_hosts({debug => $debug});
my $query = "
SELECT
variable_uuid,
variable_name,
variable_value,
variable_default,
variable_description,
variable_section,
variable_source_uuid,
variable_source_table,
modified_date,
round(extract(epoch from modified_date))
FROM
variables
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $variable_uuid = $row->[0];
my $variable_name = $row->[1];
my $variable_value = $row->[2];
my $variable_default = $row->[3];
my $variable_description = $row->[4];
my $variable_section = $row->[5];
my $variable_source_uuid = defined $row->[6] ? $row->[6] : "";
my $variable_source_table = $row->[7];
my $modified_date = $row->[8];
my $modified_date_unix = $row->[9];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
variable_uuid => $variable_uuid,
variable_name => $variable_name,
variable_value => $variable_value,
variable_default => $variable_default,
variable_description => $variable_description,
variable_section => $variable_section,
variable_source_uuid => $variable_source_uuid,
variable_source_table => $variable_source_table,
modified_date => $modified_date,
modified_date_unix => $modified_date_unix,
}});
if ($variable_source_table)
{
# Store it under the associated table
$variable_source_uuid = "--" if not $variable_source_uuid; # This should never be needed, but just in case...
$anvil->data->{variables}{source_table}{$variable_source_table}{source_uuid}{$variable_source_uuid}{variable_uuid}{$variable_uuid}{variable_name} = $variable_name;
$anvil->data->{variables}{source_table}{$variable_source_table}{source_uuid}{$variable_source_uuid}{variable_uuid}{$variable_uuid}{variable_value} = $variable_value;
$anvil->data->{variables}{source_table}{$variable_source_table}{source_uuid}{$variable_source_uuid}{variable_uuid}{$variable_uuid}{variable_default} = $variable_default;
$anvil->data->{variables}{source_table}{$variable_source_table}{source_uuid}{$variable_source_uuid}{variable_uuid}{$variable_uuid}{variable_description} = $variable_description;
$anvil->data->{variables}{source_table}{$variable_source_table}{source_uuid}{$variable_source_uuid}{variable_uuid}{$variable_uuid}{modified_date} = $modified_date;
$anvil->data->{variables}{source_table}{$variable_source_table}{source_uuid}{$variable_source_uuid}{variable_uuid}{$variable_uuid}{modified_date_unix} = $modified_date_unix;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"variables::source_table::${variable_source_table}::source_uuid::${variable_source_uuid}::variable_uuid::${variable_uuid}::variable_name" => $anvil->data->{variables}{source_table}{$variable_source_table}{source_uuid}{$variable_source_uuid}{variable_uuid}{$variable_uuid}{variable_name},
"variables::source_table::${variable_source_table}::source_uuid::${variable_source_uuid}::variable_uuid::${variable_uuid}::variable_value" => $anvil->data->{variables}{source_table}{$variable_source_table}{source_uuid}{$variable_source_uuid}{variable_uuid}{$variable_uuid}{variable_value},
"variables::source_table::${variable_source_table}::source_uuid::${variable_source_uuid}::variable_uuid::${variable_uuid}::variable_default" => $anvil->data->{variables}{source_table}{$variable_source_table}{source_uuid}{$variable_source_uuid}{variable_uuid}{$variable_uuid}{variable_default},
"variables::source_table::${variable_source_table}::source_uuid::${variable_source_uuid}::variable_uuid::${variable_uuid}::variable_description" => $anvil->data->{variables}{source_table}{$variable_source_table}{source_uuid}{$variable_source_uuid}{variable_uuid}{$variable_uuid}{variable_description},
"variables::source_table::${variable_source_table}::source_uuid::${variable_source_uuid}::variable_uuid::${variable_uuid}::modified_date" => $anvil->data->{variables}{source_table}{$variable_source_table}{source_uuid}{$variable_source_uuid}{variable_uuid}{$variable_uuid}{modified_date},
"variables::source_table::${variable_source_table}::source_uuid::${variable_source_uuid}::variable_uuid::${variable_uuid}::modified_date_unix" => $anvil->data->{variables}{source_table}{$variable_source_table}{source_uuid}{$variable_source_uuid}{variable_uuid}{$variable_uuid}{modified_date_unix},
}});
}
else
{
# Global variable
$anvil->data->{variables}{variable_uuid}{$variable_uuid}{global}{variable_name} = $variable_name;
$anvil->data->{variables}{variable_uuid}{$variable_uuid}{global}{variable_value} = $variable_value;
$anvil->data->{variables}{variable_uuid}{$variable_uuid}{global}{variable_default} = $variable_default;
$anvil->data->{variables}{variable_uuid}{$variable_uuid}{global}{variable_description} = $variable_description;
$anvil->data->{variables}{variable_uuid}{$variable_uuid}{global}{modified_date} = $modified_date;
$anvil->data->{variables}{variable_uuid}{$variable_uuid}{global}{modified_date_unix} = $modified_date_unix;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"variables::variable_uuid::${variable_uuid}::global::variable_name" => $anvil->data->{variables}{variable_uuid}{$variable_uuid}{global}{variable_name},
"variables::variable_uuid::${variable_uuid}::global::variable_value" => $anvil->data->{variables}{variable_uuid}{$variable_uuid}{global}{variable_value},
"variables::variable_uuid::${variable_uuid}::global::variable_default" => $anvil->data->{variables}{variable_uuid}{$variable_uuid}{global}{variable_default},
"variables::variable_uuid::${variable_uuid}::global::variable_description" => $anvil->data->{variables}{variable_uuid}{$variable_uuid}{global}{variable_description},
"variables::variable_uuid::${variable_uuid}::global::modified_date" => $anvil->data->{variables}{variable_uuid}{$variable_uuid}{global}{modified_date},
"variables::variable_uuid::${variable_uuid}::global::modified_date_unix" => $anvil->data->{variables}{variable_uuid}{$variable_uuid}{global}{modified_date_unix},
}});
}
}
return(0);
}
=head2 initialize
This will initialize a database using a given file.
@ -18026,6 +18173,8 @@ AND
This method disconnects from any connected databases, re-reads the config, and then tries to reconnect to any databases again. The number of connected datbaases is returned.
B<< Note >>: This calls C<< Database->disconnect({cleanup => 0}); >> to prevent attempts to talk to the potentially lost database handle.
Parameters;
=head3 lost_uuid (optional)
@ -18057,7 +18206,10 @@ sub reconnect
}
# Disconnect from all databases and then stop the daemon, then reconnect.
$anvil->Database->disconnect({debug => $debug});
$anvil->Database->disconnect({
debug => $debug,
cleanup => 0,
});
sleep 2;
# Refresh configs.
@ -19428,7 +19580,10 @@ sub write
if ($problem)
{
# We can't use this DB.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "warn", key => "warning_0182", variables => { uuid => $uuid }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "warn", key => "warning_0182", variables => {
uuid => $uuid,
query => (not $secure) ? $query : $anvil->Log->is_secure($query),
}});
next;
}
}
@ -20795,7 +20950,7 @@ sub _test_access
# Try to reconnect.
$anvil->Database->reconnect({
debug => $debug,
debug => 2,
lost_uuid => $uuid,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {

@ -4283,6 +4283,24 @@ sub ping
target => $target,
is_local => $is_local,
}});
if ($timeout)
{
if ($is_local)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0855", variables => {
shell_call => $shell_call,
timeout => $timeout,
}});
}
else
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0853", variables => {
shell_call => $shell_call,
timeout => $timeout,
target => $target,
}});
}
}
if ($is_local)
{
### Local calls

@ -309,6 +309,10 @@ sub call
if ($timeout)
{
# Prepend a timeout.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => $secure, key => "log_0855", variables => {
shell_call => $shell_call,
timeout => $timeout,
}});
$shell_call = $anvil->data->{path}{exe}{timeout}." ".$timeout." ".$shell_call;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { shell_call => $shell_call }});
}
@ -595,7 +599,13 @@ sub check_daemon
This returns C<< 1 >> is the system has finished initial configuration, and C<< 0 >> if not.
This method takes no parameters.
Parameters;
=head3 thorough (optional, default 0)
If this is set to C<< 1 >>, then a check is made of the host's network. If the network is found to be unconfigured, then C<< 0 >> is returned, Even if the variable C<< system::configured >> is set to C<< 1 >>.
If this test fails, the C<< system::configured >> variable will be reset to C<< 0 >>. Likewise, the C<< path::data::host_configured >> file (usually C<< /etc/anvil/host.configured >>) wil be updated also.
=cut
sub check_if_configured
@ -606,6 +616,11 @@ sub check_if_configured
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "System->check_if_configured()" }});
my $thorough = defined $parameter->{thorough} ? $parameter->{thorough} : 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
thorough => $thorough,
}});
my ($configured, $variable_uuid, $modified_date) = $anvil->Database->read_variable({
debug => $debug,
variable_name => "system::configured",
@ -640,7 +655,7 @@ sub check_if_configured
if (($variable eq "system::configured") && ($value eq "1"))
{
# Write the database entry.
my $variable_uuid = $anvil->Database->insert_or_update_variables({
$variable_uuid = $anvil->Database->insert_or_update_variables({
debug => 2,
variable_name => "system::configured",
variable_value => 1,
@ -660,12 +675,134 @@ sub check_if_configured
}
}
if (($configured) && (not -f $anvil->data->{path}{data}{host_configured}))
# Are we configured?
if ($configured)
{
if ($thorough)
{
# OK, but are we really though?
my $host_uuid = $anvil->Get->host_uuid({debug => $debug});
my $short_host_name = $anvil->Get->short_host_name({debug => $debug});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
host_uuid => $host_uuid,
short_host_name => $short_host_name,
}});
$anvil->Network->get_ips({debug => $debug, target => $short_host_name});
$anvil->Network->collect_data({debug => $debug});
$anvil->Database->get_variables({debug => $debug});
my $reconfigure = 0;
if (not exists $anvil->data->{variables}{source_table}{hosts}{source_uuid}{$host_uuid})
{
# Report that we can't validate this host's configuration. We'll have
# to rely on the variable value, if any.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "warning_0183"});
$reconfigure = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { reconfigure => $reconfigure }});
}
else
{
foreach my $variable_uuid (sort {$a cmp $b} keys %{$anvil->data->{variables}{source_table}{hosts}{source_uuid}{$host_uuid}{variable_uuid}})
{
my $variable_name = $anvil->data->{variables}{source_table}{hosts}{source_uuid}{$host_uuid}{variable_uuid}{$variable_uuid}{variable_name};
my $variable_value = $anvil->data->{variables}{source_table}{hosts}{source_uuid}{$host_uuid}{variable_uuid}{$variable_uuid}{variable_value};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
variable_uuid => $variable_uuid,
variable_name => $variable_name,
variable_value => $variable_value,
}});
if ($variable_name =~ /form::config_step2::(\w+)n(\d+)_create_bridge::value/)
{
my $network = $1."n".$2;
my $bridge_name = $network."_bridge1";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
network => $network,
bridge_name => $bridge_name,
}});
# Does this bridge exist?
if (not exists $anvil->data->{nmcli}{bridge}{$bridge_name})
{
# Missing bridge!
$reconfigure = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { reconfigure => $reconfigure }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "warning_0184", variables => { interface => $bridge_name }});
}
}
if ($variable_name =~ /form::config_step2::(\d+)n(\d+)_link(\d)_mac_to_set::value/)
{
my $interface = $1."n".$2."_link".$3;
my $bond_name = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { interface => $interface }});
# If this is link2, look for a bond.
if ($3 eq "2")
{
# We'll check for a bond after we check for this interface.
$bond_name = $1."n".$2."_bond1";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { bond_name => $bond_name }});
}
# Does this interface exist? See if we can find it via the
# MAC address record or the interface name. Note that if the
# 'match.interface-name' is set, both interface names are
# stored, even if NM decided not to use one of them.
if ((not exists $anvil->data->{nmcli}{perm_mac_address}{$interface}) && (not exists $anvil->data->{nmcli}{interface}{$interface}))
{
# Missing interface!
$reconfigure = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { reconfigure => $reconfigure }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "warning_0184", variables => { interface => $interface }});
}
if (($bond_name) && (not exists $anvil->data->{nmcli}{bond}{$bond_name}))
{
# Missing bond!
$reconfigure = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { reconfigure => $reconfigure }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "warning_0184", variables => { interface => $bond_name }});
}
}
}
}
if ($reconfigure)
{
$configured = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { configured => $configured }});
# Mark this host as not configured.
$variable_uuid = $anvil->Database->insert_or_update_variables({
debug => $debug,
variable_name => "system::configured",
variable_value => $configured,
variable_default => "",
variable_description => "striker_0048",
variable_section => "system",
variable_source_uuid => $host_uuid,
variable_source_table => "hosts",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { variable_uuid => $variable_uuid }});
my $failed = $anvil->Storage->write_file({
debug => $debug,
file => $anvil->data->{path}{data}{host_configured},
backup => 0,
overwrite => 1,
body => "system::configured = ".$configured."\n",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }});
}
}
}
if (not -f $anvil->data->{path}{data}{host_configured})
{
my $failed = $anvil->Storage->write_file({
debug => $debug,
file => $anvil->data->{path}{data}{host_configured},
body => "system::configured = 1\n",
body => "system::configured = ".$configured."\n",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }});
}

@ -1,4 +1,4 @@
policy_module(anvil-subnode, 1.0.0)
policy_module(anvil-subnode, 1.1.0)
########################################
#
@ -15,8 +15,10 @@ policy_module(anvil-subnode, 1.0.0)
#
require {
type mnt_t;
type sysctl_vm_t;
type svirt_t;
type virsh_t;
class file { open read };
class file { getattr open read };
}
@ -27,3 +29,8 @@ require {
#============= virsh_t ==============
# Needed for virsh to access the domain XMLs under /mnt.
allow virsh_t mnt_t:file { open read };
#============= svirt_t ==============
# Workaround until QEMU fixes its policy for RHEL/Almalinux >= 9.4
allow svirt_t sysctl_vm_t:file { getattr open read };

@ -1991,7 +1991,7 @@ The database connection error was:
<key name="log_0189">Database archiving is disabled, skipping archive checks.</key>
<key name="log_0190">Peer: [#!variable!peer!#], database: [#!variable!name!#], password: [#!variable!password!#], host UUID: [#!variable!uuid!#]</key>
<key name="log_0191">Connection only to: [#!variable!db_uuid!#], skipping: [#!variable!uuid!#].</key>
<key name="log_0192">The connection to the database: [#!variable!server!#] has failed. Will attempt to reconnect to databases-.</key>
<key name="log_0192">The connection to the database: [#!variable!server!#] has failed. Will attempt to reconnect to the database(s).</key>
<key name="log_0193">Switching the default database handle to use the database: [#!variable!server!#] prior to reconnect attempt.</key>
<key name="log_0194">Switching the default database to read from to the database: [#!variable!server!#] prior to reconnect attempt.</key>
<key name="log_0195">Ready to try to reconnect to: [#!variable!server!#], but delaying for: [#!variable!delay!#] seconds to give the database a chance to come back online in case this is a transient issue.</key>
@ -2742,6 +2742,10 @@ new key: [#!variable!new_key!#]</key>
<key name="log_0852">Finished configuring bonds.</key>
<key name="log_0853">Now configuring bridges.</key>
<key name="log_0854">The connection to the database: [#!variable!server!#] has been restored!</key>
<key name="log_0855">Calling: [#!variable!shell_call!#] via a 'timeout' of: [#!variable!timeout!#] seconds.</key>
<key name="log_0856">Calling: [#!variable!shell_call!#] on: [#!variable!target!#] via a 'timeout' of: [#!variable!timeout!#] seconds.</key>
<key name="log_0857">Cleaning up the connection to the database on striker: [#!variable!uuid!#].</key>
<key name="log_0858">Disconnecting from the database on striker: [#!variable!uuid!#].</key>
<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>
@ -3109,7 +3113,7 @@ Are you sure that you want to delete the server: [#!variable!server_name!#]? [Ty
<key name="message_0263">Preparing to get server VM screenshot.</key>
<key name="message_0264">Finished getting server VM screenshot.</key>
<key name="message_0265">Failed to get server VM screenshot; got non-zero return code.</key>
<key name="message_0266">Finished attempting to get server VM screenshot; no operations happened because requirements not met.</key>>>> master
<key name="message_0266">Finished attempting to get server VM screenshot; no operations happened because requirements not met.</key>
<key name="message_0267">Preparing to manage DR for a server.</key>
<key name="message_0268">UUID Column counts for: [history.#!variable!table!#]:</key>
<key name="message_0269">Counting entries for each unique: [#!variable!column!#] in the table [#!variable!table!#]. Please be patient.</key>
@ -4207,7 +4211,9 @@ We will try to proceed anyway.</key>
<key name="warning_0179">[ Warning ] - The connection to the database: [#!variable!server!#] was not restored, unable to this database.</key>
<key name="warning_0180">[ Warning ] - We were asked to read from the database: [#!variable!server!#], but it is not or is no longer available.</key>
<key name="warning_0181">[ Warning ] - We were asked to read from the database: [#!variable!server!#], but it is no longer available, and no alternative databases are available.</key>
<key name="warning_0182">[ Warning ] - We appear to have lost access to the database: [#!variable!uuid!#], during a write. Skipping this database.</key>
<key name="warning_0182">[ Warning ] - We appear to have lost access to the database: [#!variable!uuid!#], during the write: [#!variable!query!#]. Skipping this database.</key>
<key name="warning_0183">[ Warning ] - We were asked to thoroughly check if this host is configured or not, but there are no records in the 'variables' table for this host. Unable to check the network config! Relying on the 'system::configured' value.</key>
<key name="warning_0184">[ Warning ] - We were asked to thoroughly check if this host is configured or not, and the interface: [#!variable!interface!#] doesn't exist! Marking this host as unconfigured!</key>
</language>
<!-- 日本語 -->

@ -561,7 +561,7 @@ ORDER BY
{
$anvil->System->stop_daemon({debug => 2, daemon => "NetworkManager.service"});
$anvil->System->start_daemon({debug => 2, daemon => "NetworkManager.service"});
sleep 5;
sleep 2;
}
# Reconnect!
@ -612,7 +612,7 @@ ORDER BY
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0829", variables => { time_left => $time_left }});
sleep 5;
sleep 2;
}
else
{
@ -1248,7 +1248,7 @@ sub reconfigure_bonds
# NM seems to have a race issue, so we sleep a second after nmcli calls.
$anvil->data->{sys}{restart_nm} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::restart_nm' => $anvil->data->{sys}{restart_nm} }});
sleep 5;
sleep 2;
if ($bond_uuid)
{
@ -1280,7 +1280,7 @@ sub reconfigure_bonds
# NM seems to have a race issue, so we sleep a second after nmcli calls.
$anvil->data->{sys}{restart_nm} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::restart_nm' => $anvil->data->{sys}{restart_nm} }});
sleep 5;
sleep 2;
}
# Done! Rescanning the network config
@ -1418,7 +1418,7 @@ sub reconfigure_bonds
# NM seems to have a race issue, so we sleep a second after nmcli calls.
$anvil->data->{sys}{restart_nm} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::restart_nm' => $anvil->data->{sys}{restart_nm} }});
sleep 5;
sleep 2;
if ($return_code)
{
@ -1505,7 +1505,7 @@ sub wait_for_nm
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
}
# Not found yet.
sleep 1;
sleep 2;
}
else
{
@ -1851,7 +1851,7 @@ sub reconfigure_ip_addresses
# NM seems to have a race issue, so we sleep a second after nmcli calls.
$anvil->data->{sys}{restart_nm} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::restart_nm' => $anvil->data->{sys}{restart_nm} }});
sleep 5;
sleep 2;
}
# Now assign the IP.
@ -1878,7 +1878,7 @@ sub reconfigure_ip_addresses
# NM seems to have a race issue, so we sleep a second after nmcli calls.
$anvil->data->{sys}{restart_nm} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::restart_nm' => $anvil->data->{sys}{restart_nm} }});
sleep 5;
sleep 2;
# Restart the interface
$anvil->Job->update_progress({
@ -1922,7 +1922,7 @@ sub reconfigure_interfaces
# network manager device to the ip addr name or the biosdevname.
foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{nmcli}{uuid}})
{
### TODO: Left off here... any devices that are down, set:
### TODO: Any devices that are down, set:
### nmcli connection modify <uuid> ipv4.method manual ipv4.addresses 169.0.0.x/8' (x == Wired connection x)
my $connection_id = $anvil->data->{nmcli}{uuid}{$uuid}{'connection.id'} // "";
my $general_ip_iface = $anvil->data->{nmcli}{uuid}{$uuid}{'GENERAL.IP-IFACE'} // "";
@ -1960,7 +1960,7 @@ sub reconfigure_interfaces
return_code => $return_code,
}});
# NM seems to have a race issue, so we sleep a second after nmcli calls.
sleep 1;
sleep 2;
# Rescan.
$anvil->Job->update_progress({
@ -2119,7 +2119,7 @@ sub reconfigure_interfaces
if ($link2_nm_uuid)
{
my $found = 0;
my $match_interface_name = $anvil->data->{nmcli}{uuid}{$link2_nm_uuid}{'match.interface-name'} // "";
my $match_interface_name = $anvil->data->{nmcli}{uuid}{$link2_nm_uuid}{'match.interface-name'} ? $anvil->data->{nmcli}{uuid}{$link2_nm_uuid}{'match.interface-name'} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { match_interface_name => $match_interface_name }});
foreach my $interface (split/,/, $match_interface_name)
{
@ -2351,7 +2351,7 @@ sub rename_interface
return_code => $return_code,
}});
# NM seems to have a race issue, so we sleep a second after nmcli calls.
sleep 1;
sleep 2;
$shell_call = $anvil->data->{path}{exe}{nmcli}." --get-values connection.interface-name connection show ".$nm_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
@ -2424,7 +2424,7 @@ sub rename_interface
return_code => $return_code,
}});
# NM seems to have a race issue, so we sleep a second after nmcli calls.
sleep 1;
sleep 2;
# Read it back
$shell_call = $anvil->data->{path}{exe}{nmcli}." --get-values match.interface-name connection show ".$nm_uuid;
@ -2475,7 +2475,7 @@ sub rename_interface
return_code => $return_code,
}});
# NM seems to have a race issue, so we sleep a second after nmcli calls.
sleep 1;
sleep 2;
# Read it back
$shell_call = $anvil->data->{path}{exe}{nmcli}." --get-values connection.id connection show ".$nm_uuid;
@ -2674,7 +2674,7 @@ AND
my ($out_of_cluster) = $anvil->Cluster->parse_cib();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
out_of_cluster => $out_of_cluster,
"cib::parsed::local::ready" => $anvil->data->{cib}{parsed}{'local'}{ready},
"cib::parsed::local::ready" => defined $anvil->data->{cib}{parsed}{'local'}{ready} ? $anvil->data->{cib}{parsed}{'local'}{ready} : "",
}});
if ((not $out_of_cluster) && ($anvil->data->{cib}{parsed}{'local'}{ready}))
{

@ -203,6 +203,50 @@ sub process_rejoin
}});
}
# Does this host need to be reconfigured? If so, register a job, if one doesn't exist yet, and exit.
my $configured = $anvil->System->check_if_configured({
debug => 2,
thorough => 1,
});
if ($configured)
{
print "The host appears to already be configured, good.\n";
}
else
{
print "[ Note ] - The host appears to NOT be configured yet!\n";
if (exists $anvil->data->{anvils}{host_uuid}{$host_uuid})
{
# Can we find the data needed to reconfigure this subnode?
my $job_uuid = create_anvil_configure_host_job($anvil, $host_uuid);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
if ($job_uuid)
{
print "[ Success ] - Found the data needed to configure the network on this subnode! The anvil-join-anvil job has\n";
print "[ Success ] been created with the job UUID: [".$job_uuid."]\n";
print "[ Note ] - This subnode will reboot one minute after the network configuration completes. Once rebooted,\n";
print "[ Note ] please run this again.\n";
$anvil->nice_exit({exit_code => 0});
}
else
{
print "[ Warning ] - Failed to automatically create the anvil-join-anvil job.\n";
print "[ Warning ] - Please use Striker to initialize this host. Once the network has been configured, try rejoining\n";
print "[ Warning ] this subnode again.\n";
$anvil->nice_exit({exit_code => 1});
}
}
else
{
# We can't configure it automatically.
print "[ Warning ] - This is a new host, so the information needed to configure the network does not exist.\n";
print "[ Warning ] - Please use Striker to initialize this host. Once the network has been configured, try rejoining\n";
print "[ Warning ] this subnode again.\n";
$anvil->nice_exit({exit_code => 1});
}
}
die;
if ($manifest)
{
# Did we get a valid manifest?
@ -226,7 +270,7 @@ sub process_rejoin
}
else
{
print "[ Error ] - The manifest: [".$manifest."] was not found in the database.\n";
print "[ Error ] - The manifest: [".$manifest."] was not found in the database.\n";
print "[ Error ] - Try again without this switch to see the available manifests.\n";
$anvil->nice_exit({exit_code => 1});
}
@ -402,6 +446,204 @@ sub process_rejoin
return(0);
}
sub create_anvil_configure_host_job
{
my ($anvil, $host_uuid) = @_;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_uuid => $host_uuid }});
# This is just to make the job_data cleaner.
if (exists $anvil->data->{job_vars})
{
delete $anvil->data->{job_vars};
}
if (exists $anvil->data->{iface_map})
{
delete $anvil->data->{iface_map};
}
# Load the network interface data to validate/update the MACs to interface names. If the subnode was
# rebuilt, the device name could have reverted. So we walk through the history to see if the same MAC
# used to have a name we recognize.
my $query = "
SELECT
network_interface_uuid,
network_interface_mac_address,
network_interface_name,
network_interface_device,
round(extract(epoch from modified_date))
FROM
history.network_interfaces
WHERE
network_interface_operational != 'DELETED'
AND
network_interface_host_uuid = ".$anvil->Database->quote($host_uuid)."
ORDER BY
modified_date DESC
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $network_interface_uuid = $row->[0];
my $mac_address = $row->[1];
my $name = $row->[2];
my $device = $row->[3];
my $modified_date_unix = $row->[4];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:network_interface_uuid" => $network_interface_uuid,
"s2:mac_address" => $mac_address,
"s3:name" => $name,
"s4:device" => $device,
"s5:modified_date_unix" => $modified_date_unix,
}});
$anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{mac_address} = $mac_address;
$anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{name} = $name;
$anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{device} = $device;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"s1:iface_map::${network_interface_uuid}::timestamp::${modified_date_unix}::mac_address" => $anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{mac_address},
"s2:iface_map::${network_interface_uuid}::timestamp::${modified_date_unix}::name" => $anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{name},
"s3:iface_map::${network_interface_uuid}::timestamp::${modified_date_unix}::device" => $anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{device},
}});
}
# Load the variables, if not already done so.
if (not exists $anvil->data->{variables}{source_table}{hosts}{source_uuid}{$host_uuid})
{
my $short_host_name = $anvil->Get->short_host_name();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { short_host_name => $short_host_name }});
$anvil->Network->get_ips({debug => 3, target => $short_host_name});
$anvil->Network->collect_data({debug => 3});
$anvil->Database->get_variables({debug => 2});
}
# The variables were already read in, so it's easy to recreate the job.
my $form_data_found = 0;
foreach my $variable_uuid (sort {$a cmp $b} keys %{$anvil->data->{variables}{source_table}{hosts}{source_uuid}{$host_uuid}{variable_uuid}})
{
my $variable_name = $anvil->data->{variables}{source_table}{hosts}{source_uuid}{$host_uuid}{variable_uuid}{$variable_uuid}{variable_name};
my $variable_value = $anvil->data->{variables}{source_table}{hosts}{source_uuid}{$host_uuid}{variable_uuid}{$variable_uuid}{variable_value};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"s1:variable_uuid" => $variable_uuid,
"s2:variable_name" => $variable_name,
"s3:variable_value" => $variable_value,
}});
next if $variable_name !~ /^form::config_step\d::/;
if (not $form_data_found)
{
$form_data_found = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { form_data_found => $form_data_found }});
}
# If the config value is the MAC address, check to see if there was a change since the
# original variables were recorded.
if ($variable_name =~ /form::config_step2::(.*?)_mac_to_set::value/)
{
my $interface = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { interface => $interface }});
my $iface_found = "";
my $mac_found = "";
foreach my $network_interface_uuid (sort {$a cmp $b} keys %{$anvil->data->{iface_map}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_interface_uuid => $network_interface_uuid }});
foreach my $modified_date_unix (sort {$b <=> $a} keys %{$anvil->data->{iface_map}{$network_interface_uuid}{timestamp}})
{
my $mac_address = $anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{mac_address};
my $name = $anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{name};
my $device = $anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{device};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
#"s1:modified_date_unix" => $modified_date_unix,
"s2:mac_address" => $mac_address,
"s3:name" => $name,
"s4:device" => $device,
}});
if ((not $iface_found) && ($device eq $interface))
{
$iface_found = $mac_address;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { iface_found => $iface_found }});
}
if ((not $iface_found) && ($name eq $interface))
{
$iface_found = $mac_address;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { iface_found => $iface_found }});
}
if ((not $mac_found) && ($mac_found eq $variable_value))
{
$mac_found = $interface;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mac_found => $mac_found }});
}
}
}
if ((not $iface_found) && (not $mac_found))
{
# Abort, we can't find this interface by name or MAC address.
print "[ Error ] - The interface: [".$interface."] which was originally assigned to the device with the MAC\n";
print "[ Error ] address: [".$variable_value."] could not be found. We checked for the interface name in the\n";
print "[ Error ] 'network_interfaces' historical schema, in case the interface was moved to a new NIC in the\n";
print "[ Error ] past. You will need to use the Striker UI (Striker logo -> Anvil -> Prepare Network) to \n";
print "[ Error ] initialize the network for this node before it can rejoin the cluster.\n";
return("");
}
# If I found the mac address, we're done. If not, but we found the interface, update
# the variable_value to the new MAC address.
if (not $iface_found)
{
$variable_value = $mac_found;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_value => $variable_value }});
}
}
$anvil->data->{job_vars}{$variable_name} = $variable_value;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"job_vars::${variable_name}" => $anvil->data->{job_vars}{$variable_name},
}});
}
if (not $form_data_found)
{
print "[ Error ] - It appears that the data used to configure this node is no longer stored in the database.\n";
print "[ Error ] You will need to use the Striker UI (Striker logo -> Anvil -> Prepare Network) to initialize\n";
print "[ Error ] the network for this node before it can rejoin the cluster.\n";
return("");
}
my $job_lines = 0;
my $job_data = "";
foreach my $variable_name (sort {$a cmp $b} keys %{$anvil->data->{job_vars}})
{
$job_data .= $variable_name."=".$anvil->data->{job_vars}{$variable_name}."\n";
$job_lines++;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:job_lines" => $job_lines,
"s2:job_data" => $job_data,
}});
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_data => $job_data }});
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
job_command => $anvil->data->{path}{exe}{'anvil-configure-host'}.$anvil->Log->switches,
job_data => $job_data,
job_name => "configure::network",
job_title => "job_0001",
job_description => "job_0071",
job_progress => 0,
job_host_uuid => $host_uuid,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
return($job_uuid)
}
# Make sure the hosts file has entries for all nets for both subnodes
sub wait_for_etc_hosts
{

Loading…
Cancel
Save