* Started work on caching network state change in tools/anvil-update-states.

* Fixed a bug where ip_addresses could break resync when 2+ machines had the same IP (ie: 192.168.122.1).
* Updated logging of DB transactions to show the DB host's IP instead of the UUID.
* Updated Get->date_and_time to take a 'use_utc' parameter to return the time using GMT time instead of the host's TZ.
* Updated anvil-daemon to periodically call tools/anvil-update-states. Also upadted anvil-daemon to delay daily jobs by 2 hours except for the dashboard with the highest sorted UUID to minimize dual runs of tasks that only need to run once per day per cluster.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 5 years ago
parent e8d15112da
commit 530d379f59
  1. 1
      Anvil/Tools.pm
  2. 24
      Anvil/Tools/Database.pm
  3. 17
      Anvil/Tools/Get.pm
  4. 4
      Anvil/Tools/Network.pm
  5. 8
      rpm/SPECS/anvil.spec
  6. 4
      share/words.xml
  7. 2
      tools/anvil-configure-host
  8. 68
      tools/anvil-daemon
  9. 26
      tools/anvil-update-states
  10. 4
      tools/striker-initialize-host
  11. 18
      tools/test.pl

@ -1032,6 +1032,7 @@ sub _set_paths
httpd_conf => "/etc/httpd/conf/httpd.conf",
host_ssh_key => "/etc/ssh/ssh_host_ecdsa_key.pub",
host_uuid => "/etc/anvil/host.uuid",
network_cache => "/tmp/network_cache.anvil",
passwd => "/etc/passwd",
'redhat-release' => "/etc/redhat-release",
},

@ -1121,8 +1121,8 @@ sub connect
# local, use it.
if (($host eq $anvil->_host_name) or
($host eq $anvil->_short_host_name) or
($host eq "localhost") or
($host eq "127.0.0.1") or
($host eq "localhost") or
($host eq "127.0.0.1") or
(not $anvil->data->{sys}{database}{read_uuid}))
{
$anvil->data->{sys}{database}{read_uuid} = $uuid;
@ -1308,7 +1308,7 @@ sub connect
$anvil->Database->mark_active({debug => $debug, set => 1});
# Sync the database, if needed.
$anvil->Database->resync_databases({debug => $debug});
$anvil->Database->resync_databases({debug => 3});
# Add ourselves to the database, if needed.
$anvil->Database->insert_or_update_hosts({debug => $debug});
@ -3642,9 +3642,11 @@ sub insert_or_update_ip_addresses
}
}
# If we don't have a UUID, see if we can find one for the given ip_address server name.
# If we don't have a UUID, see if we can find one for the given ip_address name.
if (not $ip_address_uuid)
{
### NOTE: An IP can be on multiple machines at the same time (ie: 192.168.122.1), so we need
### to restrict to this host.
my $query = "
SELECT
ip_address_uuid
@ -3652,6 +3654,8 @@ FROM
ip_addresses
WHERE
ip_address_address = ".$anvil->Database->quote($ip_address_address)."
AND
ip_address_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
@ -7018,7 +7022,7 @@ sub query
if ($anvil->data->{sys}{database}{log_transactions})
{
$anvil->Log->entry({source => $source, line => $line, secure => $secure, level => 0, key => "log_0074", variables => {
uuid => $uuid,
uuid => $anvil->data->{database}{$uuid}{host},
query => $query,
}});
}
@ -7251,7 +7255,7 @@ sub resync_databases
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 2;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->resync_databases()" }});
# If a resync isn't needed, just return.
@ -7380,7 +7384,7 @@ sub resync_databases
$query .= " WHERE ".$host_column." = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid});
}
$query .= " ORDER BY utc_modified_date DESC;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0074", variables => { uuid => $uuid, query => $query }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0074", variables => { uuid => $anvil->data->{database}{$uuid}{host}, query => $query }});
my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
@ -7515,7 +7519,7 @@ sub resync_databases
$query .= "$column_name = ".$column_value.", ";
}
$query .= "modified_date = ".$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC' WHERE $uuid_column = ".$anvil->Database->quote($row_uuid).";";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0074", variables => { uuid => $uuid, query => $query }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0460", variables => { uuid => $anvil->data->{database}{$uuid}{host}, query => $query }});
# Now record the query in the array
push @{$anvil->data->{db_resync}{$uuid}{public}{sql}}, $query;
@ -7551,7 +7555,7 @@ sub resync_databases
# Add the host column.
$query = "INSERT INTO public.$table ($host_column, $uuid_column, ".$columns."modified_date) VALUES (".$anvil->Database->quote($anvil->data->{sys}{host_uuid}).", ".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');";
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0074", variables => { uuid => $uuid, query => $query }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0460", variables => { uuid => $anvil->data->{database}{$uuid}{host}, query => $query }});
# Now record the query in the array
push @{$anvil->data->{db_resync}{$uuid}{public}{sql}}, $query;
@ -7599,7 +7603,7 @@ sub resync_databases
# Add the host column.
$query = "INSERT INTO history.$table ($host_column, $uuid_column, ".$columns."modified_date) VALUES (".$anvil->Database->quote($anvil->data->{sys}{host_uuid}).", ".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');";
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0074", variables => { uuid => $uuid, query => $query }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0460", variables => { uuid => $anvil->data->{database}{$uuid}{host}, query => $query }});
# Now record the query in the array
push @{$anvil->data->{db_resync}{$uuid}{history}{sql}}, $query;

@ -432,6 +432,10 @@ This can be set to a unix timestamp. If it is not set, the current time is used.
If set, only the time will be returned (in C<< hh:mm:ss >> format).
=head3 use_utc (optional)
If set, C<< gmtime >> is used instead of C<< localtime >>. The effect of this is that GMTime (greenwhich mean time, UTC-0) is used instead of the local system's time zone.
=cut
sub date_and_time
{
@ -442,6 +446,7 @@ sub date_and_time
my $offset = defined $parameter->{offset} ? $parameter->{offset} : 0;
my $use_time = defined $parameter->{use_time} ? $parameter->{use_time} : time;
my $use_utc = defined $parameter->{use_utc} ? $parameter->{use_utc} : 0;
my $file_name = defined $parameter->{file_name} ? $parameter->{file_name} : 0;
my $time_only = defined $parameter->{time_only} ? $parameter->{time_only} : 0;
my $date_only = defined $parameter->{date_only} ? $parameter->{date_only} : 0;
@ -464,8 +469,16 @@ sub date_and_time
#print $THIS_FILE." ".__LINE__."; [ Debug ] - adjusted_time: [$adjusted_time]\n";
# Get the date and time pieces
($time->{sec}, $time->{min}, $time->{hour}, $time->{mday}, $time->{mon}, $time->{year}, $time->{wday}, $time->{yday}, $time->{isdst}) = localtime($adjusted_time);
#print $THIS_FILE." ".__LINE__."; [ Debug ] - time->{sec}: [".$time->{sec}."], time->{min}: [".$time->{min}."], time->{hour}: [".$time->{hour}."], time->{mday}: [".$time->{mday}."], time->{mon}: [".$time->{mon}."], time->{year}: [".$time->{year}."], time->{wday}: [".$time->{wday}."], time->{yday}: [".$time->{yday}."], time->{isdst}: [".$time->{isdst}."]\n";
if ($use_utc)
{
($time->{sec}, $time->{min}, $time->{hour}, $time->{mday}, $time->{mon}, $time->{year}, $time->{wday}, $time->{yday}, $time->{isdst}) = gmtime($adjusted_time);
#print $THIS_FILE." ".__LINE__."; [ Debug ] - time->{sec}: [".$time->{sec}."], time->{min}: [".$time->{min}."], time->{hour}: [".$time->{hour}."], time->{mday}: [".$time->{mday}."], time->{mon}: [".$time->{mon}."], time->{year}: [".$time->{year}."], time->{wday}: [".$time->{wday}."], time->{yday}: [".$time->{yday}."], time->{isdst}: [".$time->{isdst}."]\n";
}
else
{
($time->{sec}, $time->{min}, $time->{hour}, $time->{mday}, $time->{mon}, $time->{year}, $time->{wday}, $time->{yday}, $time->{isdst}) = localtime($adjusted_time);
#print $THIS_FILE." ".__LINE__."; [ Debug ] - time->{sec}: [".$time->{sec}."], time->{min}: [".$time->{min}."], time->{hour}: [".$time->{hour}."], time->{mday}: [".$time->{mday}."], time->{mon}: [".$time->{mon}."], time->{year}: [".$time->{year}."], time->{wday}: [".$time->{wday}."], time->{yday}: [".$time->{yday}."], time->{isdst}: [".$time->{isdst}."]\n";
}
# Process the raw data
$time->{pad_hour} = sprintf("%02d", $time->{hour});

@ -994,7 +994,7 @@ AND
ORDER BY
modified_date DESC
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0124", variables => { query => $query }});
$results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
@ -1075,7 +1075,7 @@ ORDER BY
$anvil->data->{network}{$host}{interface}{$network_interface_name}{bridge_name} = $bridge_name;
$anvil->data->{network}{$host}{interface}{$network_interface_name}{type} = "interface";
$anvil->data->{network}{$host}{interface}{$network_interface_name}{changed_order} = $this_change_orger;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"network::${host}::interface::${network_interface_name}::uuid" => $anvil->data->{network}{$host}{interface}{$network_interface_name}{uuid},
"network::${host}::interface::${network_interface_name}::mac_address" => $anvil->data->{network}{$host}{interface}{$network_interface_name}{mac_address},
"network::${host}::interface::${network_interface_name}::speed" => $anvil->data->{network}{$host}{interface}{$network_interface_name}{speed},

@ -3,7 +3,7 @@
%define anvilgroup admin
Name: anvil
Version: 3.0
Release: 29%{?dist}
Release: 30%{?dist}
Summary: Alteeve Anvil! complete package.
License: GPLv2+
@ -24,6 +24,7 @@ WARNING: This is an alpha-stage project. Many features are missing and this
%package core
Summary: Alteeve's Anvil! Core package
Requires: bash-completion
Requires: chrony
Requires: bind-utils
Requires: dmidecode
Requires: dnf-utils
@ -205,6 +206,8 @@ setenforce 0
sed -i "1s/^.*$/%{version}-%{release}/" /%{_sysconfdir}/anvil/anvil.version
# Enable and start the anvil-daemon
### TODO: check it if was disabled (if it existed before) and, if so, leave it disabled.
systemctl enable chronyd.service
systemctl start chronyd.service
systemctl enable anvil-daemon.service
systemctl start anvil-daemon.service
@ -349,6 +352,9 @@ fi
%changelog
* tbd Madison Kelly <mkelly@alteeve.ca> 3.0-30
- Enabled/started chronyd in core's post.
* Thu Nov 7 2019 Madison Kelly <mkelly@alteeve.ca> 3.0-29
- Added '/etc/anvil/type.X' file creation to more directly mark a system as a
specific type, rather than divining by name.

@ -791,6 +791,7 @@ Failed to promote the DRBD resource: [#!variable!resource!#] primary. Expected a
<key name="log_0457">Removing archived records.</key>
<key name="log_0458">Vacuuming the database to purge the removed records.</key>
<key name="log_0459">Skipping the table: [#!variable!table!#], it is excluded from archiving.</key>
<key name="log_0460">Queing up to run: [#!variable!uuid!#]:[#!variable!query!#]</key>
<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
<key name="t_0000">Test</key>
@ -1128,7 +1129,7 @@ Failure! The return code: [#!variable!return_code!#] was received ('0' was expec
<key name="job_0069">Unable to find a matching network, skipping this database.</key>
<key name="job_0070">Something went wrong adding this database. Please see: [#!data!path::log::main!#] for details.</key>
<!-- Warnings -->
<!-- Warnings: TODO: Take 'striker_' off -->
<key name="striker_warning_0001">The IP address will change. You will need to reconnect after applying these changes.</key>
<key name="striker_warning_0002">The access information appears to not be valid.</key>
<key name="striker_warning_0003">Test access to the peer (using SSH) failed. There may be details in the log file.</key>
@ -1144,6 +1145,7 @@ Failure! The return code: [#!variable!return_code!#] was received ('0' was expec
<key name="striker_warning_0013"><![CDATA[The target's host key has changed. If the target has been rebuilt, or the target IP reused, the old key will need to be removed. <a href="?striker=true&task=keys" target="_new">Click here</a> to resolve.]]></key>
<key name="striker_warning_0014">The host UUID: [#!variable!host_uuid!#] was not found in the #!data!path::json::all_status!# file on the local dashboard.</key>
<key name="striker_warning_0015">To configure a host as either an Anvil! node or a disaster recovery host, there must be at least 6 network interfaces. This machine only has: [#!variable!interface_count!#] interfaces.</key>
<key name="warning_0016">No databases are available. Changes to the network interfaces will be cached.</key>
<!-- Errors -->
<key name="error_0001">There are not enough network interfaces on this machine. You have: [#!variable!interface_count!#] interface(s), and you need at least: [#!variable!required_interfaces_for_single!#] interfaces to connect to the requested networks (one for Back-Channel and one for each Internet-Facing network).</key>

@ -1,6 +1,6 @@
#!/usr/bin/perl
#
# This is called when striker, an node or a DR host needs to configure the local network and user accounts.
# This is called when striker, a node or a DR host needs to configure the local network and user accounts.
#
# Exit codes;
# 0 = Normal exit.

@ -138,32 +138,37 @@ run_once($anvil) if not $anvil->data->{switches}{'main-loop-only'};
# Calculate my sum so that we can exit if it changes later.
$anvil->Storage->record_md5sums;
# Disconnect. We'll reconnect inside the loop
$anvil->Database->disconnect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0203"});
# This will prevent restarting while jobs are running.
$anvil->data->{sys}{jobs_running} = 0;
# What time is it, Mr. Fox?
my $now_time = time;
# To avoid multiple dashboards running a network scan and OUI parse, the dashboard peer with the lowest
# host_uuid sets it's daily checks to run now, and the other(s) will get a two hour's delay.
my $delay = set_delay($anvil);
# Once a minute, we'll check the md5sums and see if we should restart.
# Once a day, we'll refresh an Install Target's RPM repository (has no effect on non-Striker dashboards).
$anvil->data->{timing}{minute_checks} = 60;
$anvil->data->{timing}{daily_checks} = 86400;
$anvil->data->{timing}{repo_update_interval} = 86400;
$anvil->data->{timing}{next_minute_check} = $now_time - 1;
$anvil->data->{timing}{next_daily_check} = $now_time - 1;
$anvil->data->{timing}{next_daily_check} = ($now_time + $delay) - 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:timing::minute_checks" => $anvil->data->{timing}{minute_checks},
"s1:timing::daily_checks" => $anvil->data->{timing}{daily_checks},
"s2:timing::repo_update_interval" => $anvil->data->{timing}{repo_update_interval},
"s3:now_time" => $now_time,
"s4:timing::next_minute_check" => $anvil->data->{timing}{next_minute_check},
"s4:timing::next_daily_check" => $anvil->data->{timing}{next_daily_check},
"s2:timing::daily_checks" => $anvil->data->{timing}{daily_checks},
"s3:timing::repo_update_interval" => $anvil->data->{timing}{repo_update_interval},
"s4:now_time" => $now_time,
"s5:delay" => $delay,
"s6:timing::next_minute_check" => $anvil->data->{timing}{next_minute_check},
"s7:timing::next_daily_check" => $anvil->data->{timing}{next_daily_check},
}});
# Disconnect. We'll reconnect inside the loop
$anvil->Database->disconnect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0203"});
# This will prevent restarting while jobs are running.
$anvil->data->{sys}{jobs_running} = 0;
# When we periodically check if system files have changed, we'll also ask Database>connect() to check if it
# needs to be configured or updated. This is done periodically as it is expensive to run on every loop.
my $check_if_database_is_configured = 0;
@ -212,6 +217,40 @@ $anvil->nice_exit({code => 0});
# Functions #
#############################################################################################################
# This decides if the local system will delay daily runs on start-up.
sub set_delay
{
my ($anvil) = @_;
my $delay = 7200;
my $type = $anvil->System->get_host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { type => $type }});
if ($type eq "dashboard")
{
foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{database}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"sys::host_uuid" => $anvil->data->{sys}{host_uuid},
uuid => $uuid,
}});
if ($uuid eq $anvil->data->{sys}{host_uuid})
{
$delay = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { delay => $delay }});
}
last;
}
}
else
{
# Not a dashboard, don't delay
$delay = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { delay => $delay }});
}
return($delay);
}
# This handles running tasks that only run on some loops.
sub handle_periodic_tasks
{
@ -255,6 +294,9 @@ sub handle_periodic_tasks
"s2:timing::next_minute_check" => $anvil->data->{timing}{next_minute_check},
}});
# Scan the local network.
update_state_file($anvil);
# Make sure the shared directories exist.
foreach my $target (sort {$a cmp $b} keys %{$anvil->data->{path}{directories}{shared}})
{

@ -26,11 +26,14 @@ $anvil->Database->connect({debug => 3});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, exit.
print $anvil->Words->string({key => "error_0003"})."\n";
$anvil->nice_exit({exit_code => 2});
# No database, but we need to keep going. If the user unplugged the only cable connecting us to the
# network, we'll lose all DBs, but we still need to record the order the NICs went up and down.
print $anvil->Words->string({key => "warning_0016"})."\n";
}
# Check the network cache for records we need to insert.
process_network_cache($anvil);
update_network($anvil);
$anvil->nice_exit({exit_code => 0});
@ -40,6 +43,23 @@ $anvil->nice_exit({exit_code => 0});
# Functions #
#############################################################################################################
# This reads in the network cache file and looks for records that haven't been stored in the database yet.
sub process_network_cache
{
my ($anvil) = @_;
# Does the file exist? If so, read it in.
if (-e $anvil->data->{path}{data}{network_cache})
{
my $body = $anvil->Storage->read_file({cache => 0, force_read => 1, file => $anvil->data->{path}{data}{network_cache}});
foreach my $line (split/\n/, $body)
{
}
}
return(0);
}
# This reports the current network interface states, tracked by the MAC address.
sub update_network
{

@ -464,8 +464,8 @@ EOF
{
# We're going to need these repos when it's all said and done. The first two are
# enabled by default.
$anvil->data->{repos}{'rhel-8-for-x86_64-baseos-rpms'} = 0;
$anvil->data->{repos}{'rhel-8-for-x86_64-appstream-rpms'} = 0;
$anvil->data->{repos}{'rhel-8-for-x86_64-baseos-rpms'} = 0;
$anvil->data->{repos}{'rhel-8-for-x86_64-appstream-rpms'} = 0;
$anvil->data->{repos}{'codeready-builder-for-rhel-8-x86_64-rpms'} = 0;
# Both target types need 'codeready-builder-for-rhel-8-x86_64-rpms', but only nodes

@ -27,24 +27,16 @@ $anvil->Log->level({set => 2});
$anvil->Database->connect({debug => 3, check_if_configured => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
my $interface_uuid = "ffd6d29b-100c-452f-be4f-51cbc94eb069";
my $query = "SELECT network_interface_bridge_uuid FROM network_interfaces WHERE network_interface_uuid = ".$anvil->Database->quote($interface_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
my $network_interface_bridge_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_interface_bridge_uuid => $network_interface_bridge_uuid }});
my $utc_tz = $anvil->Get->date_and_time({use_utc => 1});
$utc_tz =~ s/\//-/g;
print "DB Timestamp; [".$anvil->data->{sys}{database}{timestamp}."]\n";
print "GM Timestamp; [".$utc_tz."]\n";
die;
#$anvil->System->generate_state_json({debug => 3});
#$anvil->Striker->parse_all_status_json({debug => 3});
#print Dumper $anvil->data->{json}{all_status}{hosts}{'el8-a01n01.digimer.ca'};
die;
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{json}{all_status}{hosts}})
{

Loading…
Cancel
Save