This is a set of changes all stemming from trying to debug frequent resyncs. More bugs still to be fixed.

* Updated Database->get_host_from_uuid() to cache results.
* Fixed a bug in Database->get_storage_group_data where a DELETE wasn't deleting from the history schema as well.
* In Database->resync_databases(), references to the old 'host_uuid' that we used to use to resync just the local host's data was removed. Added also a check where two or more entries in a given history schema had the same modified_date and, when found, the newest entry is preserved and the rest are deleted. Before this, a resync where two+ records had the same modified_time would only sync the last record, leaving a mismatch in history schema entries triggering repeated resyncs.
* Fixed a bug in Email->send_alerts() where the 'alerts' table was being updated without a modified_date being set.
* Fixed a bug in System->test_ipmi() where the 'hosts' table was being updated without a modified_date being set.
* Updated scan-network to clear up old deleted ip_addresses, bonds and bridges. Also fixed bugs where public schema records were being deleted without history records being deleted.
* Updated anvil-update-states to fix bugs where DELETEs were happening without setting the modified_date.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 3 years ago
parent 1770e9e0e0
commit 24f5d39dff
  1. 219
      Anvil/Tools/Database.pm
  2. 10
      Anvil/Tools/Email.pm
  3. 2
      Anvil/Tools/Network.pm
  4. 10
      Anvil/Tools/System.pm
  5. 19
      notes
  6. 241
      scancore-agents/scan-network/scan-network
  7. 3
      scancore-agents/scan-network/scan-network.xml
  8. 1
      share/words.xml
  9. 31
      tools/anvil-update-states
  10. 2
      tools/striker-prep-database

@ -809,8 +809,8 @@ sub configure_pgsql
# If we didn't get the $local_uuid, then there is no entry for this system in anvil.conf yet, so we'll add it.
if (not $local_uuid)
{
$local_uuid = $anvil->Database->_add_to_local_config({debug => 2});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { local_uuid => $local_uuid }});
$local_uuid = $anvil->Database->_add_to_local_config({debug => $debug});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { local_uuid => $local_uuid }});
if ($local_uuid eq "!!error!!")
{
@ -1730,7 +1730,7 @@ sub connect
if ($anvil->data->{sys}{database}{read_uuid} eq $uuid)
{
$anvil->data->{sys}{database}{read_uuid} = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid} }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid} }});
}
next;
}
@ -3210,6 +3210,7 @@ sub get_host_from_uuid
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->get_host_from_uuid()" }});
my $host_name = "";
my $short_host_name = "";
my $host_uuid = defined $parameter->{host_uuid} ? $parameter->{host_uuid} : "";
my $include_deleted = defined $parameter->{include_deleted} ? $parameter->{include_deleted} : 0;
my $short = defined $parameter->{short} ? $parameter->{short} : 0;
@ -3226,6 +3227,19 @@ sub get_host_from_uuid
return($host_name);
}
# If we queried this before, return the cached value.
if (exists $anvil->data->{host_from_uuid}{$host_uuid})
{
if ($short)
{
return($anvil->data->{host_from_uuid}{$host_uuid}{short});
}
else
{
return($anvil->data->{host_from_uuid}{$host_uuid}{full});
}
}
my $query = "
SELECT
host_name
@ -3251,17 +3265,29 @@ AND
}});
if ($count)
{
$host_name = $results->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_name => $host_name }});
$host_name = $results->[0]->[0];
$short_host_name = ($host_name =~ /^(.*?)\..*$/)[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
host_name => $host_name,
short_host_name => $short_host_name,
}});
if ($short)
{
$host_name =~ s/\..*$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_name => $host_name }});
}
$anvil->data->{host_from_uuid}{$host_uuid}{full} = $host_name;
$anvil->data->{host_from_uuid}{$host_uuid}{short} = $short_host_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"host_from_uuid::${host_uuid}::full" => $anvil->data->{host_from_uuid}{$host_uuid}{full},
"host_from_uuid::${host_uuid}::short" => $anvil->data->{host_from_uuid}{$host_uuid}{short},
}});
}
return($host_name);
if ($short)
{
return($anvil->data->{host_from_uuid}{$host_uuid}{short});
}
else
{
return($anvil->data->{host_from_uuid}{$host_uuid}{full});
}
}
@ -5120,8 +5146,12 @@ WHERE
anvil_name => $anvil_name,
}});
my $query = "DELETE FROM storage_group_members WHERE storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { query => $query }});
my $query = "DELETE FROM history.storage_group_members WHERE storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
$query = "DELETE FROM storage_group_members WHERE storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
}
}
@ -5243,7 +5273,7 @@ INSERT INTO
".$anvil->Database->quote($closest_scan_lvm_vg_uuid).",
".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
);";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { query => $query }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
# Reload
@ -9859,6 +9889,7 @@ SET
WHERE
network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
}
return($network_interface_uuid);
@ -14706,7 +14737,7 @@ sub locking
}});
# Log that we've renewed the lock.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0044", variables => { host => $anvil->Get->host_name }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0044", variables => { host => $anvil->Get->short_host_name }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { set => $set }});
return($set);
@ -14803,7 +14834,7 @@ sub locking
}});
# Log that we've got the lock.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0045", variables => { host => $anvil->Get->host_name }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0045", variables => { host => $anvil->Get->short_host_name }});
}
}
@ -15966,16 +15997,12 @@ sub resync_databases
next if $table eq "states";
next if $table eq "update";
next if $table eq "oui";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { table => $table }});
# If the 'schema' is 'public', there is no table in the history schema. If there is a host
# column, the resync will be restricted to entries from this host uuid.
my $schema = $anvil->data->{sys}{database}{table}{$table}{schema} ? $anvil->data->{sys}{database}{table}{$table}{schema} : "public";
my $host_column = $anvil->data->{sys}{database}{table}{$table}{host_column} ? $anvil->data->{sys}{database}{table}{$table}{host_column} : "";
# If the 'schema' is 'public', there is no table in the history schema.
my $schema = $anvil->data->{sys}{database}{table}{$table}{schema} ? $anvil->data->{sys}{database}{table}{$table}{schema} : "public";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
table => $table,
schema => $schema,
host_column => $host_column,
table => $table,
schema => $schema,
}});
# If there is a column name that is '<table>_uuid', or the same with the table's name minus
@ -16061,16 +16088,24 @@ sub resync_databases
$anvil->data->{db_resync}{$uuid}{public}{sql} = [];
$anvil->data->{db_resync}{$uuid}{history}{sql} = [];
# Read in the data, modified_date first as we'll need that for all entries we record.
my $query = "SELECT DISTINCT modified_date AT time zone 'UTC' AS utc_modified_date, $uuid_column, ";
### NOTE: The history_id is used to insure that the first duplicate entry is the one
### we want to save, and any others are deleted.
# Read in the data, history_id and modified_date first as we'll need that for all entries we record.
my $query = "SELECT modified_date AT time zone 'UTC' AS utc_modified_date, $uuid_column, ";
my $read_columns = [];
if ($schema eq "history")
{
# Most tables are reading from history, but some aren't.
$query = "SELECT history_id, modified_date AT time zone 'UTC' AS utc_modified_date, $uuid_column, ";
push @{$read_columns}, "history_id";
}
push @{$read_columns}, "modified_date";
push @{$read_columns}, $uuid_column;
foreach my $column_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{database}{table}{$table}{column}})
{
# We'll skip the host column as we'll use it in the conditional.
next if $column_name eq "history_id";
next if $column_name eq "modified_date";
next if $column_name eq $host_column;
next if $column_name eq $uuid_column;
$query .= $column_name.", ";
@ -16080,15 +16115,18 @@ sub resync_databases
# Strip the last comma and the add the schema.table name.
$query =~ s/, $/ /;
$query .= "FROM ".$schema.".".$table;
### NOTE: No longer restricting to the host, given only the strikers can do resyncs now.
# # Restrict to this host if a host column was found.
# if ($host_column)
# {
# $query .= " WHERE ".$host_column." = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid});
# }
$query .= " ORDER BY utc_modified_date DESC;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0074", variables => { uuid => $uuid, query => $query }});
if ($schema eq "history")
{
$query .= " ORDER BY utc_modified_date DESC, history_id DESC;";
}
else
{
$query .= " ORDER BY utc_modified_date DESC;";
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0074", variables => {
uuid => $anvil->Database->get_host_from_uuid({short => 1, host_uuid => $uuid}),
query => $query,
}});
my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
@ -16104,10 +16142,13 @@ sub resync_databases
}});
next if not $count;
my $row_number = 0;
# In some cases, a single 'modified_date::uuid_column' can exist multiple times
my $last_record = "";
my $row_number = 0;
foreach my $row (@{$results})
{
$row_number++;
my $history_id = "";
my $modified_date = "";
my $row_uuid = "";
for (my $column_number = 0; $column_number < @{$read_columns}; $column_number++)
@ -16131,7 +16172,14 @@ sub resync_databases
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { column_value => $column_value }});
}
# The modified_date should be the first row.
# The history_id should be the first row.
if ($column_name eq "history_id")
{
$history_id = $column_value;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { history_id => $history_id }});
next;
}
# The modified_date should be the second row.
if ($column_name eq "modified_date")
{
$modified_date = $column_value;
@ -16145,6 +16193,33 @@ sub resync_databases
$row_uuid = $column_value;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { row_uuid => $row_uuid }});
# We should have the modified_date already, is this a
# duplicate?
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
last_record => $last_record,
"modified_date::row_uuid" => "${modified_date}::${row_uuid}",
}});
if (($last_record) && ($schema eq "history") && ($last_record eq "${modified_date}::${row_uuid}"))
{
# Duplicate!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0363", variables => {
table => $table,
key => $last_record,
query => $query,
host => $anvil->Database->get_host_from_uuid({short => 1, host_uuid => $uuid}),
}});
# Delete this entry.
my $query = "DELETE FROM history.".$table." WHERE history_id = ".$history_id.";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
#$anvil->Database->write({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
next;
}
$last_record = $modified_date."::".$row_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { last_record => $last_record }});
# This is used to determine if a given entry needs to be
# updated or inserted into the public schema
$anvil->data->{db_data}{$uuid}{$table}{$uuid_column}{$row_uuid}{'exists'} = 1;
@ -16279,12 +16354,7 @@ sub resync_databases
'values' => $values,
}});
my $query = "INSERT INTO public.$table (".$uuid_column.", ".$columns."modified_date) VALUES (".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');";
if ($host_column)
{
# Add the host column.
$query = "INSERT INTO public.$table ($host_column, $uuid_column, ".$columns."modified_date) VALUES (".$anvil->Database->quote($anvil->data->{sys}{host_uuid}).", ".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');";
}
my $query = "INSERT INTO public.".$table." (".$uuid_column.", ".$columns."modified_date) VALUES (".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0460", variables => { uuid => $anvil->data->{database}{$uuid}{host}, query => $query }});
### NOTE: After an archive operationg, a record can
@ -16359,11 +16429,6 @@ sub resync_databases
}});
my $query = "INSERT INTO history.$table (".$uuid_column.", ".$columns."modified_date) VALUES (".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');";
if ($host_column)
{
# Add the host column.
$query = "INSERT INTO history.$table ($host_column, $uuid_column, ".$columns."modified_date) VALUES (".$anvil->Database->quote($anvil->data->{sys}{host_uuid}).", ".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');";
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0460", variables => { uuid => $anvil->data->{database}{$uuid}{host}, query => $query }});
# Now record the query in the array
@ -16871,7 +16936,7 @@ sub write
if (($anvil->data->{sys}{database}{log_transactions}) or ($debug <= $anvil->Log->level))
{
$anvil->Log->entry({source => $source, line => $line, secure => $secure, level => 0, key => "log_0083", variables => {
uuid => $uuid,
uuid => $anvil->Database->get_host_from_uuid({short => 1, host_uuid => $uuid}),
query => $query,
}});
}
@ -17268,7 +17333,7 @@ sub _age_out_data
if ($commits)
{
# Commit the DELETEs.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0622", variables => {
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0622", variables => {
age => $age,
table => $child_table,
database => $anvil->Get->host_name_from_uuid({host_uuid => $uuid, debug => $debug}),
@ -17734,7 +17799,7 @@ sub _find_behind_databases
}});
}
# Look at all the databases and find the most recent time stamp (and the ID of the DB). Do this by
# Look at all the databases and find the most recent time stamp (and the UUID of the DB). Do this by
# table then by database to keep the counts close together and reduce the chance of tables changing
# between counts.
my $source_updated_time = 0;
@ -17753,45 +17818,6 @@ sub _find_behind_databases
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
next if not $count;
### Get the host_uuid column (we can get this from whatever DB we're reading from.
# Some tables, like 'servers', has a host_uuid column, but it's not used to restrict data to
# a host, but instead show which host a movable resource is on. This prevents us from using
# the column by accident.
my $host_column = "";
if (($table ne "servers") &&
($table ne "jobs"))
{
# Does this table have a '*_host_uuid' column?
my $test_columns = [$table."_host_uuid"];
if ($table =~ /^(.*)s$/)
{
push @{$test_columns}, $1."_host_uuid";
}
if ($table =~ /^(.*)es$/)
{
push @{$test_columns}, $1."_host_uuid";
}
foreach my $test_column (@{$test_columns})
{
my $query = "SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = 'public' AND column_name = ".$anvil->Database->quote($test_column)." AND table_name = ".$anvil->Database->quote($table).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
# See if there is a column that ends in '_host_uuid'. If there is, we'll use
# it later to restrict resync activity to these columns with the local
# 'sys::host_uuid'.
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
if ($count)
{
$host_column = $test_column;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_column => $host_column }});
last;
}
}
}
# Does this table have a history schema version?
$query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'history' AND table_name = ".$anvil->Database->quote($table).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
@ -17817,14 +17843,7 @@ sub _find_behind_databases
SELECT DISTINCT
round(extract(epoch from modified_date)) AS unix_modified_date
FROM
".$schema.".".$table." ";
# if ($host_column)
# {
# $query .= "
# WHERE
# ".$host_column." = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid}) ;
# }
$query .= "
".$schema.".".$table."
ORDER BY
unix_modified_date DESC
;";
@ -17850,13 +17869,11 @@ ORDER BY
$anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated} = $last_updated;
$anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count} = $row_count;
$anvil->data->{sys}{database}{table}{$table}{schema} = $schema;
$anvil->data->{sys}{database}{table}{$table}{host_column} = $host_column;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::table::${table}::uuid::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated},
"sys::database::table::${table}::uuid::${uuid}::row_count" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count},
"sys::database::table::${table}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{last_updated},
"sys::database::table::${table}::schema" => $anvil->data->{sys}{database}{table}{$table}{schema},
"sys::database::table::${table}::host_column" => $anvil->data->{sys}{database}{table}{$table}{host_column},
}});
if ($anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count} > $anvil->data->{sys}{database}{table}{$table}{row_count})

@ -669,7 +669,15 @@ Reply-To: ".$reply_to."
next if $alert_processed;
next if $alert_host_uuid ne $host_uuid;
my $query = "UPDATE alerts SET alert_processed = 1 WHERE alert_uuid = ".$anvil->Database->quote($alert_uuid).";";
my $query = "
UPDATE
alerts
SET
alert_processed = 1,
modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
WHERE
alert_uuid = ".$anvil->Database->quote($alert_uuid)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
}

@ -1524,7 +1524,7 @@ FROM
WHERE
network_interface_operational != 'DELETED'
AND
network_interface_host_uuid = ".$anvil->Database->quote($host_uuid)."
network_interface_host_uuid = ".$anvil->Database->quote($host_uuid)."
ORDER BY
modified_date DESC
;";

@ -5015,7 +5015,15 @@ sub test_ipmi
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => 1, level => $debug, list => { shell_call => $shell_call }});
# Update it.
my $query = "UPDATE hosts SET host_ipmi = ".$anvil->Database->quote($shell_call)." WHERE host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid).";";
my $query = "
UPDATE
hosts
SET
host_ipmi = ".$anvil->Database->quote($shell_call).",
modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
WHERE
host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
;";
$anvil->Database->write({debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__});
}

19
notes

@ -206,6 +206,25 @@ su - postgres -c "dropdb anvil" && su - postgres -c "createdb --owner admin anvi
Reload the DB;
su - postgres -c "dropdb anvil" && su - postgres -c "createdb --owner admin anvil" && su - postgres -c "psql anvil < /anvil.out" && su - postgres -c "psql anvil"
### Load client data
## Workstation setup
dnf -y install postgresql postgresql-server postgresql-plperl
postgresql-setup --initdb --unit postgresql
vim /var/lib/pgsql/data/postgresql.conf
# Add around line 60: listen_addresses = '*'
vim /var/lib/pgsql/data/pg_hba.conf
# Add around like 84: host all all all md5
systemctl start postgresql.service
su - postgres -c "createuser --no-superuser --createdb --no-createrole admin"
su - postgres -c "psql template1 -c \"ALTER ROLE postgres WITH PASSWORD 'Initial1';\""
su - postgres -c "psql template1 -c \"ALTER ROLE admin WITH PASSWORD 'Initial1';\""
# If there was a previous DB
su - postgres -c "dropdb client"
# Copy and load
cp /path/to/client_anvil.out /tmp/anvil.out
su - postgres -c "createdb --owner admin client" && su - postgres -c "psql client < /tmp/anvil.out" && su - postgres -c "psql client"
Changes made using tools such as nmcli do not require a reload but do require the associated interface to be put down and then up again. That can be done by using commands in the following format:

@ -103,23 +103,78 @@ sub clear_old_interfaces
# Read in all interfaces and for each, delete historical records over the age-out time.
my $age = $anvil->data->{scancore}{database}{age_out};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { age => $age }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { age => $age }});
if ($age =~ /\D/)
{
# Age is not valid, set it to defaults.
$age = 24;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { age => $age }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { age => $age }});
}
my $query = "SELECT now() - '".$age."h'::interval;";
my $old_timestamp = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
query => $query,
old_timestamp => $old_timestamp,
}});
$query = "
# It is possible that a record exists on one DB, but not the other. Unsure how this happens, but this
# cleans it up.
foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
uuid => $anvil->Database->get_host_from_uuid({short => 1, host_uuid => $uuid})." (".$uuid.")",
}});
my $query = "
SELECT
ip_address_uuid,
ip_address_address
FROM
ip_addresses
WHERE
ip_address_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
AND
ip_address_note = 'DELETED'
AND
modified_date < '".$old_timestamp."'
ORDER BY
ip_address_address ASC
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $ip_address_uuid = $row->[0];
my $ip_address_address = $row->[1];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:ip_address_uuid' => $ip_address_uuid,
's2:ip_address_address' => $ip_address_address,
}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_network_log_0005", variables => {
age => $age,
ip => $ip_address_address,
}});
my $queries = [];
push @{$queries}, "DELETE FROM history.ip_addresses WHERE ip_address_uuid = '".$ip_address_uuid."';";
push @{$queries}, "DELETE FROM ip_addresses WHERE ip_address_uuid = '".$ip_address_uuid."';";
foreach my $query (@{$queries})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
}
# Write to both DBs.
$anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__});
}
# Remove interfaces
$query = "
SELECT
network_interface_uuid,
network_interface_mac_address,
@ -127,45 +182,144 @@ SELECT
FROM
network_interfaces
WHERE
network_interface_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
network_interface_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
AND
network_interface_operational = 'DELETED'
AND
modified_date < '".$old_timestamp."'
modified_date < '".$old_timestamp."'
ORDER BY
network_interface_name ASC;
network_interface_name ASC
;";
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $network_interface_uuid = $row->[0];
my $network_interface_mac_address = $row->[1];
my $network_interface_name = $row->[2];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
$count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:network_interface_uuid' => $network_interface_uuid,
's2:network_interface_mac_address' => $network_interface_mac_address,
's3:network_interface_name' => $network_interface_name,
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $network_interface_uuid = $row->[0];
my $network_interface_mac_address = $row->[1];
my $network_interface_name = $row->[2];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:network_interface_uuid' => $network_interface_uuid,
's2:network_interface_mac_address' => $network_interface_mac_address,
's3:network_interface_name' => $network_interface_name,
}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_network_log_0002", variables => {
age => $age,
mac => $network_interface_mac_address,
name => $network_interface_name,
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_network_log_0002", variables => {
age => $age,
mac => $network_interface_mac_address,
name => $network_interface_name,
}});
my $queries = [];
push @{$queries}, "DELETE FROM history.network_interfaces WHERE network_interface_uuid = '".$network_interface_uuid."';";
push @{$queries}, "DELETE FROM network_interfaces WHERE network_interface_uuid = '".$network_interface_uuid."';";
foreach my $query (@{$queries})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
}
# Write to both DBs.
$anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__});
}
# Delete old bonds
$query = "
SELECT
bond_uuid,
bond_name
FROM
bonds
WHERE
bond_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
AND
bond_mode = 'DELETED'
AND
modified_date < '".$old_timestamp."'
ORDER BY
bond_name ASC
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
$count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $bond_uuid = $row->[0];
my $bond_name = $row->[1];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:bond_uuid' => $bond_uuid,
's2:bond_name' => $bond_name,
}});
my $queries = [];
push @{$queries}, "DELETE FROM history.network_interfaces WHERE network_interface_uuid = '".$network_interface_uuid."';";
push @{$queries}, "DELETE FROM network_interfaces WHERE network_interface_uuid = '".$network_interface_uuid."';";
foreach my $query (@{$queries})
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_network_log_0003", variables => {
age => $age,
name => $bond_name,
}});
my $queries = [];
push @{$queries}, "DELETE FROM history.bonds WHERE bond_uuid = '".$bond_uuid."';";
push @{$queries}, "DELETE FROM bonds WHERE bond_uuid = '".$bond_uuid."';";
foreach my $query (@{$queries})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
}
# Write to both DBs.
$anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__});
}
# Delete old bridges
$query = "
SELECT
bridge_uuid,
bridge_name
FROM
bridges
WHERE
bridge_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
AND
bridge_id = 'DELETED'
AND
modified_date < '".$old_timestamp."'
ORDER BY
bridge_name ASC
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
$count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $bridge_uuid = $row->[0];
my $bridge_name = $row->[1];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:bridge_uuid' => $bridge_uuid,
's2:bridge_name' => $bridge_name,
}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_network_log_0004", variables => {
age => $age,
name => $bridge_name,
}});
my $queries = [];
push @{$queries}, "DELETE FROM history.bridges WHERE bridge_uuid = '".$bridge_uuid."';";
push @{$queries}, "DELETE FROM bridges WHERE bridge_uuid = '".$bridge_uuid."';";
foreach my $query (@{$queries})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
}
# Write to both DBs.
$anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__});
}
$anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__});
}
return(0);
@ -1274,7 +1428,8 @@ FROM
WHERE
network_interface_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
ORDER BY
network_interface_name ASC;";
network_interface_name ASC
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
$results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$count = @{$results};
@ -1353,9 +1508,14 @@ ORDER BY
uuid => $network_interface_uuid,
}});
my $query = "DELETE FROM network_interfaces WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
my $queries = [];
push @{$queries}, "DELETE FROM history.network_interfaces WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";";
push @{$queries}, "DELETE FROM network_interfaces WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";";
foreach my $query (@{$queries})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
}
$anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__});
$count--;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
@ -1372,9 +1532,14 @@ ORDER BY
uuid => $network_interface_uuid,
}});
my $query = "DELETE FROM network_interfaces WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
my $queries = [];
push @{$queries}, "DELETE FROM history.network_interfaces WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";";
push @{$queries}, "DELETE FROM network_interfaces WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";";
foreach my $query (@{$queries})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
}
$anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__});
$count--;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});

@ -216,5 +216,8 @@ This mode is NOT supported by the Anvil! Intelligent Availability™ platform!
<key name="scan_network_log_0001">Aging out RX and TX data under: [#!variable!records!#] interfaces. These have 1 or more historical records older than: [#!variable!age!#] hours old from the database host: [#!variable!host!#].</key>
<key name="scan_network_log_0002">The old network interface: [#!variable!name!#] with the MAC address: [#!variable!mac!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database.</key>
<key name="scan_network_log_0003">The old bond: [#!variable!name!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database.</key>
<key name="scan_network_log_0004">The old bridge: [#!variable!name!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database.</key>
<key name="scan_network_log_0005">The old IP address: [#!variable!ip!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database.</key>
</language>
</words>

@ -508,6 +508,7 @@ The output, if any, was;
<key name="error_0360">Unable to find the Anvil! information for the Anvil! UUID: [#!variable!anvil_uuid!#].</key>
<key name="error_0361">Unable to find the DRBD config from either node in the Anvil! with the Anvil! UUID: [#!variable!anvil_uuid!#]. Has scan_drbd (as part of scancore) run on either nodes?</key>
<key name="error_0362"><![CDATA[The level: [#!variable!level!#] is invalid. Please use '--level <critical,warning,notice,info>' to specify the alert level of the test message.]]></key>
<key name="error_0363">There are two or more entries on the host: [#!variable!host!#] in the history table: [#!variable!table!#]! The duplicate modidied_date and column UUID are: [#!variable!key!#] (time is UTC), and the query that exposed the dupplicate was: [#!variable!query!#]. This is likely caused by two database writes where the 'modified_date' wasn't updated between writes.</key>
<!-- Files templates -->
<!-- NOTE: Translating these files requires an understanding of which lines are translatable -->

@ -861,7 +861,15 @@ AND
if ((not exists $anvil->data->{seen}{bond}{$bond_name}) or (not $anvil->data->{seen}{bond}{$bond_name}))
{
# Mark it as deleted.
my $query = "UPDATE bonds SET bond_mode = 'DELETED' WHERE bond_uuid = ".$anvil->Database->quote($bond_uuid).";";
my $query = "
UPDATE
bonds
SET
bond_mode = 'DELETED',
modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
WHERE
bond_uuid = ".$anvil->Database->quote($bond_uuid)."
;";
$anvil->Database->write({debug => 3, query => $query, source => $THIS_FILE, line => __LINE__});
# Remove it from the hash so we don't add it to the .json and .xml files.
@ -918,7 +926,14 @@ AND
if ((not exists $anvil->data->{seen}{bridge}{$bridge_name}) or (not $anvil->data->{seen}{bridge}{$bridge_name}))
{
# Mark it as deleted.
my $query = "UPDATE bridges SET bridge_id = 'DELETED' WHERE bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
my $query = "
UPDATE
bridges
SET
bridge_id = 'DELETED',
modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
WHERE
bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
$anvil->Database->write({debug => 3, query => $query, source => $THIS_FILE, line => __LINE__});
# Remove it from the hash so we don't add it to the .json and .xml files.
@ -997,8 +1012,16 @@ ORDER BY
if ($anvil->data->{network_interfaces}{$network_interface_uuid}{network_interface_operational} ne "DELETED")
{
# Mark it as deleted.
my $query = "UPDATE network_interfaces SET network_interface_operational = 'DELETED' WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $query = "
UPDATE
network_interfaces
SET
network_interface_operational = 'DELETED',
modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
WHERE
network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid)."
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0124", variables => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
}

@ -311,7 +311,7 @@ if ($local_uuid)
if ($create_user)
{
# Create the user
my $shell_call = $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{createuser}." --no-superuser --createdb --no-createrole $database_user\"";
my $shell_call = $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{createuser}." --no-superuser --createdb --no-createrole ".$database_user."\"";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($create_output, $return_code) = $anvil->System->call({shell_call => $shell_call, debug => 2, source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {

Loading…
Cancel
Save