diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 3c0c7cd2..e160977a 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -627,7 +627,7 @@ sub check_agent_data # Hold if a lock has been requested. $anvil->Database->locking({debug => $debug}); - # Mark that we're not active. + # Mark that we're now active. $anvil->Database->mark_active({debug => $debug, set => 1}); # Sync the database, if needed. @@ -809,8 +809,8 @@ sub configure_pgsql # If we didn't get the $local_uuid, then there is no entry for this system in anvil.conf yet, so we'll add it. if (not $local_uuid) { - $local_uuid = $anvil->Database->_add_to_local_config({debug => 2}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { local_uuid => $local_uuid }}); + $local_uuid = $anvil->Database->_add_to_local_config({debug => $debug}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { local_uuid => $local_uuid }}); if ($local_uuid eq "!!error!!") { @@ -1730,7 +1730,7 @@ sub connect if ($anvil->data->{sys}{database}{read_uuid} eq $uuid) { $anvil->data->{sys}{database}{read_uuid} = ""; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid} }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid} }}); } next; } @@ -1779,12 +1779,17 @@ sub connect push @{$successful_connections}, $uuid; } - # Before we try to connect, see if this is a local database and, if so, make sure it's setup. - if ($is_local) + # We always use the first DB we connect to, even if we're a DB ourselves. This helps with + # consistency and leaves second (or third...) as backups. + if (not $anvil->data->{sys}{database}{read_uuid}) { $anvil->data->{sys}{database}{read_uuid} = $uuid; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid} }}); - + } + + # Before we try to connect, see if this is a local database and, if so, make sure it's setup. + if ($is_local) + { # If we're being called with 'all', don't set active as we could be just checking if # we're active or not. if (not $all) @@ -1811,15 +1816,9 @@ sub connect } } } - elsif (not $anvil->data->{sys}{database}{read_uuid}) - { - $anvil->data->{sys}{database}{read_uuid} = $uuid; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid} }}); - } - # If this isn't a local database, read the target's Anvil! version (if available) and make # sure it matches ours. If it doesn't, skip this database. - if (not $is_local) + else { my ($local_anvil_version, $local_schema_version) = $anvil->_anvil_version({debug => $debug}); my ($remote_anvil_version, $remote_schema_version) = $anvil->Get->anvil_version({ @@ -1847,8 +1846,9 @@ sub connect # Delete the information about this database. We'll try again on next # ->connect(). - delete $anvil->data->{database}{$uuid}; + $anvil->data->{sys}{database}{read_uuid} = "" if $anvil->data->{sys}{database}{read_uuid} eq $uuid; $anvil->data->{sys}{database}{connections}--; + delete $anvil->data->{database}{$uuid}; next; } } @@ -2172,21 +2172,22 @@ sub connect } # If we have a previous count and the new count is higher, resync. - if (exists $anvil->data->{sys}{database}{last_db_count}) + if (not exists $anvil->data->{sys}{database}{last_db_count}) { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "sys::database::last_db_count" => $anvil->data->{sys}{database}{last_db_count}, - "sys::database::connections" => $anvil->data->{sys}{database}{connections}, - }}); - if ($anvil->data->{sys}{database}{connections} > $anvil->data->{sys}{database}{last_db_count}) - { - $check_for_resync = 1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { check_for_resync => $check_for_resync }}); - } + $anvil->data->{sys}{database}{last_db_count} = 0; + } + + # If "last_db_count" is the lower than the current number of connections, check for a resync. + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "sys::database::last_db_count" => $anvil->data->{sys}{database}{last_db_count}, + "sys::database::connections" => $anvil->data->{sys}{database}{connections}, + }}); + if ($anvil->data->{sys}{database}{connections} > $anvil->data->{sys}{database}{last_db_count}) + { + $check_for_resync = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { check_for_resync => $check_for_resync }}); } - # If we have a "last_db_count" and it's the lower than the current number of connections, check for a - # resync. $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::connections" => $anvil->data->{sys}{database}{connections}, check_for_resync => $check_for_resync, @@ -3210,6 +3211,7 @@ sub get_host_from_uuid $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->get_host_from_uuid()" }}); my $host_name = ""; + my $short_host_name = ""; my $host_uuid = defined $parameter->{host_uuid} ? $parameter->{host_uuid} : ""; my $include_deleted = defined $parameter->{include_deleted} ? $parameter->{include_deleted} : 0; my $short = defined $parameter->{short} ? $parameter->{short} : 0; @@ -3226,6 +3228,19 @@ sub get_host_from_uuid return($host_name); } + # If we queried this before, return the cached value. + if (exists $anvil->data->{host_from_uuid}{$host_uuid}) + { + if ($short) + { + return($anvil->data->{host_from_uuid}{$host_uuid}{short}); + } + else + { + return($anvil->data->{host_from_uuid}{$host_uuid}{full}); + } + } + my $query = " SELECT host_name @@ -3251,17 +3266,29 @@ AND }}); if ($count) { - $host_name = $results->[0]->[0]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_name => $host_name }}); + $host_name = $results->[0]->[0]; + $short_host_name = ($host_name =~ /^(.*?)\..*$/)[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + host_name => $host_name, + short_host_name => $short_host_name, + }}); - if ($short) - { - $host_name =~ s/\..*$//; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_name => $host_name }}); - } + $anvil->data->{host_from_uuid}{$host_uuid}{full} = $host_name; + $anvil->data->{host_from_uuid}{$host_uuid}{short} = $short_host_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "host_from_uuid::${host_uuid}::full" => $anvil->data->{host_from_uuid}{$host_uuid}{full}, + "host_from_uuid::${host_uuid}::short" => $anvil->data->{host_from_uuid}{$host_uuid}{short}, + }}); } - return($host_name); + if ($short) + { + return($anvil->data->{host_from_uuid}{$host_uuid}{short}); + } + else + { + return($anvil->data->{host_from_uuid}{$host_uuid}{full}); + } } @@ -3613,9 +3640,9 @@ SELECT FROM bridges WHERE - bridge_host_uuid = ".$anvil->Database->quote($host_uuid)." + bridge_host_uuid = ".$anvil->Database->quote($host_uuid)." AND - bridge_id != 'DELETED' + bridge_id != 'DELETED' ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); @@ -3687,7 +3714,7 @@ SELECT FROM network_interfaces WHERE - network_interface_host_uuid = ".$anvil->Database->quote($host_uuid)." + network_interface_host_uuid = ".$anvil->Database->quote($host_uuid)." AND network_interface_operational != 'DELETED' ;"; @@ -3779,7 +3806,15 @@ AND { # This isn't a network we should know about (ie: it might be a stray 'virbrX' # birdge), delete this IP. - my $query = "UPDATE ip_addresses SET ip_address_note = 'DELETED' WHERE ip_address_uuid = ".$anvil->Database->quote($ip_address_uuid).";"; + my $query = " +UPDATE + ip_addresses +SET + ip_address_note = 'DELETED', + modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." +WHERE + ip_address_uuid = ".$anvil->Database->quote($ip_address_uuid)." +;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); next; @@ -5112,8 +5147,12 @@ WHERE anvil_name => $anvil_name, }}); - my $query = "DELETE FROM storage_group_members WHERE storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid).";"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { query => $query }}); + my $query = "DELETE FROM history.storage_group_members WHERE storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid).";"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); + + $query = "DELETE FROM storage_group_members WHERE storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid).";"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); } } @@ -5235,7 +5274,7 @@ INSERT INTO ".$anvil->Database->quote($closest_scan_lvm_vg_uuid).", ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." );"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { query => $query }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); # Reload @@ -9851,6 +9890,7 @@ SET WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid)." ;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); } return($network_interface_uuid); @@ -12288,7 +12328,7 @@ sub insert_or_update_states $hosts_ok = 0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { hosts_ok => $hosts_ok }}); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "warning_0144", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "warning_0144", variables => { state_info => $state_name." -> ".$state_note, db_uuid => $db_uuid, host_uuid => $state_host_uuid, @@ -12379,50 +12419,9 @@ INSERT INTO } else { - # Query the rest of the values and see if anything changed. + # There's no history schema so we just UPDATE (in case, as in DB locking, the age since last + # update is important). my $query = " -SELECT - state_name, - state_host_uuid, - state_note -FROM - states -WHERE - state_uuid = ".$anvil->Database->quote($state_uuid)." -;"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - - my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); - my $count = @{$results}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - results => $results, - count => $count, - }}); - if (not $count) - { - # I have a state_uuid but no matching record. Probably an error. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0216", variables => { uuid_name => "state_uuid", uuid => $state_uuid }}); - return(""); - } - - foreach my $row (@{$results}) - { - my $old_state_name = $row->[0]; - my $old_state_host_uuid = $row->[1]; - my $old_state_note = $row->[2]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - old_state_name => $old_state_name, - old_state_host_uuid => $old_state_host_uuid, - old_state_note => $old_state_note, - }}); - - # Anything change? - if (($old_state_name ne $state_name) or - ($old_state_host_uuid ne $state_host_uuid) or - ($old_state_note ne $state_note)) - { - # Something changed, save. - my $query = " UPDATE states SET @@ -12433,10 +12432,8 @@ SET WHERE state_uuid = ".$anvil->Database->quote($state_uuid)." "; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - $anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); - } - } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + $anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { state_uuid => $state_uuid }}); @@ -14584,10 +14581,10 @@ sub locking my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->locking()" }}); - my $request = defined $parameter->{request} ? $parameter->{request} : 0; - my $release = defined $parameter->{release} ? $parameter->{release} : 0; - my $renew = defined $parameter->{renew} ? $parameter->{renew} : 0; - my $check = defined $parameter->{check} ? $parameter->{check} : 0; + my $request = defined $parameter->{request} ? $parameter->{request} : 0; + my $release = defined $parameter->{release} ? $parameter->{release} : 0; + my $renew = defined $parameter->{renew} ? $parameter->{renew} : 0; + my $check = defined $parameter->{check} ? $parameter->{check} : 0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { request => $request, release => $release, @@ -14596,21 +14593,47 @@ sub locking }}); # These are used to ID this lock. - my $source_name = $anvil->Get->host_name; - my $source_uuid = $anvil->data->{sys}{host_uuid}; + my $source_name = $anvil->Get->short_host_name; + my $source_uuid = $anvil->Get->host_uuid; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { source_name => $source_name, source_uuid => $source_uuid, }}); my $set = 0; - my $variable_name = "lock_request"; - my $variable_value = $source_name."::".$source_uuid."::".time; + my $state_name = "lock_request"; + my $new_state_note = $source_name."::".$source_uuid."::".time; + my $old_state_note = $source_name."::".$source_uuid."::%"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - variable_name => $variable_name, - variable_value => $variable_value, + state_name => $state_name, + new_state_note => $new_state_note, + old_state_note => $old_state_note, }}); + my $wildcard_select_query = " +SELECT + state_note +FROM + states +WHERE + state_host_uuid = ".$anvil->Database->quote($source_uuid)." +AND + state_name = ".$anvil->Database->quote($state_name)." +AND + state_note LIKE ".$anvil->Database->quote($old_state_note)." +;"; + my $wildcard_delete_query = " +DELETE FROM + states +WHERE + state_host_uuid = ".$anvil->Database->quote($source_uuid)." +AND + state_name = ".$anvil->Database->quote($state_name)." +AND + state_note LIKE ".$anvil->Database->quote($old_state_note)." +;"; + + # Make sure we have a sane lock age if ((not defined $anvil->data->{sys}{database}{locking}{reap_age}) or (not $anvil->data->{sys}{database}{locking}{reap_age}) or @@ -14621,17 +14644,15 @@ sub locking $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::locking::reap_age" => $anvil->data->{sys}{database}{locking}{reap_age} }}); } - # If I have been asked to check, we will return the variable_uuid if a lock is set. + # If I have been asked to check, we will return the state_note if a lock is set. if ($check) { - my ($lock_value, $variable_uuid, $modified_date) = $anvil->Database->read_variable({debug => $debug, variable_name => $variable_name}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - lock_value => $lock_value, - variable_uuid => $variable_uuid, - modified_date => $modified_date, - }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { wildcard_select_query => $wildcard_select_query }}); + my $state_note = $anvil->Database->query({query => $wildcard_select_query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $state_note = "" if not defined $state_note; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { state_note => $state_note }}); - return($lock_value); + return($state_note); } # If I've been asked to clear a lock, do so now. @@ -14639,32 +14660,25 @@ sub locking { # We check to see if there is a lock before we clear it. This way we don't log that we # released a lock unless we really released a lock. - my ($lock_value, $variable_uuid, $modified_date) = $anvil->Database->read_variable({debug => $debug, line => __LINE__, variable_name => $variable_name}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - lock_value => $lock_value, - variable_uuid => $variable_uuid, - modified_date => $modified_date, - }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { wildcard_select_query => $wildcard_select_query }}); + my $results = $anvil->Database->query({debug => $debug, query => $wildcard_select_query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); - if ($lock_value) + if ($count) { - my $variable_uuid = $anvil->Database->insert_or_update_variables({ - variable_name => $variable_name, - variable_value => "", - variable_default => "", - variable_description => "striker_0289", - variable_section => "database", - variable_source_uuid => "NULL", - variable_source_table => "", - }); + ### NOTE: There is not history schema for states, so we just delete it. + # Delete the state(s). + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { wildcard_delete_query => $wildcard_delete_query }}); + $anvil->Database->write({debug => $debug, query => $wildcard_delete_query, source => $THIS_FILE, line => __LINE__}); + $anvil->data->{sys}{database}{local_lock_active} = 0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - variable_uuid => $variable_uuid, "sys::local_lock_active" => $anvil->data->{sys}{database}{local_lock_active}, }}); # Log that the lock has been released. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0039", variables => { host => $anvil->Get->host_name }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0039", variables => { host => $anvil->Get->host_name }}); } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { set => $set }}); @@ -14674,34 +14688,41 @@ sub locking # If I've been asked to renew, do so now. if ($renew) { - # Yup, do it. - my $variable_uuid = $anvil->Database->insert_or_update_variables({ - variable_name => $variable_name, - variable_value => $variable_value, - variable_default => "", - variable_description => "striker_0289", - variable_section => "database", - variable_source_uuid => "NULL", - variable_source_table => "", - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { variable_uuid => $variable_uuid }}); - - if ($variable_uuid) + # Yup, do it. Delete any old states first, thoguh. Batch them together to avoid there being + # a time where another process could falsely see no locks are held. + my $queries = []; + push @{$queries}, $wildcard_delete_query; + push @{$queries}, " +INSERT INTO + states +( + state_uuid, + state_name, + state_host_uuid, + state_note, + modified_date +) VALUES ( + ".$anvil->Database->quote($anvil->Get->uuid).", + ".$anvil->Database->quote($state_name).", + ".$anvil->Database->quote($source_uuid).", + ".$anvil->Database->quote($new_state_note).", + ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." +);"; + foreach my $query (@{$queries}) { - $set = 1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { set => $set }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); } + $anvil->Database->write({debug => $debug, query => $queries, source => $THIS_FILE, line => __LINE__}); + $anvil->data->{sys}{database}{local_lock_active} = time; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - variable_uuid => $variable_uuid, "sys::local_lock_active" => $anvil->data->{sys}{database}{local_lock_active}, }}); # Log that we've renewed the lock. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0044", variables => { host => $anvil->Get->host_name }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0044", variables => { host => $anvil->Get->short_host_name }}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { set => $set }}); - return($set); + return(1); } # We always check for, and then wait for, locks. Read in the locks, if any. If any are set and they are @@ -14713,12 +14734,13 @@ sub locking $waiting = 0; # See if we had a lock. - my ($lock_value, $variable_uuid, $modified_date) = $anvil->Database->read_variable({debug => $debug, variable_name => $variable_name}); + my ($lock_value, $state_uuid, $modified_date) = $anvil->Database->read_state({debug => $debug, state_name => $state_name}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - waiting => $waiting, - lock_value => $lock_value, - variable_uuid => $variable_uuid, - modified_date => $modified_date, + waiting => $waiting, + lock_value => $lock_value, + state_uuid => $state_uuid, + state_host_uuid => $anvil->Get->host_uuid, + modified_date => $modified_date, }}); if ($lock_value =~ /^(.*?)::(.*?)::(\d+)/) { @@ -14740,17 +14762,11 @@ sub locking # If the lock is stale, delete it. if ($current_time > $timeout_time) { + ### NOTE: There is no history schema for states. # The lock is stale. - my $variable_uuid = $anvil->Database->insert_or_update_variables({ - variable_name => $variable_name, - variable_value => "", - variable_default => "", - variable_description => "striker_0289", - variable_section => "database", - variable_source_uuid => "", - variable_source_table => "", - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { variable_uuid => $variable_uuid }}); + my $query = "DELETE FROM states WHERE state_uuid = ".$state_uuid.";"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + $anvil->Database->write({debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__}); } # Only wait if this isn't our own lock. elsif ($lock_source_uuid ne $source_uuid) @@ -14773,30 +14789,50 @@ sub locking if ($request) { # Yup, do it. - my $variable_uuid = $anvil->Database->insert_or_update_variables({ - variable_name => $variable_name, - variable_value => $variable_value, - variable_default => "", - variable_description => "striker_0289", - variable_section => "database", - variable_source_uuid => "NULL", - variable_source_table => "", - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { variable_uuid => $variable_uuid }}); + my $state_uuid = $anvil->Get->uuid; + my $queries = []; + push @{$queries}, $wildcard_delete_query; + push @{$queries}, " +INSERT INTO + states +( + state_uuid, + state_name, + state_host_uuid, + state_note, + modified_date +) VALUES ( + ".$anvil->Database->quote($state_uuid).", + ".$anvil->Database->quote($state_name).", + ".$anvil->Database->quote($source_uuid).", + ".$anvil->Database->quote($new_state_note).", + ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." +);"; + foreach my $query (@{$queries}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + } + my $problem = $anvil->Database->write({debug => $debug, query => $queries, source => $THIS_FILE, line => __LINE__}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }}); - if ($variable_uuid) + # If we have a problem, it could be that we're locking against a database not yet in the + # hosts file. + if ($problem) { - $set = 1; - $anvil->data->{sys}{database}{local_lock_active} = time; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - set => $set, - variable_uuid => $variable_uuid, - "sys::local_lock_active" => $anvil->data->{sys}{database}{local_lock_active}, - }}); - - # Log that we've got the lock. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0045", variables => { host => $anvil->Get->host_name }}); + # No lock + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { set => $set }}); + return($set); } + + $set = 1; + $anvil->data->{sys}{database}{local_lock_active} = time; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + set => $set, + "sys::local_lock_active" => $anvil->data->{sys}{database}{local_lock_active}, + }}); + + # Log that we've got the lock. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0045", variables => { host => $anvil->Get->short_host_name }}); } # Now return. @@ -15327,31 +15363,32 @@ sub mark_active return(0); } - my $value = $set ? 1 : 0; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { value => $value }}); - + # Record that we're using each available striker DB UUID. foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}}) { - # TODO: When unsetting, should we just go directly to a deletion? This method gets us the - # state_uuid though, which is convenient. - my $pid = $$; - my $state_name = "db_in_use::".$uuid."::".$pid; - my $state_uuid = $anvil->Database->insert_or_update_states({ - debug => $debug, - state_name => $state_name, - state_host_uuid => $anvil->data->{sys}{host_uuid}, - state_note => $value, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { state_uuid => $state_uuid }}); + my $state_name = "db_in_use::".$uuid."::".$$; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + set => $set, + state_name => $state_name, + }}); - # Being a state, if we're clearing, now delete the entry. - # NOTE: The 'state' table has no history schema - if (not $set) + if ($set) { - # Broadly clear all states that are '0' now. - my $query = "DELETE FROM states WHERE state_name LIKE 'db_in_use%' AND state_note != '1';"; + my $state_uuid = $anvil->Database->insert_or_update_states({ + debug => $debug, + state_name => $state_name, + state_host_uuid => $anvil->Get->host_uuid, + state_note => "1", + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { state_uuid => $state_uuid }}); + } + else + { + ### NOTE: The 'state' table has no history schema + # Delete this specific db_in_use, if it exists. + my $query = "DELETE FROM states WHERE state_name = ".$anvil->Database->quote($state_name)." AND state_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid).";"; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0124", variables => { query => $query }}); - $anvil->Database->write({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); + $anvil->Database->write({debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__}); } } return(0); @@ -15737,6 +15774,123 @@ sub read } +=head2 read_state + +This reads a C<< state_note >> from the C<< states >> table. An anonymous array reference is returned with the C<< state_name >>, C<< state_uuid >>, and C<< modified_date >> (in unix time format) in that order. + +If anything goes wrong, C<< !!error!! >> is returned for all values in the array reference. If the state didn't exist in the database, an empty string will be returned. + +Parameters; + +=head3 state_uuid (optional) + +If specified, this specifies the state UUID to read. When this parameter is specified, the C<< state_name >> parameter is ignored. + +=head3 state_name (required) + +This is the name of the state we're reading. + +=head3 state_host_uuid (optional) + +This is the C<< host_uuid >> of the state we're reading + +=head3 uuid (optional) + +If set, this specified which database to read the C<< state_note >> from. + +=cut +sub read_state +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->read_state()" }}); + + my $state_uuid = $parameter->{state_uuid} ? $parameter->{state_uuid} : ""; + my $state_name = $parameter->{state_name} ? $parameter->{state_name} : ""; + my $state_host_uuid = $parameter->{state_host_uuid} ? $parameter->{state_host_uuid} : ""; + my $uuid = $parameter->{uuid} ? $parameter->{uuid} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + uuid => $uuid, + state_uuid => $state_uuid, + state_name => $state_name, + state_host_uuid => $state_host_uuid, + }}); + + if ((not $uuid) && ($anvil->data->{sys}{database}{read_uuid})) + { + $uuid = $anvil->data->{sys}{database}{read_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }}); + } + + # Do we have either the state name or UUID? + if ((not $state_name) && (not $state_uuid)) + { + # Throw an error and exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0704"}); + return("!!error!!", "!!error!!", "!!error!!"); + } + + # If we don't have a UUID, see if we can find one for the given SMTP server name. + my $query = " +SELECT + state_note, + state_uuid, + round(extract(epoch from modified_date)) AS mtime +FROM + states +WHERE "; + if ($state_uuid) + { + $query .= " + state_uuid = ".$anvil->Database->quote($state_uuid); + } + else + { + $query .= " + state_name = ".$anvil->Database->quote($state_name); + if ($state_host_uuid ne "") + { + $query .= " +AND + state_host_uuid = ".$anvil->Database->quote($state_host_uuid)." +"; + } + } + $query .= ";"; + $query =~ s/'NULL'/NULL/g; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0124", variables => { query => $query }}); + + my $state_note = ""; + my $mtime = ""; + my $results = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + $state_note = $row->[0]; + $state_uuid = $row->[1]; + $mtime = $row->[2]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + state_note => $state_note, + state_uuid => $state_uuid, + mtime => $mtime, + }}); + } + + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + state_note => $state_note, + state_uuid => $state_uuid, + mtime => $mtime, + }}); + return($state_note, $state_uuid, $mtime); +} + + =head2 read_variable This reads a variable from the C<< variables >> table. Be sure to only use the reply from here to override what might have been set in a config file. This method always returns the data from the database itself. @@ -15883,10 +16037,38 @@ sub refresh_timestamp my $parameter = shift; my $anvil = $self->parent; - my $query = "SELECT cast(now() AS timestamp with time zone);"; - my $new_time = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + my $match = 0; + my $ok = 0; + until ($ok) + { + my $query = "SELECT cast(now() AS timestamp with time zone);"; + my $new_time = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + + if (($anvil->data->{sys}{database}{timestamp}) && ($anvil->data->{sys}{database}{timestamp} eq $new_time)) + { + # Log that we hit this, then loop until we get a different result. + if (not $match) + { + $match = 1; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0702"}); + } + } + else + { + # Different result. If we looped, log that we're clear now. + $ok = 1; + if ($match) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0703", variables => { + old_time => $anvil->data->{sys}{database}{timestamp}, + new_time => $new_time, + }}); + } + # Store the time stamp. + $anvil->data->{sys}{database}{timestamp} = $new_time; + } + } - $anvil->data->{sys}{database}{timestamp} = $new_time; return($anvil->data->{sys}{database}{timestamp}); } @@ -15952,17 +16134,18 @@ sub resync_databases { # We don't sync 'states' as it's transient and sometimes per-DB. next if $table eq "states"; + + # Don't sync any table that doesn't have a history schema + next if $table eq "alert_sent"; + next if $table eq "states"; + next if $table eq "update"; next if $table eq "oui"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { table => $table }}); - # If the 'schema' is 'public', there is no table in the history schema. If there is a host - # column, the resync will be restricted to entries from this host uuid. - my $schema = $anvil->data->{sys}{database}{table}{$table}{schema} ? $anvil->data->{sys}{database}{table}{$table}{schema} : "public"; - my $host_column = $anvil->data->{sys}{database}{table}{$table}{host_column} ? $anvil->data->{sys}{database}{table}{$table}{host_column} : ""; + # If the 'schema' is 'public', there is no table in the history schema. + my $schema = $anvil->data->{sys}{database}{table}{$table}{schema} ? $anvil->data->{sys}{database}{table}{$table}{schema} : "public"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - table => $table, - schema => $schema, - host_column => $host_column, + table => $table, + schema => $schema, }}); # If there is a column name that is '_uuid', or the same with the table's name minus @@ -16048,16 +16231,24 @@ sub resync_databases $anvil->data->{db_resync}{$uuid}{public}{sql} = []; $anvil->data->{db_resync}{$uuid}{history}{sql} = []; - # Read in the data, modified_date first as we'll need that for all entries we record. - my $query = "SELECT DISTINCT modified_date AT time zone 'UTC' AS utc_modified_date, $uuid_column, "; + ### NOTE: The history_id is used to insure that the first duplicate entry is the one + ### we want to save, and any others are deleted. + # Read in the data, history_id and modified_date first as we'll need that for all entries we record. + my $query = "SELECT modified_date AT time zone 'UTC' AS utc_modified_date, $uuid_column, "; my $read_columns = []; + if ($schema eq "history") + { + # Most tables are reading from history, but some aren't. + $query = "SELECT history_id, modified_date AT time zone 'UTC' AS utc_modified_date, $uuid_column, "; + push @{$read_columns}, "history_id"; + } push @{$read_columns}, "modified_date"; push @{$read_columns}, $uuid_column; foreach my $column_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{database}{table}{$table}{column}}) { # We'll skip the host column as we'll use it in the conditional. + next if $column_name eq "history_id"; next if $column_name eq "modified_date"; - next if $column_name eq $host_column; next if $column_name eq $uuid_column; $query .= $column_name.", "; @@ -16067,15 +16258,18 @@ sub resync_databases # Strip the last comma and the add the schema.table name. $query =~ s/, $/ /; $query .= "FROM ".$schema.".".$table; - - ### NOTE: No longer restricting to the host, given only the strikers can do resyncs now. -# # Restrict to this host if a host column was found. -# if ($host_column) -# { -# $query .= " WHERE ".$host_column." = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid}); -# } - $query .= " ORDER BY utc_modified_date DESC;"; - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0074", variables => { uuid => $uuid, query => $query }}); + if ($schema eq "history") + { + $query .= " ORDER BY utc_modified_date DESC, history_id DESC;"; + } + else + { + $query .= " ORDER BY utc_modified_date DESC;"; + } + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0074", variables => { + uuid => $anvil->Database->get_host_from_uuid({short => 1, host_uuid => $uuid}), + query => $query, + }}); my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; @@ -16091,10 +16285,13 @@ sub resync_databases }}); next if not $count; - my $row_number = 0; + # In some cases, a single 'modified_date::uuid_column' can exist multiple times + my $last_record = ""; + my $row_number = 0; foreach my $row (@{$results}) { $row_number++; + my $history_id = ""; my $modified_date = ""; my $row_uuid = ""; for (my $column_number = 0; $column_number < @{$read_columns}; $column_number++) @@ -16118,7 +16315,14 @@ sub resync_databases $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { column_value => $column_value }}); } - # The modified_date should be the first row. + # The history_id should be the first row. + if ($column_name eq "history_id") + { + $history_id = $column_value; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { history_id => $history_id }}); + next; + } + # The modified_date should be the second row. if ($column_name eq "modified_date") { $modified_date = $column_value; @@ -16132,6 +16336,33 @@ sub resync_databases $row_uuid = $column_value; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { row_uuid => $row_uuid }}); + # We should have the modified_date already, is this a + # duplicate? + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + last_record => $last_record, + "modified_date::row_uuid" => "${modified_date}::${row_uuid}", + }}); + if (($last_record) && ($schema eq "history") && ($last_record eq "${modified_date}::${row_uuid}")) + { + # Duplicate! + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0363", variables => { + table => $table, + key => $last_record, + query => $query, + host => $anvil->Database->get_host_from_uuid({short => 1, host_uuid => $uuid}), + }}); + + # Delete this entry. + my $query = "DELETE FROM history.".$table." WHERE history_id = ".$history_id.";"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + + $anvil->Database->write({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); + next; + } + + $last_record = $modified_date."::".$row_uuid; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { last_record => $last_record }}); + # This is used to determine if a given entry needs to be # updated or inserted into the public schema $anvil->data->{db_data}{$uuid}{$table}{$uuid_column}{$row_uuid}{'exists'} = 1; @@ -16266,12 +16497,7 @@ sub resync_databases 'values' => $values, }}); - my $query = "INSERT INTO public.$table (".$uuid_column.", ".$columns."modified_date) VALUES (".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');"; - if ($host_column) - { - # Add the host column. - $query = "INSERT INTO public.$table ($host_column, $uuid_column, ".$columns."modified_date) VALUES (".$anvil->Database->quote($anvil->data->{sys}{host_uuid}).", ".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');"; - } + my $query = "INSERT INTO public.".$table." (".$uuid_column.", ".$columns."modified_date) VALUES (".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');"; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0460", variables => { uuid => $anvil->data->{database}{$uuid}{host}, query => $query }}); ### NOTE: After an archive operationg, a record can @@ -16346,11 +16572,6 @@ sub resync_databases }}); my $query = "INSERT INTO history.$table (".$uuid_column.", ".$columns."modified_date) VALUES (".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');"; - if ($host_column) - { - # Add the host column. - $query = "INSERT INTO history.$table ($host_column, $uuid_column, ".$columns."modified_date) VALUES (".$anvil->Database->quote($anvil->data->{sys}{host_uuid}).", ".$anvil->Database->quote($row_uuid).", ".$values.$anvil->Database->quote($modified_date)."::timestamp AT TIME ZONE 'UTC');"; - } $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0460", variables => { uuid => $anvil->data->{database}{$uuid}{host}, query => $query }}); # Now record the query in the array @@ -16459,9 +16680,24 @@ sub shutdown variable_source_table => "", }); + # This query will be called repeatedly. + my $query = " +SELECT + state_uuid, + state_name, + state_host_uuid +FROM + states +WHERE + state_name +LIKE + 'db_in_use::".$host_uuid."::%' +AND + state_note = '1' +;"; + # Now wait for all clients to disconnect. my $waiting = 1; - my $query = "SELECT state_uuid, state_name FROM states WHERE state_name LIKE 'db_in_use::".$host_uuid."::%' AND state_note = '1';"; my $wait_time = 600; my $stop_waiting = time + $wait_time; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { @@ -16471,34 +16707,39 @@ sub shutdown }}); while($waiting) { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - + # PIDs will track pids using our DB locally. Users tracks how many other clients are using + # our DB. my $pids = ""; - my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); + my $users = 0; + + # Check for any users using us. + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + results => $results, + count => $count, + }}); if ($count) { - my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); - my $count = @{$results}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - results => $results, - count => $count, - }}); - if ($count) + # Do the same checks we do in anvil-daemon + $anvil->System->pids(); + foreach my $row (@{$results}) { - # Do the same checks we do in anvil-daemon - $anvil->System->pids(); - foreach my $row (@{$results}) + my $state_uuid = $row->[0]; + my $state_name = $row->[1]; + my $state_host_uuid = $row->[2]; + my $state_pid = ($state_name =~ /db_in_use::.*?::(.*)$/)[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:state_uuid' => $state_uuid, + 's2:state_name' => $state_name, + 's3:state_pid' => $state_pid, + 's4:state_host_uuid' => $state_host_uuid, + 's4:our_pid' => $$, + }}); + # If this is held by us, make sure we ignore our active PID. + if ($state_host_uuid eq $anvil->Get->host_uuid) { - my $state_uuid = $row->[0]; - my $state_name = $row->[1]; - my $state_pid = ($state_name =~ /db_in_use::.*?::(.*)$/)[0]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - 's1:state_uuid' => $state_uuid, - 's2:state_name' => $state_name, - 's3:state_pid' => $state_pid, - 's4:our_pid' => $$, - }}); if ($state_pid eq $$) { # This is us, ignore it. @@ -16521,14 +16762,19 @@ sub shutdown $pids .= $state_pid.","; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { pids => $pids }}); } + $pids =~ s/,$//; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { pids => $pids }}); + } + else + { + $users++; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { users => $users }}); } - $pids =~ s/,$//; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { pids => $pids }}); } } # If there's no count, we're done. - if (not $pids) + if ((not $pids) && (not $users)) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0697"}); $waiting = 0; @@ -16555,19 +16801,7 @@ sub shutdown $anvil->Database->write({debug => $debug, uuid => $host_uuid, query => $query, source => $THIS_FILE, line => __LINE__}); # Mark ourself as no longer using the DB - my $pid = $$; - my $state_name = "db_in_use::".$host_uuid."::".$pid; - my $state_uuid = $anvil->Database->insert_or_update_states({ - debug => $debug, - state_name => $state_name, - state_host_uuid => $anvil->data->{sys}{host_uuid}, - state_note => "0", - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { state_uuid => $state_uuid }}); - - $query = "DELETE FROM states WHERE state_name LIKE 'db_in_use%' AND state_note != '1';"; - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0124", variables => { query => $query }}); - $anvil->Database->write({debug => $debug, uuid => $host_uuid, query => $query, source => $THIS_FILE, line => __LINE__}); + $anvil->Database->mark_active->({set => 0}); # Close our own connection. $anvil->Database->locking({debug => $debug, release => 1}); @@ -16858,7 +17092,7 @@ sub write if (($anvil->data->{sys}{database}{log_transactions}) or ($debug <= $anvil->Log->level)) { $anvil->Log->entry({source => $source, line => $line, secure => $secure, level => 0, key => "log_0083", variables => { - uuid => $uuid, + uuid => $anvil->Database->get_host_from_uuid({short => 1, host_uuid => $uuid}), query => $query, }}); } @@ -16869,12 +17103,28 @@ sub write next; } - # Do the do. - $anvil->data->{cache}{database_handle}{$uuid}->do($query) or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0090", variables => { + # Do the do. Do it in an eval block though so that if it fails, we can do something + # useful. + my $test = eval { $anvil->data->{cache}{database_handle}{$uuid}->do($query); }; + $test = "" if not defined $test; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:test' => $test, + 's2:$@' => $@, + }}); + if (not $test) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0090", variables => { query => (not $secure) ? $query : $anvil->Log->is_secure($query), server => $say_server, db_error => $DBI::errstr, }}); + if (($count) or ($transaction)) + { + # Commit the changes. + $anvil->data->{cache}{database_handle}{$uuid}->rollback(); + } + return(1); + } } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); @@ -17255,7 +17505,7 @@ sub _age_out_data if ($commits) { # Commit the DELETEs. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0622", variables => { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0622", variables => { age => $age, table => $child_table, database => $anvil->Get->host_name_from_uuid({host_uuid => $uuid, debug => $debug}), @@ -17622,8 +17872,8 @@ sub _find_column return('!!error!!') if not $table; - my $query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = ".$anvil->Database->quote($table)." AND data_type = 'uuid' AND is_nullable = 'NO' AND column_name LIKE '\%_".$search_column."';"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); + my $query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'anvil' AND table_schema = 'public' AND table_name = ".$anvil->Database->quote($table)." AND data_type = 'uuid' AND column_name LIKE '\%_".$search_column."';"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { @@ -17633,7 +17883,7 @@ sub _find_column if ($count) { my $host_uuid_column = $results->[0]->[0]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_uuid_column => $host_uuid_column }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid_column => $host_uuid_column }}); push @{$anvil->data->{sys}{database}{uuid_tables}}, { table => $table, @@ -17721,7 +17971,7 @@ sub _find_behind_databases }}); } - # Look at all the databases and find the most recent time stamp (and the ID of the DB). Do this by + # Look at all the databases and find the most recent time stamp (and the UUID of the DB). Do this by # table then by database to keep the counts close together and reduce the chance of tables changing # between counts. my $source_updated_time = 0; @@ -17740,45 +17990,6 @@ sub _find_behind_databases $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); next if not $count; - ### Get the host_uuid column (we can get this from whatever DB we're reading from. - # Some tables, like 'servers', has a host_uuid column, but it's not used to restrict data to - # a host, but instead show which host a movable resource is on. This prevents us from using - # the column by accident. - my $host_column = ""; - if (($table ne "servers") && - ($table ne "jobs")) - { - # Does this table have a '*_host_uuid' column? - my $test_columns = [$table."_host_uuid"]; - if ($table =~ /^(.*)s$/) - { - push @{$test_columns}, $1."_host_uuid"; - } - if ($table =~ /^(.*)es$/) - { - push @{$test_columns}, $1."_host_uuid"; - } - - foreach my $test_column (@{$test_columns}) - { - my $query = "SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = 'public' AND column_name = ".$anvil->Database->quote($test_column)." AND table_name = ".$anvil->Database->quote($table).";"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - - # See if there is a column that ends in '_host_uuid'. If there is, we'll use - # it later to restrict resync activity to these columns with the local - # 'sys::host_uuid'. - my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }}); - - if ($count) - { - $host_column = $test_column; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_column => $host_column }}); - last; - } - } - } - # Does this table have a history schema version? $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'history' AND table_name = ".$anvil->Database->quote($table).";"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); @@ -17804,14 +18015,7 @@ sub _find_behind_databases SELECT DISTINCT round(extract(epoch from modified_date)) AS unix_modified_date FROM - ".$schema.".".$table." "; -# if ($host_column) -# { -# $query .= " -# WHERE -# ".$host_column." = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid}) ; -# } - $query .= " + ".$schema.".".$table." ORDER BY unix_modified_date DESC ;"; @@ -17837,13 +18041,11 @@ ORDER BY $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated} = $last_updated; $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count} = $row_count; $anvil->data->{sys}{database}{table}{$table}{schema} = $schema; - $anvil->data->{sys}{database}{table}{$table}{host_column} = $host_column; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::table::${table}::uuid::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated}, "sys::database::table::${table}::uuid::${uuid}::row_count" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count}, "sys::database::table::${table}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{last_updated}, "sys::database::table::${table}::schema" => $anvil->data->{sys}{database}{table}{$table}{schema}, - "sys::database::table::${table}::host_column" => $anvil->data->{sys}{database}{table}{$table}{host_column}, }}); if ($anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count} > $anvil->data->{sys}{database}{table}{$table}{row_count}) @@ -17891,7 +18093,9 @@ ORDER BY foreach my $table (sort {$a cmp $b} keys %{$anvil->data->{sys}{database}{table}}) { # We don't sync 'states' as it's transient and sometimes per-DB. + next if $table eq "alert_sent"; next if $table eq "states"; + next if $table eq "update"; next if $table eq "oui"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { diff --git a/Anvil/Tools/Email.pm b/Anvil/Tools/Email.pm index ec9a7157..434b781c 100644 --- a/Anvil/Tools/Email.pm +++ b/Anvil/Tools/Email.pm @@ -669,7 +669,15 @@ Reply-To: ".$reply_to." next if $alert_processed; next if $alert_host_uuid ne $host_uuid; - my $query = "UPDATE alerts SET alert_processed = 1 WHERE alert_uuid = ".$anvil->Database->quote($alert_uuid).";"; + my $query = " +UPDATE + alerts +SET + alert_processed = 1, + modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." +WHERE + alert_uuid = ".$anvil->Database->quote($alert_uuid)." +;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); } diff --git a/Anvil/Tools/Network.pm b/Anvil/Tools/Network.pm index cc0195a8..9b3e437e 100644 --- a/Anvil/Tools/Network.pm +++ b/Anvil/Tools/Network.pm @@ -1524,7 +1524,7 @@ FROM WHERE network_interface_operational != 'DELETED' AND - network_interface_host_uuid = ".$anvil->Database->quote($host_uuid)." + network_interface_host_uuid = ".$anvil->Database->quote($host_uuid)." ORDER BY modified_date DESC ;"; diff --git a/Anvil/Tools/ScanCore.pm b/Anvil/Tools/ScanCore.pm index 226d236d..da503dd4 100644 --- a/Anvil/Tools/ScanCore.pm +++ b/Anvil/Tools/ScanCore.pm @@ -1602,7 +1602,7 @@ sub post_scan_analysis_node # If we're still here, at least one issue exists. Any kind of load-shed or preventative live # migration decision now depends on our peer's state. So see if we're both in the cluster or not. - my $problem = $anvil->Cluster->parse_cib({debug => 2}); + my $problem = $anvil->Cluster->parse_cib({debug => $debug}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }}); if (not $problem) { diff --git a/Anvil/Tools/System.pm b/Anvil/Tools/System.pm index 3236afd3..a6acd89f 100644 --- a/Anvil/Tools/System.pm +++ b/Anvil/Tools/System.pm @@ -5015,7 +5015,15 @@ sub test_ipmi $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, secure => 1, level => $debug, list => { shell_call => $shell_call }}); # Update it. - my $query = "UPDATE hosts SET host_ipmi = ".$anvil->Database->quote($shell_call)." WHERE host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid).";"; + my $query = " +UPDATE + hosts +SET + host_ipmi = ".$anvil->Database->quote($shell_call).", + modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." +WHERE + host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)." +;"; $anvil->Database->write({debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__}); } @@ -5051,7 +5059,7 @@ sub update_hosts $anvil->Database->get_ip_addresses({debug => $debug}); # Load the IPs we manage. If we find any entries for these that we don't expect, we'll remove them. - $anvil->Database->get_ip_addresses({debug => $debug}); + $anvil->Network->load_ips({debug => $debug}); foreach my $host_uuid (keys %{$anvil->data->{hosts}{host_uuid}}) { diff --git a/notes b/notes index 41aeeed3..544bf03e 100644 --- a/notes +++ b/notes @@ -206,6 +206,25 @@ su - postgres -c "dropdb anvil" && su - postgres -c "createdb --owner admin anvi Reload the DB; su - postgres -c "dropdb anvil" && su - postgres -c "createdb --owner admin anvil" && su - postgres -c "psql anvil < /anvil.out" && su - postgres -c "psql anvil" +### Load client data +## Workstation setup +dnf -y install postgresql postgresql-server postgresql-plperl +postgresql-setup --initdb --unit postgresql +vim /var/lib/pgsql/data/postgresql.conf +# Add around line 60: listen_addresses = '*' +vim /var/lib/pgsql/data/pg_hba.conf +# Add around like 84: host all all all md5 +systemctl start postgresql.service +su - postgres -c "createuser --no-superuser --createdb --no-createrole admin" +su - postgres -c "psql template1 -c \"ALTER ROLE postgres WITH PASSWORD 'Initial1';\"" +su - postgres -c "psql template1 -c \"ALTER ROLE admin WITH PASSWORD 'Initial1';\"" + +# If there was a previous DB +su - postgres -c "dropdb client" + +# Copy and load +cp /path/to/client_anvil.out /tmp/anvil.out +su - postgres -c "createdb --owner admin client" && su - postgres -c "psql client < /tmp/anvil.out" && su - postgres -c "psql client" Changes made using tools such as nmcli do not require a reload but do require the associated interface to be put down and then up again. That can be done by using commands in the following format: diff --git a/scancore-agents/scan-network/scan-network b/scancore-agents/scan-network/scan-network index dbf345b3..91cd894b 100755 --- a/scancore-agents/scan-network/scan-network +++ b/scancore-agents/scan-network/scan-network @@ -103,23 +103,78 @@ sub clear_old_interfaces # Read in all interfaces and for each, delete historical records over the age-out time. my $age = $anvil->data->{scancore}{database}{age_out}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { age => $age }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { age => $age }}); if ($age =~ /\D/) { # Age is not valid, set it to defaults. $age = 24; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { age => $age }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { age => $age }}); } my $query = "SELECT now() - '".$age."h'::interval;"; my $old_timestamp = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query, old_timestamp => $old_timestamp, }}); - $query = " + # It is possible that a record exists on one DB, but not the other. Unsure how this happens, but this + # cleans it up. + foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + uuid => $anvil->Database->get_host_from_uuid({short => 1, host_uuid => $uuid})." (".$uuid.")", + }}); + my $query = " +SELECT + ip_address_uuid, + ip_address_address +FROM + ip_addresses +WHERE + ip_address_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)." +AND + ip_address_note = 'DELETED' +AND + modified_date < '".$old_timestamp."' +ORDER BY + ip_address_address ASC +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $ip_address_uuid = $row->[0]; + my $ip_address_address = $row->[1]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:ip_address_uuid' => $ip_address_uuid, + 's2:ip_address_address' => $ip_address_address, + }}); + + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_network_log_0005", variables => { + age => $age, + ip => $ip_address_address, + }}); + + my $queries = []; + push @{$queries}, "DELETE FROM history.ip_addresses WHERE ip_address_uuid = '".$ip_address_uuid."';"; + push @{$queries}, "DELETE FROM ip_addresses WHERE ip_address_uuid = '".$ip_address_uuid."';"; + foreach my $query (@{$queries}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + } + # Write to both DBs. + $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); + } + + # Remove interfaces + $query = " SELECT network_interface_uuid, network_interface_mac_address, @@ -127,45 +182,144 @@ SELECT FROM network_interfaces WHERE - network_interface_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)." + network_interface_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)." AND network_interface_operational = 'DELETED' -AND - modified_date < '".$old_timestamp."' +AND + modified_date < '".$old_timestamp."' ORDER BY - network_interface_name ASC; + network_interface_name ASC ;"; - my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); - my $count = @{$results}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - results => $results, - count => $count, - }}); - foreach my $row (@{$results}) - { - my $network_interface_uuid = $row->[0]; - my $network_interface_mac_address = $row->[1]; - my $network_interface_name = $row->[2]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); + $count = @{$results}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - 's1:network_interface_uuid' => $network_interface_uuid, - 's2:network_interface_mac_address' => $network_interface_mac_address, - 's3:network_interface_name' => $network_interface_name, + results => $results, + count => $count, }}); - - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_network_log_0002", variables => { - age => $age, - mac => $network_interface_mac_address, - name => $network_interface_name, + foreach my $row (@{$results}) + { + my $network_interface_uuid = $row->[0]; + my $network_interface_mac_address = $row->[1]; + my $network_interface_name = $row->[2]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:network_interface_uuid' => $network_interface_uuid, + 's2:network_interface_mac_address' => $network_interface_mac_address, + 's3:network_interface_name' => $network_interface_name, + }}); + + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_network_log_0002", variables => { + age => $age, + mac => $network_interface_mac_address, + name => $network_interface_name, + }}); + + my $queries = []; + push @{$queries}, "DELETE FROM history.network_interfaces WHERE network_interface_uuid = '".$network_interface_uuid."';"; + push @{$queries}, "DELETE FROM network_interfaces WHERE network_interface_uuid = '".$network_interface_uuid."';"; + foreach my $query (@{$queries}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + } + # Write to both DBs. + $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); + } + + # Delete old bonds + $query = " +SELECT + bond_uuid, + bond_name +FROM + bonds +WHERE + bond_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)." +AND + bond_mode = 'DELETED' +AND + modified_date < '".$old_timestamp."' +ORDER BY + bond_name ASC +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); + $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, }}); + foreach my $row (@{$results}) + { + my $bond_uuid = $row->[0]; + my $bond_name = $row->[1]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:bond_uuid' => $bond_uuid, + 's2:bond_name' => $bond_name, + }}); - my $queries = []; - push @{$queries}, "DELETE FROM history.network_interfaces WHERE network_interface_uuid = '".$network_interface_uuid."';"; - push @{$queries}, "DELETE FROM network_interfaces WHERE network_interface_uuid = '".$network_interface_uuid."';"; - foreach my $query (@{$queries}) + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_network_log_0003", variables => { + age => $age, + name => $bond_name, + }}); + + my $queries = []; + push @{$queries}, "DELETE FROM history.bonds WHERE bond_uuid = '".$bond_uuid."';"; + push @{$queries}, "DELETE FROM bonds WHERE bond_uuid = '".$bond_uuid."';"; + foreach my $query (@{$queries}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + } + # Write to both DBs. + $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); + } + + # Delete old bridges + $query = " +SELECT + bridge_uuid, + bridge_name +FROM + bridges +WHERE + bridge_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)." +AND + bridge_id = 'DELETED' +AND + modified_date < '".$old_timestamp."' +ORDER BY + bridge_name ASC +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__}); + $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + my $bridge_uuid = $row->[0]; + my $bridge_name = $row->[1]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:bridge_uuid' => $bridge_uuid, + 's2:bridge_name' => $bridge_name, + }}); + + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_network_log_0004", variables => { + age => $age, + name => $bridge_name, + }}); + + my $queries = []; + push @{$queries}, "DELETE FROM history.bridges WHERE bridge_uuid = '".$bridge_uuid."';"; + push @{$queries}, "DELETE FROM bridges WHERE bridge_uuid = '".$bridge_uuid."';"; + foreach my $query (@{$queries}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + } + # Write to both DBs. + $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); } - $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); } return(0); @@ -1274,7 +1428,8 @@ FROM WHERE network_interface_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)." ORDER BY - network_interface_name ASC;"; + network_interface_name ASC +;"; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); $count = @{$results}; @@ -1353,9 +1508,14 @@ ORDER BY uuid => $network_interface_uuid, }}); - my $query = "DELETE FROM network_interfaces WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); - $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); + my $queries = []; + push @{$queries}, "DELETE FROM history.network_interfaces WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";"; + push @{$queries}, "DELETE FROM network_interfaces WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";"; + foreach my $query (@{$queries}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + } + $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); $count--; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); @@ -1372,9 +1532,14 @@ ORDER BY uuid => $network_interface_uuid, }}); - my $query = "DELETE FROM network_interfaces WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); - $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); + my $queries = []; + push @{$queries}, "DELETE FROM history.network_interfaces WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";"; + push @{$queries}, "DELETE FROM network_interfaces WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";"; + foreach my $query (@{$queries}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + } + $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); $count--; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); @@ -1456,9 +1621,14 @@ ORDER BY uuid => $ip_address_uuid, }}); - my $query = "DELETE FROM ip_addresses WHERE ip_address_uuid = ".$anvil->Database->quote($ip_address_uuid).";"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); - $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); + my $queries = []; + push @{$queries}, "DELETE FROM history.ip_addresses WHERE ip_address_uuid = ".$anvil->Database->quote($ip_address_uuid).";"; + push @{$queries}, "DELETE FROM ip_addresses WHERE ip_address_uuid = ".$anvil->Database->quote($ip_address_uuid).";"; + foreach my $query (@{$queries}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + } + $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); $count--; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); @@ -1476,9 +1646,14 @@ ORDER BY uuid => $ip_address_uuid, }}); - my $query = "DELETE FROM ip_addresses WHERE ip_address_uuid = ".$anvil->Database->quote($ip_address_uuid).";"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); - $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); + my $queries = []; + push @{$queries}, "DELETE FROM history.ip_addresses WHERE ip_address_uuid = ".$anvil->Database->quote($ip_address_uuid).";"; + push @{$queries}, "DELETE FROM ip_addresses WHERE ip_address_uuid = ".$anvil->Database->quote($ip_address_uuid).";"; + foreach my $query (@{$queries}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + } + $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); $count--; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); diff --git a/scancore-agents/scan-network/scan-network.xml b/scancore-agents/scan-network/scan-network.xml index 459a172a..c512841c 100644 --- a/scancore-agents/scan-network/scan-network.xml +++ b/scancore-agents/scan-network/scan-network.xml @@ -216,5 +216,8 @@ This mode is NOT supported by the Anvil! Intelligent Availability™ platform! Aging out RX and TX data under: [#!variable!records!#] interfaces. These have 1 or more historical records older than: [#!variable!age!#] hours old from the database host: [#!variable!host!#].The old network interface: [#!variable!name!#] with the MAC address: [#!variable!mac!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database. + The old bond: [#!variable!name!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database. + The old bridge: [#!variable!name!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database. + The old IP address: [#!variable!ip!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database. diff --git a/share/words.xml b/share/words.xml index 9f8de2db..73e7b71c 100644 --- a/share/words.xml +++ b/share/words.xml @@ -508,6 +508,8 @@ The output, if any, was; Unable to find the Anvil! information for the Anvil! UUID: [#!variable!anvil_uuid!#].Unable to find the DRBD config from either node in the Anvil! with the Anvil! UUID: [#!variable!anvil_uuid!#]. Has scan_drbd (as part of scancore) run on either nodes?' to specify the alert level of the test message.]]> + There are two or more entries on the host: [#!variable!host!#] in the history table: [#!variable!table!#]! The duplicate modidied_date and column UUID are: [#!variable!key!#] (time is UTC), and the query that exposed the dupplicate was: [#!variable!query!#]. This is likely caused by two database writes where the 'modified_date' wasn't updated between writes. + [ Error ] - There was a problem purging records. The details of the problem should be in the logs. @@ -2120,6 +2122,9 @@ The file: [#!variable!file!#] needs to be updated. The difference is: [ Note ] - The Striker database host: [#!variable!host!#] is inactive, skipping it.[ Note ] - Deleting the contents of the hash: [#!variable!hash!#].Running the scan agent: [#!variable!agent_name!#]... + I was asked to update the timestamp, but the returned timestamp matches the last one. Will loop until a new timestamp is returned. + The timestamp has been updated from: [#!variable!old_time!#] to: [#!variable!new_time!#]. + read_state() was called but both the 'state_name' and 'state_uuid' parameters were not passed or both were empty.]]>The host name: [#!variable!target!#] does not resolve to an IP address. @@ -2540,6 +2545,7 @@ Available options; #!variable!total_cores!#c (#!variable!sockets!#s, #!variable!cores!#c, #!variable!threads!#t), #!variable!model!#, #!variable!mode!##!variable!cores!#c (#!variable!threads!#t)-=] Server Usage and Anvil! Node Resource Availability + This program is currently disabled, please see NOTE in the header for more information.Saved the mail server information successfully! @@ -3211,7 +3217,7 @@ We will sleep a bit and try again. [ Warning ] - Timed out waiting for the connections to the peers, and the local resource(s) is not in 'UpToDate' state. Booting the server will likely fail.[ Warning ] - Timed out waiting for the connections to the peers.[ Warning ] - We're using: [#!variable!ram_used!#] (#!variable!ram_used_bytes!# Bytes). but there is a job: [#!variable!job_command!#] is runnng, which might be why the RAM is high. NOT exiting while this program is running. - [ Warning ] - A no-longer active PID: [#!variable!pid!#] had marked our database as "in_use", but the PID is gone now. Reaping the flag. + [ Warning ] - A no-longer active PID: [#!variable!pid!#] had marked the database: [#!variable!db!#] as "in_use", but the PID is gone now. Reaping the flag.[ Warning ] - We waited for: [#!variable!wait_time!#] seconds for all users of the local database to exit. Giving up waiting and taking the database down now.[ Warning ] - The command: [#!variable!command!#] is still using our database.[ Warning ] - While evaluating database shutdown, the host UUID: [#!variable!host_uuid!#] was not yet found in the database on host: [#!variable!db_uuid!#]. DB shutdown will not happen until all hosts are in all DBs. diff --git a/tools/anvil-daemon b/tools/anvil-daemon index b42d06a8..6843c399 100755 --- a/tools/anvil-daemon +++ b/tools/anvil-daemon @@ -143,6 +143,7 @@ $anvil->data->{switches}{'refresh-json'} = ""; $anvil->data->{switches}{'run-once'} = 0; $anvil->data->{switches}{'main-loop-only'} = 0; $anvil->data->{switches}{'no-start'} = 0; +$anvil->data->{switches}{'purge'} = 0; $anvil->data->{switches}{'startup-only'} = 0; $anvil->Get->switches; @@ -681,6 +682,11 @@ sub handle_periodic_tasks } } + # Reap old db_in_use states over 6 hours old. + my $query = "DELETE FROM states WHERE state_name LIKE 'db_in_use%' AND modified_date < (SELECT now() - interval '6 hour');\n"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); + $anvil->Database->write({debug => 2, query => $query, source => $THIS_FILE, line => __LINE__}); + # Update the next check time. $anvil->data->{timing}{next_ten_minute_check} = $now_time + $anvil->data->{timing}{ten_minute_checks}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { @@ -718,7 +724,7 @@ sub handle_periodic_tasks job_description => "job_0017", job_progress => 0, }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { job_uuid => $job_uuid }}); # Update the OUI data. ($job_uuid) = $anvil->Database->insert_or_update_jobs({ @@ -766,9 +772,7 @@ sub check_db_in_use_states # We only reap db_in_use entries for us. $anvil->System->pids(); - my $host_uuid = $anvil->Database->quote($anvil->Get->host_uuid); - $host_uuid =~ s/^'(.*)'$/$1/; - my $query = " + my $query = " SELECT state_uuid, state_name, @@ -776,7 +780,7 @@ SELECT FROM states WHERE - state_name LIKE 'db_in_use::".$host_uuid."::%' + state_name LIKE 'db_in_use::%' AND state_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)." ;"; @@ -794,18 +798,25 @@ AND my $state_uuid = $row->[0]; my $state_name = $row->[1]; my $state_note = $row->[2]; - my $state_pid = ($state_name =~ /db_in_use::.*?::(.*)$/)[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:state_uuid' => $state_uuid, 's2:state_name' => $state_name, 's3:state_note' => $state_note, - 's4:state_pid' => $state_pid, + }}); + + my ($db_uuid, $state_pid) = ($state_name =~ /db_in_use::(.*?)::(\d+)$/); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:db_uuid' => $anvil->Get->host_name_from_uuid({host_uuid => $db_uuid})." (".$db_uuid.")", + 's4:state_pid' => $state_pid, }}); if (not exists $anvil->data->{pids}{$state_pid}) { # Reap the 'db_is_use'. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0140", variables => { pid => $state_pid }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0140", variables => { + db => $anvil->Get->host_name_from_uuid({host_uuid => $db_uuid})." (".$db_uuid.")", + pid => $state_pid, + }}); my $query = "DELETE FROM states WHERE state_uuid = ".$anvil->Database->quote($state_uuid).";"; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); @@ -1348,6 +1359,21 @@ sub handle_special_cases $anvil->DRBD->_initialize_kmod({debug => 2}); } + ### TODO: Remove these later. This is here to clean up how we used to handle db_in_use and lock_request flags. + if (1) + { + # Broadly clear all states that are '0' now. + my $queries = []; + push @{$queries}, "DELETE FROM states WHERE state_name LIKE 'db_in_use::%' AND state_note != '1';"; + push @{$queries}, "DELETE FROM history.variables WHERE variable_name = 'lock_request';"; + push @{$queries}, "DELETE FROM variables WHERE variable_name = 'lock_request';"; + foreach my $query (@{$queries}) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); + } + $anvil->Database->write({debug => 2, query => $queries, source => $THIS_FILE, line => __LINE__}); + } + return(0); } diff --git a/tools/anvil-update-states b/tools/anvil-update-states index 7c9365f8..7299aa38 100755 --- a/tools/anvil-update-states +++ b/tools/anvil-update-states @@ -861,7 +861,15 @@ AND if ((not exists $anvil->data->{seen}{bond}{$bond_name}) or (not $anvil->data->{seen}{bond}{$bond_name})) { # Mark it as deleted. - my $query = "UPDATE bonds SET bond_mode = 'DELETED' WHERE bond_uuid = ".$anvil->Database->quote($bond_uuid).";"; + my $query = " +UPDATE + bonds +SET + bond_mode = 'DELETED', + modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." +WHERE + bond_uuid = ".$anvil->Database->quote($bond_uuid)." +;"; $anvil->Database->write({debug => 3, query => $query, source => $THIS_FILE, line => __LINE__}); # Remove it from the hash so we don't add it to the .json and .xml files. @@ -918,7 +926,14 @@ AND if ((not exists $anvil->data->{seen}{bridge}{$bridge_name}) or (not $anvil->data->{seen}{bridge}{$bridge_name})) { # Mark it as deleted. - my $query = "UPDATE bridges SET bridge_id = 'DELETED' WHERE bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";"; + my $query = " +UPDATE + bridges +SET + bridge_id = 'DELETED', + modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." +WHERE + bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";"; $anvil->Database->write({debug => 3, query => $query, source => $THIS_FILE, line => __LINE__}); # Remove it from the hash so we don't add it to the .json and .xml files. @@ -997,8 +1012,16 @@ ORDER BY if ($anvil->data->{network_interfaces}{$network_interface_uuid}{network_interface_operational} ne "DELETED") { # Mark it as deleted. - my $query = "UPDATE network_interfaces SET network_interface_operational = 'DELETED' WHERE network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid).";"; - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); + my $query = " +UPDATE + network_interfaces +SET + network_interface_operational = 'DELETED', + modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." +WHERE + network_interface_uuid = ".$anvil->Database->quote($network_interface_uuid)." +;"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0124", variables => { query => $query }}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); } diff --git a/tools/fence_delay b/tools/fence_delay index 448f5605..6fc363d5 100755 --- a/tools/fence_delay +++ b/tools/fence_delay @@ -120,9 +120,6 @@ sub do_exit my ($conf, $log, $exit_status) = @_; $exit_status = 9 if not defined $exit_status; - # Close the log file handle, if it exists. - $log->close() if $log; - record($conf, $log, "system::do_chown: [".$conf->{'system'}{do_chown}."].\n", 2); if ($conf->{'system'}{do_chown}) { @@ -133,6 +130,9 @@ sub do_exit chown $uid, $gid, $conf->{'system'}{'log'}; } + # Close the log file handle, if it exists. + $log->close() if $log; + exit ($exit_status); } @@ -213,8 +213,8 @@ sub metadata `; - # Done, exit. - do_exit($conf, $log, 0); + # Done + return(0); } # This handles the actual actions. diff --git a/tools/striker-auto-initialize-all b/tools/striker-auto-initialize-all index 5eb32e8e..5c3c7c7f 100755 --- a/tools/striker-auto-initialize-all +++ b/tools/striker-auto-initialize-all @@ -618,7 +618,8 @@ sub initialize_machines if ($network !~ /^sn/) { push @{$machine_ips}, $ip_address; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ip_count => @{$machine_ips} }}); + my $ip_count = @{$machine_ips}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ip_count => $ip_count }}); } if (not $machine_host_uuid) diff --git a/tools/striker-manage-install-target b/tools/striker-manage-install-target index a6880169..304075a1 100755 --- a/tools/striker-manage-install-target +++ b/tools/striker-manage-install-target @@ -43,6 +43,7 @@ # --status - This checks to see if dhcpd is running or not, then exits (effectively checking if the # "Install Target" feature is running). # +# NOTE: This is currently broken. Re-enable when a solution to anvil-striker-extra is found. # # TODO: # - Support building the install target by mounting the ISO and checking /mnt/shared/incoming for needed @@ -186,6 +187,11 @@ if (not $configured) $anvil->nice_exit({exit_code => 9}); } +### TODO: Remove this when re-enabled. +print $anvil->Words->string({key => "message_0291"})."\n"; +update_progress($anvil, 100, "message_0291"); +$anvil->nice_exit({exit_code => 0}); + # Figure out what this machine is. load_os_info($anvil); diff --git a/tools/striker-prep-database b/tools/striker-prep-database index 68f48159..173604ec 100755 --- a/tools/striker-prep-database +++ b/tools/striker-prep-database @@ -311,7 +311,7 @@ if ($local_uuid) if ($create_user) { # Create the user - my $shell_call = $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{createuser}." --no-superuser --createdb --no-createrole $database_user\""; + my $shell_call = $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{createuser}." --no-superuser --createdb --no-createrole ".$database_user."\""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); my ($create_output, $return_code) = $anvil->System->call({shell_call => $shell_call, debug => 2, source => $THIS_FILE, line => __LINE__}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { diff --git a/tools/striker-purge-target b/tools/striker-purge-target index d1576603..db2d0a05 100755 --- a/tools/striker-purge-target +++ b/tools/striker-purge-target @@ -226,7 +226,7 @@ else } # List all database tables in reverse order with X_host_uuid tables -$anvil->Database->find_host_uuid_columns({search_column => "host_uuid", main_table => "hosts"}); +$anvil->Database->find_host_uuid_columns({debug => 3, search_column => "host_uuid", main_table => "hosts"}); # For each host foreach my $host_uuid (@{$anvil->data->{purge}{hosts}}) @@ -295,7 +295,13 @@ WHERE } # Commit. - $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); + my $problem = $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); + if ($problem) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0364"}); + $anvil->nice_exit({exit_code => 1}); + } $vacuum = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { vacuum => $vacuum }}); @@ -363,7 +369,13 @@ if ($anvil->data->{purge}{anvil_uuid}) } # Commit. - $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); + my $problem = $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); + if ($problem) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0364"}); + $anvil->nice_exit({exit_code => 1}); + } $vacuum = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { vacuum => $vacuum }});