From 2a3f0bab2429a37b41f3a72e3bbffad1281a4bf6 Mon Sep 17 00:00:00 2001 From: digimer Date: Sat, 21 Oct 2023 13:33:14 -0400 Subject: [PATCH] Reworked how and when duplicate variables are checked/cleared. Moved the logic to a new private method, and call it now from the active Striker in the once per minute loop. The duplicate variable issue seems to be not entirely uncommon. Signed-off-by: digimer --- Anvil/Tools/Database.pm | 161 ++++++++++++++++++++++------------------ tools/anvil-daemon | 6 ++ 2 files changed, 94 insertions(+), 73 deletions(-) diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 1ea2df19..1db12b3a 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -105,6 +105,7 @@ my $THIS_FILE = "Database.pm"; # _add_to_local_config # _age_out_data # _archive_table +# _check_for_duplicates # _find_column # _find_behind_database # _mark_database_as_behind @@ -18150,79 +18151,6 @@ sub resync_databases # We're done with the table data, clear it. delete $anvil->data->{sys}{database}{table}; - # Look for duplicate entries in variables. This is done here as it's too generic to tag elsewhere - if (1) - { - my $query = " -SELECT - variable_uuid, - variable_section, - variable_name, - variable_source_table, - variable_source_uuid, - variable_value, - modified_date -FROM - variables -ORDER BY - modified_date DESC; -;"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - - my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); - my $count = @{$results}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - results => $results, - count => $count, - }}); - foreach my $row (@{$results}) - { - my $variable_uuid = $row->[0]; - my $variable_section = $row->[1]; - my $variable_name = $row->[2]; - my $variable_source_table = $row->[3]; - my $variable_source_uuid = $row->[4] // "none"; - my $variable_value = $row->[5]; - my $modified_date = $row->[6]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - variable_uuid => $variable_uuid, - variable_section => $variable_section, - variable_name => $variable_name, - variable_source_table => $variable_source_table, - variable_source_uuid => $variable_source_uuid, - variable_value => $variable_value, - modified_date => $modified_date, - }}); - - if (not exists $anvil->data->{duplicate_variables}{$variable_section}{$variable_name}{$variable_source_table}{$variable_source_uuid}) - { - # Save it. - $anvil->data->{duplicate_variables}{$variable_section}{$variable_name}{$variable_source_table}{$variable_source_uuid}{variable_value} = $variable_value; - $anvil->data->{duplicate_variables}{$variable_section}{$variable_name}{$variable_source_table}{$variable_source_uuid}{variable_uuid} = $variable_uuid; - } - else - { - # Duplicate! This is older, so delete it. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0165", variables => { - section => $variable_section, - name => $variable_name, - source_table => $variable_source_table, - source_uuid => $variable_source_uuid, - value => $variable_value, - }}); - - my $queries = []; - push @{$queries}, "DELETE FROM history.variables WHERE variable_uuid = ".$anvil->Database->quote($variable_uuid).";"; - push @{$queries}, "DELETE FROM variables WHERE variable_uuid = ".$anvil->Database->quote($variable_uuid).";"; - foreach my $query (@{$queries}) - { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - } - $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); - } - } - } - # Clear the variable that indicates we need a resync. $anvil->data->{sys}{database}{resync_needed} = 0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'sys::database::resync_needed' => $anvil->data->{sys}{database}{resync_needed} }}); @@ -19722,6 +19650,93 @@ COPY history.".$table." ("; } +=head2 _check_for_duplicates + +This method looks for duplicate entries in the database and clears them, if found. + +This method takes no parameters + +=cut +sub _check_for_duplicates +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->_check_for_duplicates()" }}); + + my $query = " +SELECT + variable_uuid, + variable_section, + variable_name, + variable_source_table, + variable_source_uuid, + variable_value, + modified_date +FROM + variables +ORDER BY + modified_date DESC; +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $variable_uuid = $row->[0]; + my $variable_section = $row->[1]; + my $variable_name = $row->[2]; + my $variable_source_table = $row->[3] // "none"; + my $variable_source_uuid = $row->[4] // "none"; + my $variable_value = $row->[5]; + my $modified_date = $row->[6]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + variable_uuid => $variable_uuid, + variable_section => $variable_section, + variable_name => $variable_name, + variable_source_table => $variable_source_table, + variable_source_uuid => $variable_source_uuid, + variable_value => $variable_value, + modified_date => $modified_date, + }}); + + if (not exists $anvil->data->{duplicate_variables}{$variable_section}{$variable_name}{$variable_source_table}{$variable_source_uuid}) + { + # Save it. + $anvil->data->{duplicate_variables}{$variable_section}{$variable_name}{$variable_source_table}{$variable_source_uuid}{variable_value} = $variable_value; + $anvil->data->{duplicate_variables}{$variable_section}{$variable_name}{$variable_source_table}{$variable_source_uuid}{variable_uuid} = $variable_uuid; + } + else + { + # Duplicate! This is older, so delete it. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0165", variables => { + section => $variable_section, + name => $variable_name, + source_table => $variable_source_table, + source_uuid => $variable_source_uuid, + value => $variable_value, + }}); + + my $queries = []; + push @{$queries}, "DELETE FROM history.variables WHERE variable_uuid = ".$anvil->Database->quote($variable_uuid).";"; + push @{$queries}, "DELETE FROM variables WHERE variable_uuid = ".$anvil->Database->quote($variable_uuid).";"; + foreach my $query (@{$queries}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + } + $anvil->Database->write({query => $queries, source => $THIS_FILE, line => __LINE__}); + } + } + + return(0); +} + =head2 _find_column This takes a table name and looks for a column that ends in C<< _host_uuid >> and, if found, stores it in the C<< sys::database::uuid_tables >> array. diff --git a/tools/anvil-daemon b/tools/anvil-daemon index e2963538..ef59e072 100755 --- a/tools/anvil-daemon +++ b/tools/anvil-daemon @@ -593,6 +593,12 @@ sub handle_periodic_tasks output => $output, return_code => $return_code, }}); + + # Look for duplicates if we're the primary DB. + if ($anvil->Get->host_uuid eq $anvil->data->{sys}{database}{primary_db}) + { + $anvil->Database->_check_for_duplicates({debug => 2}); + } } }