From 25a0454dceef3d1edada8a9a05443709d17f84ae Mon Sep 17 00:00:00 2001 From: digimer Date: Wed, 29 May 2024 20:41:12 -0400 Subject: [PATCH] Better handling of lost DB connections. * Added a sync call to Tools->nice_exit() to ensure logs are flushed. * Updated Database->quote() to be in an eval block to better handle cases where the DB handle is lost. * Added an hourly check to anvil-daemon and moved the memory in use check to run only once per hour. Signed-off-by: digimer --- Anvil/Tools.pm | 4 ++++ Anvil/Tools/Database.pm | 20 +++++++++++++++++-- Anvil/Tools/Log.pm | 5 ++++- share/words.xml | 1 + tools/anvil-daemon | 44 ++++++++++++++++++++++++++++------------- 5 files changed, 57 insertions(+), 17 deletions(-) diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index eae5cd32..a40d2c3d 100644 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -428,6 +428,9 @@ sub nice_exit $anvil->data->{HANDLE}{'log'}{main} = ""; } + # Call a disk sync. + system($anvil->data->{path}{exe}{sync}); + #print "Exiting with RC: [".$exit_code."]\n"; exit($exit_code); } @@ -1320,6 +1323,7 @@ sub _set_paths su => "/usr/bin/su", 'subscription-manager' => "/usr/sbin/subscription-manager", swapon => "/usr/sbin/swapon", + sync => "/usr/bin/sync", sysctl => "/usr/sbin/sysctl", systemctl => "/usr/bin/systemctl", tail => "/usr/bin/tail", diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 02a65a6c..b9b55c94 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -17601,6 +17601,8 @@ sub query This quotes a string for safe use in database queries/writes. It operates exactly as C<< DBI >>'s C<< quote >> method. This method is simply a wrapper that uses the C<< DBI >> handle set as the currently active read database. +If there is a problem, an empty string will be returned and an error will be logged and printed to STDOUT. + Example; $anvil->Database->quote("foo"); @@ -17616,8 +17618,22 @@ sub quote my $string = shift; my $anvil = $self->parent; - $string = "" if not defined $string; - my $quoted = $anvil->Database->read->quote($string); + $string = "" if not defined $string; + + # Make sure we're using an active handle. + my $quoted = eval {$anvil->Database->read->quote($string); }; + if ($@) + { + $quoted = "" if not defined $quoted; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "warning_0177", variables => { + string => $string, + error => $@, + }}); + + # Given this might be about to get used in a DB query, return nothing. That should cause + # whatever query this was called for to error safely. + return(""); + } return($quoted); } diff --git a/Anvil/Tools/Log.pm b/Anvil/Tools/Log.pm index 11346e3c..fdd89271 100644 --- a/Anvil/Tools/Log.pm +++ b/Anvil/Tools/Log.pm @@ -544,7 +544,10 @@ sub entry # The handle has to be wrapped in a block to make 'print' happy as it doesn't like non-scalars for file handles print { $anvil->data->{HANDLE}{'log'}{alert} } $log_to_alert; - system('/usr/bin/sync'); + + ### NOTE: uncheck this is you have reason to think kernel buffering is preventing all + ### logs being flushed to disk. Obviously, this adds overhead. + #system('/usr/bin/sync'); } $anvil->data->{loop}{count} = 0; } diff --git a/share/words.xml b/share/words.xml index 6992b530..d50c4927 100644 --- a/share/words.xml +++ b/share/words.xml @@ -4201,6 +4201,7 @@ We will try to proceed anyway. [ Warning ] - The fence method: [#!variable!method!#] already existed, proceeding. [ Warning ] - The DB query: [#!variable!query!#] timed out! It was given: [#!variable!timeout!#] seconds, and alarmed with: [#!variable!error!#]. [ Warning ] - The DB query: [#!variable!query!#] failed with the error: [#!variable!error!#]. + [ Warning ] - SQL quoting string: [#!variable!string!#] failed with the error: [#!variable!error!#]. diff --git a/tools/anvil-daemon b/tools/anvil-daemon index 819ab82d..717c1830 100755 --- a/tools/anvil-daemon +++ b/tools/anvil-daemon @@ -173,21 +173,25 @@ my $delay = set_delay($anvil); # Once a day, we'll refresh an Install Target's RPM repository (has no effect on non-Striker dashboards). $anvil->data->{timing}{minute_checks} = 60; $anvil->data->{timing}{ten_minute_checks} = 600; +$anvil->data->{timing}{hourly_checks} = 3600; $anvil->data->{timing}{daily_checks} = 86400; $anvil->data->{timing}{repo_update_interval} = 86400; $anvil->data->{timing}{next_minute_check} = $now_time - 1; +$anvil->data->{timing}{next_hourly_check} = $now_time - 1; $anvil->data->{timing}{next_ten_minute_check} = $now_time - 1; $anvil->data->{timing}{next_daily_check} = ($now_time + $delay) - 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "s1:timing::minute_checks" => $anvil->data->{timing}{minute_checks}, - "s2:timing::ten_minute_checks" => $anvil->data->{timing}{ten_minute_checks}, - "s3:timing::daily_checks" => $anvil->data->{timing}{daily_checks}, - "s4:timing::repo_update_interval" => $anvil->data->{timing}{repo_update_interval}, - "s5:now_time" => $now_time, - "s6:delay" => $delay, - "s7:timing::next_minute_check" => $anvil->data->{timing}{next_minute_check}, - "s8:timing::next_ten_minute_check" => $anvil->data->{timing}{next_ten_minute_check}, - "s9:timing::next_daily_check" => $anvil->data->{timing}{next_daily_check}, + "s01:timing::minute_checks" => $anvil->data->{timing}{minute_checks}, + "s02:timing::ten_minute_checks" => $anvil->data->{timing}{ten_minute_checks}, + "s03:timing::hourly_checks" => $anvil->data->{timing}{hourly_checks}, + "s04:timing::daily_checks" => $anvil->data->{timing}{daily_checks}, + "s05:timing::repo_update_interval" => $anvil->data->{timing}{repo_update_interval}, + "s06:now_time" => $now_time, + "s07:delay" => $delay, + "s08:timing::next_minute_check" => $anvil->data->{timing}{next_minute_check}, + "s09:timing::next_ten_minute_check" => $anvil->data->{timing}{next_ten_minute_check}, + "s10:timing::next_hourly_check" => $anvil->data->{timing}{next_hourly_check}, + "s11:timing::next_daily_check" => $anvil->data->{timing}{next_daily_check}, }}); # Disconnect. We'll reconnect inside the loop @@ -231,9 +235,6 @@ while(1) $anvil->nice_exit({exit_code => 0}); } - # Check how much RAM we're using. - check_ram($anvil); - # Disconnect from the database(s) and sleep now. $anvil->Database->disconnect(); sleep(2); @@ -443,8 +444,9 @@ sub handle_periodic_tasks "s1:now_time" => $now_time, "s2:timing::next_minute_check" => $anvil->data->{timing}{next_minute_check}, "s3:timing::next_ten_minute_check" => $anvil->data->{timing}{next_ten_minute_check}, - "s4:timing::next_daily_check" => $anvil->data->{timing}{next_daily_check}, - "s5:host_type" => $host_type, + "s4:timing::next_hourly_check" => $anvil->data->{timing}{next_hourly_check}, + "s5:timing::next_daily_check" => $anvil->data->{timing}{next_daily_check}, + "s6:host_type" => $host_type, }}); # Time to run once per minute tasks. @@ -618,6 +620,20 @@ sub handle_periodic_tasks }}); } + # Now check to see if it's time to run hourly tasks. + if ($now_time >= $anvil->data->{timing}{next_hourly_check}) + { + # Check how much RAM we're using. + check_ram($anvil); + + # Update the next check time. + $anvil->data->{timing}{next_hourly_check} = $now_time + $anvil->data->{timing}{hourly_checks}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "s1:timing::hourly_checks" => $anvil->data->{timing}{hourly_checks}, + "s2:timing::next_hourly_check" => $anvil->data->{timing}{next_hourly_check}, + }}); + } + # Now check to see if it's time to run daily tasks. if ($now_time >= $anvil->data->{timing}{next_daily_check}) {