Reworked DB resync logic.

With this new system, a 'primary_db' is chosen (first connected DB UUID when sorted) and only it does resyncs. Further, resyncs have been pulled from all tools except anvil-daemon. So with this new system, the chances of duplicate, simultaneous resyncs should be removed (hopefully for real this time).

* Database->check_agent_data() no longer calls a resync after loading a
  schema.
* Removed the Database->coonnect() 'all' parameter
* The database used to read from is now always the same as the primary,
  even if there is a local DB.
* Database->connect() 'check_for_resync' parameter can now be set to
  '2', which means "check for resync _if_ I am primary", where '1' still
  checks for resync no matter what.

Signed-off-by: digimer <mkelly@alteeve.ca>

Anvil/Tools/Database.pm

@@ -620,25 +620,26 @@ sub check_agent_data
 				}});
 			}
 			
-			# Now check to see if a resync is required, it likely is.
+			### NOTE: Don't sync here, leave it for anvil-daemon to handle.
-			if ($anvil->data->{sys}{database}{connections} > 1)
+# 			# Now check to see if a resync is required, it likely is.
-			{
+# 			if ($anvil->data->{sys}{database}{connections} > 1)
-				# The source is the agent
+# 			{
-				$anvil->Database->_find_behind_databases({
+# 				# The source is the agent
-					debug  => $debug, 
+# 				$anvil->Database->_find_behind_databases({
-					source => $agent, 
+# 					debug  => $debug, 
-					tables => $tables, 
+# 					source => $agent, 
-				});
+# 					tables => $tables, 
-			}
+# 				});
-			
+# 			}
-			# Hold if a lock has been requested.
+# 			
-			$anvil->Database->locking({debug => $debug});
+# 			# Hold if a lock has been requested.
-			
+# 			$anvil->Database->locking({debug => $debug});
-			# Mark that we're now active.
+# 			
-			$anvil->Database->mark_active({debug => $debug, set => 1});
+# 			# Mark that we're now active.
-			
+# 			$anvil->Database->mark_active({debug => $debug, set => 1});
-			# Sync the database, if needed.
+# 			
-			$anvil->Database->resync_databases({debug => $debug});
+# 			# Sync the database, if needed.
+# 			$anvil->Database->resync_databases({debug => $debug, force });
 		}
 	}
 	
@@ -1199,10 +1200,6 @@ This module will return the number of databases that were successfully connected
 
 Parameters;
 
-=head3 all (optional, default 0)
-
-If set, then the checks to see if a database is active or not is ignored. If the postgresql server is running, we will connect to it.
-
 =head3 check_for_resync (optional, default 0)
 
 If set to C<< 1 >>, and there are 2 or more databases available, a check will be make to see if the databases need to be resync'ed or not. This is also set if the command line switch C<< --resync-db >> is used.
@@ -1287,7 +1284,6 @@ sub connect
 	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->connect()" }});
 	
-	my $all                 = defined $parameter->{all}                 ? $parameter->{all}                 : 0;
 	my $check_if_configured = defined $parameter->{check_if_configured} ? $parameter->{check_if_configured} : 0;
 	my $db_uuid             = defined $parameter->{db_uuid}             ? $parameter->{db_uuid}             : "";
 	my $check_for_resync    = defined $parameter->{check_for_resync}    ? $parameter->{check_for_resync}    : 0;
@@ -1299,7 +1295,6 @@ sub connect
 	my $tables              = defined $parameter->{tables}              ? $parameter->{tables}              : "";
 	my $test_table          = defined $parameter->{test_table}          ? $parameter->{test_table}          : $anvil->data->{sys}{database}{test_table};
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-		all                 => $all,
 		check_if_configured => $check_if_configured, 
 		db_uuid             => $db_uuid,
 		check_for_resync    => $check_for_resync, 
@@ -1619,24 +1614,17 @@ sub connect
 				uuid => $uuid,
 			}});
 
-			# Set this database handle as the one to use for reading, if no handle is yet set.
+			# Only the first database to connect will be "Active". This is the database used for
-			if (($is_local) or (not $anvil->data->{sys}{database}{read_uuid}) or (not $anvil->Database->read))
+			# reads and the DB that will deal with resyncs
+			if (not $anvil->data->{sys}{database}{primary_db})
 			{
+				$anvil->data->{sys}{database}{primary_db} = $uuid;
 				$anvil->data->{sys}{database}{read_uuid}  = $uuid;
 				$anvil->Database->read({set => $dbh});
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
-					'anvil->Database->read'    => $anvil->Database->read,
+					"sys::database::primary_db" => $anvil->data->{sys}{database}{primary_db},
 					"sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid},
-				}});
+					'anvil->Database->read'    => $anvil->Database->read,
-			}
-
-			# Only the first database to connect will be "Active". What this means will expand
-			# over time. As of now, only the active DB will do resyncs.
-			if (not $anvil->data->{sys}{database}{active_uuid})
-			{
-				$anvil->data->{sys}{database}{active_uuid} = $uuid;
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
-					"sys::database::active_uuid" => $anvil->data->{sys}{database}{active_uuid},
 				}});
 			}
 
@@ -1695,13 +1683,10 @@ sub connect
 			}
 			
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-				"sys::database::read_uuid"        => $anvil->data->{sys}{database}{read_uuid}, 
 				"cache::database_handle::${uuid}" => $anvil->data->{cache}{database_handle}{$uuid}, 
 			}});
 			
-			# Before I continue, see if this database is inactive (unless 'all' is set).
+			# Before I continue, see if this database is inactive.
-			if (not $all)
-			{
 			my ($active_value, undef, undef) = $anvil->Database->read_variable({
 				debug         => $debug,
 				uuid          => $uuid,
@@ -1741,10 +1726,18 @@ sub connect
 					if ($anvil->data->{sys}{database}{read_uuid} eq $uuid)
 					{
 						$anvil->data->{sys}{database}{read_uuid} = "";
-							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid} }});
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+							"sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid},
+						}});
 					}
-						next;
+					if ($anvil->data->{sys}{database}{primary_db} eq $uuid)
+					{
+						$anvil->data->{sys}{database}{primary_db} = "";
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+							"sys::database::primary_db" => $anvil->data->{sys}{database}{primary_db},
+						}});
 					}
+					next;
 				}
 			}
 			
@@ -1765,10 +1758,7 @@ sub connect
 			{
 				$anvil->Database->refresh_timestamp({debug => $debug});
 			}
-			
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-				"sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid},
-				'anvil->Database->read'    => $anvil->Database->read,
 				"sys::database::timestamp" => $anvil->data->{sys}{database}{timestamp},
 			}});
 			
@@ -1779,10 +1769,6 @@ sub connect
 		
 		# Before we try to connect, see if this is a local database and, if so, make sure it's setup.
 		if ($is_local)
-		{
-			# If we're being called with 'all', don't set active as we could be just checking if 
-			# we're active or not.
-			if (not $all)
 		{
 			# If we're a striker, set the variable saying we're active if we need to.
 			my ($active_value, undef, undef) = $anvil->Database->read_variable({
@@ -1805,7 +1791,6 @@ sub connect
 				});
 			}
 		}
-		}
 		# If this isn't a local database, read the target's Anvil! version (if available) and make 
 		# sure it matches ours. If it doesn't, skip this database.
 		else
@@ -1836,7 +1821,7 @@ sub connect
 				
 				# Delete the information about this database. We'll try again on next
 				# ->connect().
-				$anvil->data->{sys}{database}{active_uuid} = "" if $anvil->data->{sys}{database}{read_active} eq $uuid;
+				$anvil->data->{sys}{database}{primary_db} = "" if $anvil->data->{sys}{database}{read_active} eq $uuid;
 				$anvil->data->{sys}{database}{read_uuid}  = "" if $anvil->data->{sys}{database}{read_uuid}   eq $uuid;
 				$anvil->data->{sys}{database}{connections}--;
 				delete $anvil->data->{database}{$uuid};
@@ -2013,9 +1998,6 @@ sub connect
 		}
 	}
 	
-	my $total = tv_interval ($start_time, [gettimeofday]);
-	#print "Total runtime: [".$total."]\n";
-	
 	# Do I have any connections? Don't die, if not, just return.
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::connections" => $anvil->data->{sys}{database}{connections} }});
 	if (not $anvil->data->{sys}{database}{connections})
@@ -2151,46 +2133,34 @@ sub connect
 		return($anvil->data->{sys}{database}{connections});
 	}
 	
-	if (exists $anvil->data->{'log'}{scan_agent})
+	# If 'check_for_resync' is set to '2', then only check if we're primary.
-	{
-		my $agent = $anvil->data->{'log'}{scan_agent};
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { agent => $agent }});
-		if (exists $anvil->data->{scan_agent}{$agent}{last_db_count})
-		{
-			$anvil->data->{sys}{database}{last_db_count} = $anvil->data->{scan_agent}{$agent}{last_db_count};
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
-				"sys::database::last_db_count" => $anvil->data->{sys}{database}{last_db_count}, 
+		"sys::database::primary_db" => $anvil->data->{sys}{database}{primary_db},
+		"sys::host_uuid"            => $anvil->data->{sys}{host_uuid},
+		check_for_resync            => $check_for_resync, 
 	}});
-		}
+	if ($check_for_resync == 2)
-	}
-	
-	# If we have a previous count and the new count is higher, resync.
-	if (not exists $anvil->data->{sys}{database}{last_db_count})
 	{
-		$anvil->data->{sys}{database}{last_db_count} = 0;
+		if ($anvil->data->{sys}{database}{primary_db} eq $anvil->data->{sys}{host_uuid})
-	}
-	
-	# If "last_db_count" is the lower than the current number of connections, check for a resync. 
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-		"sys::database::last_db_count" => $anvil->data->{sys}{database}{last_db_count},
-		"sys::database::connections"   => $anvil->data->{sys}{database}{connections}, 
-	}});
-	if ($anvil->data->{sys}{database}{connections} > $anvil->data->{sys}{database}{last_db_count})
 		{
+			# We're primary.
 			$check_for_resync = 1;
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { check_for_resync => $check_for_resync }});
 		}
+		else
+		{
+			# We're not primary
+			$check_for_resync = 0;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { check_for_resync => $check_for_resync }});
+		}
+	}
 	
 	# Check for behind databases only if there are 2+ DBs, we're the active DB, and we're set to do so.
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
 		"sys::database::connections" => $anvil->data->{sys}{database}{connections},
-		"sys::database::active_uuid" => $anvil->data->{sys}{database}{active_uuid},
-		"sys::host_uuid"             => $anvil->data->{sys}{host_uuid},
 		check_for_resync             => $check_for_resync, 
 	}});
-	if (($anvil->data->{sys}{database}{connections} > 1)                                &&
+	if (($anvil->data->{sys}{database}{connections} > 1) && ($check_for_resync))
-	    (($anvil->data->{sys}{database}{active_uuid} eq $anvil->data->{sys}{host_uuid}) or
-	    ($check_for_resync)))
 	{
 		$anvil->Database->_find_behind_databases({
 			debug  => $debug, 

scancore-agents/scan-apc-pdu/scan-apc-pdu

@@ -186,10 +186,10 @@ if ($anvil->data->{switches}{purge})
 my $host_uuid = $anvil->Get->host_uuid();
 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
 	host_uuid                   => $host_uuid,
-	"sys::database::active_uuid" => $anvil->data->{sys}{database}{active_uuid},
+	"sys::database::primary_db" => $anvil->data->{sys}{database}{primary_db},
 	"switches::force"           => $anvil->data->{switches}{force},
 }});
-if ((not $anvil->data->{switches}{force}) && ($anvil->data->{sys}{database}{active_uuid} ne $host_uuid))
+if ((not $anvil->data->{switches}{force}) && ($anvil->data->{sys}{database}{primary_db} ne $host_uuid))
 {
 	# Don't run.
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "scan_apc_pdu_message_0041", variables => { program => $THIS_FILE }});

tools/anvil-daemon

@@ -74,7 +74,7 @@ $anvil->System->wait_on_dnf();
 # is to setup the database server.
 $anvil->Database->connect({
 	check_if_configured => 1, 
-	check_for_resync    => 1,
+	check_for_resync    => 2,
 });
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0132"});
 
@@ -89,7 +89,7 @@ if (not $anvil->data->{sys}{database}{connections})
 		prep_database($anvil);
 		
 		# Try connecting again
-		$anvil->Database->connect({check_if_configured => 1, check_for_resync => 1});
+		$anvil->Database->connect({check_if_configured => 1, check_for_resync => 2});
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0132"});
 		if (not $anvil->data->{sys}{database}{connections})
 		{
@@ -109,7 +109,7 @@ if (not $anvil->data->{sys}{database}{connections})
 			
 			check_network($anvil);
 			$anvil->refresh();
-			$anvil->Database->connect({check_if_configured => 1, check_for_resync => 1});
+			$anvil->Database->connect({check_if_configured => 1, check_for_resync => 2});
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0132"});
 			if (not $anvil->data->{sys}{database}{connections})
 			{
@@ -203,7 +203,7 @@ while(1)
 	# If, so some reason, anvil.conf is lost, create it.
 	$anvil->System->_check_anvil_conf();
 	
-	$anvil->Database->connect({check_if_configured => $check_if_database_is_configured, check_for_resync => 1});
+	$anvil->Database->connect({check_if_configured => $check_if_database_is_configured, check_for_resync => 2});
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0132"});
 	
 	# Mark that we don't want to check the database now.

tools/anvil-manage-host

@@ -39,7 +39,7 @@ $anvil->Get->switches({list => [
 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 
-$anvil->Database->connect({all => 1});
+$anvil->Database->connect();
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
 if (not $anvil->data->{sys}{database}{connections})
 {

tools/anvil-manage-keys

@@ -41,7 +41,7 @@ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list
 	'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'},
 }});
 
-$anvil->Database->connect({check_for_resync => 1});
+$anvil->Database->connect();
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
 if (not $anvil->data->{sys}{database}{connections})
 {

tools/scancore

@@ -282,7 +282,7 @@ sub prepare_for_run
 	$anvil->Storage->read_config();
 	$anvil->Get->switches();
 	$anvil->Words->read();
-	$anvil->Database->connect({check_for_resync => 1});
+	$anvil->Database->connect();
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0132"});
 	
 	# See if the mail server needs to be updated.
@@ -296,9 +296,7 @@ sub wait_for_database
 {
 	my ($anvil) = @_;
 	
-	# Don't check for resync here as we may need to load agent schemas. We'll check for resync in the 
+	$anvil->Database->connect();
-	# main loop.
-	$anvil->Database->connect({check_for_resync => 0});
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"});
 	if (not $anvil->data->{sys}{database}{connections})
 	{
@@ -314,7 +312,7 @@ sub wait_for_database
 			$anvil->_set_paths();
 			$anvil->_set_defaults();
 			$anvil->Storage->read_config();
-			$anvil->Database->connect({check_for_resync => 0});
+			$anvil->Database->connect();
 			if ($anvil->data->{sys}{database}{connections})
 			{
 				# We're good

tools/striker-db-report

@@ -22,7 +22,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 
 my $anvil = Anvil::Tools->new();
 
-$anvil->Database->connect({debug => 3, check_for_resync => 0});
+$anvil->Database->connect();
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
 if (not $anvil->data->{sys}{database}{connections})
 {

tools/striker-manage-install-target

@@ -124,7 +124,7 @@ if ($anvil->data->{switches}{status})
 }
 
 # Connect to the database(s).
-$anvil->Database->connect({check_for_resync => 1});
+$anvil->Database->connect();
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
 if (not $anvil->data->{sys}{database}{connections})
 {

tools/striker-manage-peers

@@ -60,10 +60,7 @@ if (($< != 0) && ($> != 0))
 }
 
 # We'll try to connect in case we're adding additional peers.
-$anvil->Database->connect({
+$anvil->Database->connect();
-	debug            => 3,
-	check_for_resync => 1,
-});
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
 
 # Am I adding, editing or deleting?

tools/striker-scan-network

@@ -40,7 +40,7 @@ if (($< != 0) && ($> != 0))
 	$anvil->nice_exit({exit_code => 5});
 }
 
-$anvil->Database->connect({check_for_resync => 1});
+$anvil->Database->connect();
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
 if (not $anvil->data->{sys}{database}{connections})
 {