* Fixed a bug in Database->disconnect() where the database idenitification number wasn't being removed, so connecting again triggered the duplicate DB connection check.

* Fixed a bug in Tools->_set_defaults where the order the tables were sync'ed it caused primary/foreign keys would trigger DB errors when resync'ing in some cases.
* Created Database->log_connections to make it easier to log which databases are actively in use and other data about the connections.
* Fixed bugs in striker-manage-peers that (partly because of the above bugs) failed to connect to new peers properly.

Signed-off-by: Digimer <digimer@alteeve.ca>

Anvil/Tools.pm

@@ -908,9 +908,9 @@ sub _set_defaults
 									"mail_servers", 
 									"variables",
 									"jobs",
-									"network_interfaces",
 									"bonds",
 									"bridges",
+									"network_interfaces",
 									"ip_addresses", 
 									"files", 
 									"file_locations", 

Anvil/Tools/Database.pm

@@ -849,6 +849,12 @@ sub connect
 	# This will be set to '1' if either DB needs to be initialized or if the last_updated differs on any node.
 	$anvil->data->{sys}{database}{resync_needed} = 0;
 	
+	# In case this is called when connections already exist, clear the identifiers.
+	if (exists $anvil->data->{sys}{database}{identifier})
+	{
+		delete $anvil->data->{sys}{database}{identifier};
+	}
+	
 	# Now setup or however-many connections
 	my $seen_connections       = [];
 	my $failed_connections     = [];
@@ -865,12 +871,18 @@ sub connect
 			next;
 		}
 		
+		# Make sure values are set.
+		$anvil->data->{database}{$uuid}{port}     = 5432                                if not         $anvil->data->{database}{$uuid}{port};
+		$anvil->data->{database}{$uuid}{name}     = $anvil->data->{sys}{database}{name} if not         $anvil->data->{database}{$uuid}{name};
+		$anvil->data->{database}{$uuid}{user}     = $anvil->data->{sys}{database}{user} if not         $anvil->data->{database}{$uuid}{user};
+		$anvil->data->{database}{$uuid}{password} = ""                                  if not defined $anvil->data->{database}{$uuid}{password}; 
+		
 		my $driver   = "DBI:Pg";
-		my $host     = $anvil->data->{database}{$uuid}{host}     ? $anvil->data->{database}{$uuid}{host}     : ""; # This should fail
+		my $host     = $anvil->data->{database}{$uuid}{host}; # This should fail
-		my $port     = $anvil->data->{database}{$uuid}{port}     ? $anvil->data->{database}{$uuid}{port}     : 5432;
+		my $port     = $anvil->data->{database}{$uuid}{port};
-		my $name     = $anvil->data->{database}{$uuid}{name}     ? $anvil->data->{database}{$uuid}{name}     : $anvil->data->{sys}{database}{name};
+		my $name     = $anvil->data->{database}{$uuid}{name};
-		my $user     = $anvil->data->{database}{$uuid}{user}     ? $anvil->data->{database}{$uuid}{user}     : $anvil->data->{sys}{database}{user};
+		my $user     = $anvil->data->{database}{$uuid}{user};
-		my $password = $anvil->data->{database}{$uuid}{password} ? $anvil->data->{database}{$uuid}{password} : "";
+		my $password = $anvil->data->{database}{$uuid}{password};
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			host     => $host,
 			port     => $port,
@@ -1093,6 +1105,7 @@ sub connect
 		elsif ($dbh =~ /^DBI::db=HASH/)
 		{
 			# Woot!
+			
 			$anvil->data->{sys}{database}{connections}++;
 			push @{$successful_connections}, $uuid;
 			$anvil->data->{cache}{database_handle}{$uuid} = $dbh;
@@ -1366,7 +1379,6 @@ sub connect
 	$anvil->Database->mark_active({debug => $debug, set => 1});
 	
 	# Sync the database, if needed.
-	#$anvil->Database->resync_databases({debug => $debug});
 	$anvil->Database->resync_databases({debug => $debug});
 	
 	# Add ourselves to the database, if needed.
@@ -1413,6 +1425,7 @@ sub disconnect
 	# Delete the stored DB-related values.
 	delete $anvil->data->{sys}{database}{timestamp};
 	delete $anvil->data->{sys}{database}{read_uuid};
+	delete $anvil->data->{sys}{database}{identifier};
 	$anvil->Database->read({set => "delete"});
 	
 	# Delete any database information (reconnects should re-read anvil.conf anyway).
@@ -2272,7 +2285,7 @@ AND
 				# This isn't a network we should know about (ie: it might be a stray 'virbrX'
 				# birdge), delete this IP.
 				my $query = "UPDATE ip_addresses SET ip_address_note = 'DELETED' WHERE ip_address_uuid = ".$anvil->Database->quote($ip_address_uuid).";";
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
 				$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
 				next;
 			}
@@ -9830,6 +9843,62 @@ sub locking
 }
 
 
+=head2 log_connections
+
+This method logs details about open connections to databases. It's generally only used as a debugging tool.
+
+This method takes no parameters.
+
+=cut
+sub log_connections
+{
+	my $self      = shift;
+	my $parameter = shift;
+	my $anvil     = $self->parent;
+	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->locking()" }});
+	
+	# Log how many connections there are.
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, secure => 0, key => "log_0132"});
+
+	# Reading from
+	my $read_uuid = $anvil->data->{sys}{database}{read_uuid};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { read_uuid => $read_uuid }});
+	foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}})
+	{
+		my $host = $anvil->data->{database}{$uuid}{host};
+		my $port = $anvil->data->{database}{$uuid}{port};
+		my $name = $anvil->data->{database}{$uuid}{name};
+		my $user = $anvil->data->{database}{$uuid}{user};
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0537", variables => { 
+			name => $name,
+			user => $user, 
+			host => $host,
+			port => $port,
+		}});
+		
+		my $query      = "SELECT system_identifier FROM pg_control_system();";
+		my $identifier = $anvil->Database->query({uuid => $uuid, debug => ($debug + 1), query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => ($debug + 1), key => "log_0540", variables => { 
+			uuid       => $uuid,
+			identifier => $identifier, 
+		}});
+		
+		if ($uuid eq $read_uuid)
+		{
+			# Reading from this DB
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0538"});
+		}
+		else
+		{
+			# Not reading from this DB
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0539"});
+		}
+	}
+	
+	return(0);
+}
+
 =head2 manage_anvil_conf
 
 This adds, removes or updates a database entry in a machine's anvil.conf file. This returns C<< 0 >> on success, C<< 1 >> if there was a problem.
@@ -10712,6 +10781,8 @@ sub resync_databases
 		return(0);
 	}
 	
+	#$anvil->data->{sys}{database}{log_transactions} = 1;
+	
 	# Archive old data before resync'ing
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0451"});
 	$anvil->Database->archive_database({debug => $debug});

Anvil/Tools/Network.pm

@@ -1236,6 +1236,7 @@ AND
 				results => $results, 
 				count   => $count, 
 			}});
+			next if not $count;
 			
 			$interface_name = $results->[0]->[0];
 			$interface_mac  = $results->[0]->[1];
@@ -1281,6 +1282,7 @@ AND
 				results => $results, 
 				count   => $count, 
 			}});
+			next if not $count;
 			
 			$interface_name = $results->[0]->[0];
 			$interface_mac  = $results->[0]->[1];
@@ -1326,6 +1328,7 @@ AND
 				results => $results, 
 				count   => $count, 
 			}});
+			next if not $count;
 			
 			$interface_name = $results->[0]->[0];
 			$interface_mac  = $results->[0]->[1];

notes

@@ -179,8 +179,7 @@ Drop;
 su - postgres -c "dropdb anvil" && su - postgres -c "createdb --owner admin anvil" && su - postgres -c "psql anvil"
 
 Reload the DB;
-su - postgres -c "dropdb anvil" && su - postgres -c "createdb --owner admin anvil" && su - postgres -c "psql anvil < /anvil.out"
+su - postgres -c "dropdb anvil" && su - postgres -c "createdb --owner admin anvil" && su - postgres -c "psql anvil < /anvil.out" && su - postgres -c "psql anvil"
-su - postgres -c "psql anvil"
 
 
 

rpm/SPECS/anvil.spec

@@ -3,7 +3,7 @@
 %define anvilgroup    admin
 Name:           anvil
 Version:        3.0
-Release:        34%{?dist}
+Release:        35%{?dist}
 Summary:        Alteeve Anvil! complete package.
 
 License:        GPLv2+
@@ -373,6 +373,9 @@ fi
 
 
 %changelog
+* tbd Madison Kelly <mkelly@alteeve.ca> 3.0-35
+- Updated source.
+
 * Fri Aug 28 2020 Madison Kelly <mkelly@alteeve.ca> 3.0-34
 - Added 'virt-top' as a requirement on nodes and dr hosts.
 - Added cyrus-sasl* as requirements to core.

share/words.xml

@@ -388,6 +388,7 @@ Failure! The return code: [#!variable!return_code!#] was received ('0' was expec
 		<key name="job_0128">Update not required, nothing changed.</key>
 		<key name="job_0129">Completed joining the #!string!brand_0002!#.</key>
 		<key name="job_0130">No job was found that needs to be run.</key>
+		<key name="job_0131">Reconnecting will start a synchonization of the database. This step might take a while to complete, please be patient.</key>
 		
 		<!-- Log entries -->
 		<key name="log_0001">Starting: [#!variable!program!#].</key>
@@ -1015,6 +1016,10 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
 		<key name="log_0534">Updating the configuration line from: [#!variable!old_line!#] to: [#!variable!new_line!#].</key>
 		<key name="log_0535">Starting and enabling the daemon: [#!variable!daemon!#].</key>
 		<key name="log_0536">Creating the Anvil! alert email spool directory: [#!data!path::directories::alert_emails!#].</key>
+		<key name="log_0537">Connected to the database named: [#!variable!name!#] as: [#!variable!user!#@#!variable!host!#:#!variable!port!#].</key>
+		<key name="log_0538">This IS the database queries are read from.</key>
+		<key name="log_0539">This is NOT the database queries are read from.</key>
+		<key name="log_0540">This host UUID is: [#!variable!uuid!#] and the database identifier is: [#!variable!identifier!#].</key>
 		
 		<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
 		<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>

tools/anvil-daemon

@@ -1013,7 +1013,7 @@ sub run_jobs
 			my $command = $job_command." --job-uuid ".$job_uuid;
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0210", variables => { command => $command }});
 			($anvil->data->{jobs}{handles}{$job_uuid}, my $return_code) = $anvil->System->call({
-				debug       => 2,
+				debug       => 3,
 				background  => 1, 
 				stdout_file => "/tmp/anvil.job.".$job_uuid.".stdout",
 				stderr_file => "/tmp/anvil.job.".$job_uuid.".stderr", 
@@ -1058,13 +1058,16 @@ sub update_state_file
 	
 	# Bunch of things happen here, so this allows a single variable to change for logging.
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0480"});
-	my $log_level = 3;
+	my $log_level = 2;
 	my $v         = "-";
 	for (1..$log_level)
 	{
 		$v .= "v";
 	}
-	my ($states_output, $return_code) = $anvil->System->call({debug => $log_level, shell_call => $anvil->data->{path}{exe}{'anvil-update-states'}." ".$v, source => $THIS_FILE, line => __LINE__});
+	my $shell_call = $anvil->data->{path}{exe}{'anvil-update-states'}." ".$v;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
+	
+	my ($states_output, $return_code) = $anvil->System->call({debug => $log_level, shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $log_level, list => { 
 		states_output => $states_output,
 		return_code   => $return_code, 

tools/striker-manage-peers

@@ -223,6 +223,7 @@ sub process_entry
 	if ((not $password) && ($job_uuid))
 	{
 		# Do we have a database connection?
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::database::connections" => $anvil->data->{sys}{database}{connections}}});
 		if ($anvil->data->{sys}{database}{connections})
 		{
 			# See if the password is in job_data.
@@ -337,7 +338,7 @@ sub process_entry
 			my $left_space  = $2;
 			my $right_space = $3; 
 			my $value       = $4;
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 				"s1:variable"    => $variable,
 				"s2:value"       => $value, 
 				"s3:left_space"  => $left_space, 
@@ -512,6 +513,7 @@ sub process_entry
 		
 		# Now update!
 		$anvil->Storage->write_file({
+			debug     => 3,
 			secure    => 1, 
 			file      => $anvil->data->{path}{configs}{'anvil.conf'}, 
 			body      => $new_body, 
@@ -521,13 +523,17 @@ sub process_entry
 			overwrite => 1,
 		});
 		update_progress($anvil, 80, "message_0073");
+		update_progress($anvil, 82, "job_0131");
 		
 		# Disconnect (if no connections exist, will still clear out known databases).
-		$anvil->Database->disconnect;
+		$anvil->Database->disconnect({debug => 2});
 		
 		# Re-read the config.
 		sleep 1;
-		$anvil->Storage->read_config({file => $anvil->data->{path}{configs}{'anvil.conf'}});
+		$anvil->Storage->read_config({debug => 3});
+		
+		# Reconnect
+		$anvil->Database->connect({debug => 3});
 	}
 	
 	# If I've been asked to have the peer add us, disconnect from the database, re-read the new config 
@@ -537,10 +543,6 @@ sub process_entry
 	{
 		update_progress($anvil, 85, "message_0074");
 		
-		# Reconnect
-		$anvil->Database->connect;
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
-		
 		# Now loop until we see the peer's host_uuid show up and we have a connection to the peer's
 		# database.
 		my $peer_connected = 0;

tools/test.pl

@@ -23,12 +23,12 @@ $| = 1;
 my $anvil = Anvil::Tools->new();
 $anvil->Log->level({set => 2});
 $anvil->Log->secure({set => 1});
+$anvil->Get->switches;
 
 print "Connecting to the database(s);\n";
-$anvil->Database->connect();
+$anvil->Database->connect({debug => 2});
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "log_0132"});
-$anvil->Get->switches;
+# 
-
+# my ($oldest_message) = $anvil->Email->check_queue({debug => 2});
-my ($oldest_message) = $anvil->Email->check_queue({debug => 2});
+# print "Oldest message: [".$oldest_message."]\n";
-print "Oldest message: [".$oldest_message."]\n";