* This adds the new 'file_locations' -> 'file_location_ready' column and associated methods. This is set to TRUE/1 when the file referenced is found on disk and it is the expected size and md5sum. This is meant to allow programs to wait/watch or a file to be ready if they need to use it. Files are now checked periodically via anvil-daemon.

* Removed hard-coded log levels in anvil-provision-server and anvil-manage-storage-groups.

Signed-off-by: digimer <mkelly@alteeve.ca>

Anvil/Tools/Database.pm

@@ -429,6 +429,7 @@ sub check_file_locations
 					file_location_file_uuid => $file_uuid, 
 					file_location_host_uuid => $host_uuid, 
 					file_location_active    => 1, 
+					file_location_ready     => "same",
 				});
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
 			}
@@ -3487,6 +3488,7 @@ This loads the known install file_locations into the C<< anvil::data >> hash at:
 * file_locations::file_location_uuid::<file_location_uuid>::file_location_file_uuid
 * file_locations::file_location_uuid::<file_location_uuid>::file_location_host_uuid
 * file_locations::file_location_uuid::<file_location_uuid>::file_location_active
+* file_locations::file_location_uuid::<file_location_uuid>::file_location_ready
 * file_locations::file_location_uuid::<file_location_uuid>::modified_date
 
 If the hash was already populated, it is cleared before repopulating to ensure no stale data remains. 
@@ -3513,6 +3515,7 @@ SELECT
     file_location_file_uuid, 
     file_location_host_uuid, 
     file_location_active, 
+    file_location_ready, 
     modified_date 
 FROM 
     file_locations 
@@ -3530,12 +3533,14 @@ FROM
 		my $file_location_file_uuid = $row->[1];
 		my $file_location_host_uuid = $row->[2];
 		my $file_location_active    = $row->[3]; 
-		my $modified_date           = $row->[4];
+		my $file_location_ready     = $row->[4]; 
+		my $modified_date           = $row->[5];
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			file_location_uuid      => $file_location_uuid, 
 			file_location_file_uuid => $file_location_file_uuid, 
 			file_location_host_uuid => $file_location_host_uuid, 
 			file_location_active    => $file_location_active, 
+			file_location_ready     => $file_location_ready, 
 			modified_date           => $modified_date, 
 		}});
 		
@@ -3543,11 +3548,13 @@ FROM
 		$anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid} = $file_location_file_uuid;
 		$anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_host_uuid} = $file_location_host_uuid;
 		$anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}    = $file_location_active;
+		$anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_ready}     = $file_location_ready;
 		$anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{modified_date}           = $modified_date;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			"file_locations::file_location_uuid::${file_location_uuid}::file_location_file_uuid" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid}, 
 			"file_locations::file_location_uuid::${file_location_uuid}::file_location_host_uuid" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_host_uuid}, 
 			"file_locations::file_location_uuid::${file_location_uuid}::file_location_active"    => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}, 
+			"file_locations::file_location_uuid::${file_location_uuid}::file_location_ready"     => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_ready}, 
 			"file_locations::file_location_uuid::${file_location_uuid}::modified_date"           => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{modified_date}, 
 		}});
 		
@@ -8490,6 +8497,14 @@ This is set to C<< 1 >> or C<< 0 >>, and indicates if the file should be on the
 
 When set to C<< 1 >>, the file will be copied by the Anvil! member machines (by the member machines, they pull the files using rsync). If set to C<< 0 >>, the file is marked as inactive. If the file exists on the Anvil! members, it will be deleted.
 
+=head3 file_location_ready (optional, default '0')
+
+This is set to C<< 1 >> or C<< 0 >>, and indicates if the file is on the system and ready to be used. 
+
+B<< Note >>: This can also be set to C<< same >>. If set, and the file exists in the database, the existing value is retained. If the entry is inserted, this is set to C<< 0 >>.
+
+When set to C<< 1 >>, the file's size and md5sum have been confirmed to match on disk what is recorded in the database. When set to C<< 0 >>, the file _may_ be ready, but it probably isn't yet. Any process needing the file should check that it's ready before using it.
+
 =cut
 sub insert_or_update_file_locations
 {
@@ -8506,7 +8521,8 @@ sub insert_or_update_file_locations
 	my $file_location_anvil_uuid = defined $parameter->{file_location_anvil_uuid} ? $parameter->{file_location_anvil_uuid} : "";
 	my $file_location_file_uuid  = defined $parameter->{file_location_file_uuid}  ? $parameter->{file_location_file_uuid}  : "";
 	my $file_location_host_uuid  = defined $parameter->{file_location_host_uuid}  ? $parameter->{file_location_host_uuid}  : "";
-	my $file_location_active     = defined $parameter->{file_location_active}     ? $parameter->{file_location_active}     : "";
+	my $file_location_active     = defined $parameter->{file_location_active}     ? $parameter->{file_location_active}     : 0;
+	my $file_location_ready      = defined $parameter->{file_location_ready}      ? $parameter->{file_location_ready}      : "";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		uuid                     => $uuid, 
 		file                     => $file,
@@ -8516,6 +8532,7 @@ sub insert_or_update_file_locations
 		file_location_file_uuid  => $file_location_file_uuid, 
 		file_location_host_uuid  => $file_location_host_uuid, 
 		file_location_active     => $file_location_active, 
+		file_location_ready      => $file_location_ready, 
 	}});
 	
 	if (not $file_location_file_uuid)
@@ -8536,6 +8553,12 @@ sub insert_or_update_file_locations
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_file_locations()", parameter => "file_location_active" }});
 		return("");
 	}
+	if (($file_location_ready ne "0") && ($file_location_ready ne "1") && ($file_location_ready ne "same"))
+	{
+		# Throw an error and exit.
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_file_locations()", parameter => "file_location_ready" }});
+		return("");
+	}
 	
 	# If we've got an Anvil! uuid, find out the hosts and DR links connected to the Anvil! are found and 
 	# this method is recursively called for each host.
@@ -8577,6 +8600,7 @@ sub insert_or_update_file_locations
 				file_location_file_uuid  => $file_location_file_uuid, 
 				file_location_host_uuid  => $host_uuid, 
 				file_location_active     => $file_location_active, 
+				file_location_ready      => $file_location_ready, 
 			});
 			$file_location_uuids .= $host_uuid."=".$file_location_uuid.",";
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
@@ -8626,6 +8650,12 @@ AND
 		$file_location_uuid = $anvil->Get->uuid();
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
 		
+		if ($file_location_ready eq "same")
+		{
+			$file_location_ready = 0;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_ready => $file_location_ready }});
+		}
+		
 		my $query = "
 INSERT INTO 
     file_locations 
@@ -8634,12 +8664,14 @@ INSERT INTO
     file_location_file_uuid, 
     file_location_host_uuid, 
     file_location_active, 
+    file_location_ready, 
     modified_date 
 ) VALUES (
     ".$anvil->Database->quote($file_location_uuid).", 
     ".$anvil->Database->quote($file_location_file_uuid).", 
     ".$anvil->Database->quote($file_location_host_uuid).", 
     ".$anvil->Database->quote($file_location_active).", 
+    ".$anvil->Database->quote($file_location_ready).", 
     ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
 );
 ";
@@ -8653,7 +8685,8 @@ INSERT INTO
 SELECT 
     file_location_file_uuid, 
     file_location_host_uuid, 
-    file_location_active 
+    file_location_active, 
+    file_location_ready 
 FROM 
     file_locations 
 WHERE 
@@ -8678,16 +8711,25 @@ WHERE
 			my $old_file_location_file_uuid = $row->[0];
 			my $old_file_location_host_uuid = $row->[1];
 			my $old_file_location_active    = $row->[2];
+			my $old_file_location_ready     = $row->[3];
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 				old_file_location_file_uuid => $old_file_location_file_uuid, 
 				old_file_location_host_uuid => $old_file_location_host_uuid, 
 				old_file_location_active    => $old_file_location_active, 
+				old_file_location_ready     => $old_file_location_ready, 
 			}});
 			
+			if ($file_location_ready eq "same")
+			{
+				$file_location_ready = $old_file_location_ready;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_ready => $file_location_ready }});
+			}
+			
 			# Anything change?
 			if (($old_file_location_file_uuid ne $file_location_file_uuid) or 
 			    ($old_file_location_host_uuid ne $file_location_host_uuid) or 
-			    ($old_file_location_active    ne $file_location_active))
+			    ($old_file_location_active    ne $file_location_active)    or
+			    ($old_file_location_ready     ne $file_location_ready))
 			{
 				# Something changed, save.
 				my $query = "
@@ -8697,6 +8739,7 @@ SET
     file_location_file_uuid = ".$anvil->Database->quote($file_location_file_uuid).", 
     file_location_host_uuid = ".$anvil->Database->quote($file_location_host_uuid).", 
     file_location_active    = ".$anvil->Database->quote($file_location_active).", 
+    file_location_ready     = ".$anvil->Database->quote($file_location_ready).", 
     modified_date           = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." 
 WHERE 
     file_location_uuid      = ".$anvil->Database->quote($file_location_uuid)." 
@@ -18185,6 +18228,7 @@ sub track_files
 						file_location_file_uuid => $file_uuid,
 						file_location_host_uuid => $host_uuid,
 						file_location_active    => 1,
+						file_location_ready     => "same",
 					});
 					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 						reload             => $reload,
@@ -18250,6 +18294,7 @@ sub track_files
 						file_location_file_uuid => $file_uuid,
 						file_location_host_uuid => $host_uuid,
 						file_location_active    => 1,
+						file_location_ready     => "same",
 					});
 					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
 				}
@@ -18267,6 +18312,7 @@ sub track_files
 							file_location_file_uuid => $file_uuid,
 							file_location_host_uuid => $host_uuid,
 							file_location_active    => 1,
+							file_location_ready     => "same",
 						});
 						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
 					}
@@ -18323,6 +18369,7 @@ sub track_files
 							file_location_file_uuid => $file_uuid,
 							file_location_host_uuid => $host_uuid,
 							file_location_active    => 1,
+							file_location_ready     => "same",
 						});
 						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
 					}

Anvil/Tools/Storage.pm

@@ -19,6 +19,7 @@ my $THIS_FILE = "Storage.pm";
 # backup
 # change_mode
 # change_owner
+# check_files
 # check_md5sums
 # compress
 # copy_file
@@ -583,6 +584,142 @@ sub change_owner
 	return($error);
 }
 
+
+=head2 check_files
+
+This method checks the files on the local system. Specifically, it looks in C<< file_locations >> table and then checks if the file is "ready" or not. Depending on the results, C<< file_location_ready >> is updated if needed.
+
+This method takes no parameters.
+
+=cut
+sub check_files
+{
+	my $self      = shift;
+	my $parameter = shift;
+	my $anvil     = $self->parent;
+	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Storage->check_md5sums()" }});
+	
+	$anvil->Database->get_files({debug => $debug});
+	$anvil->Database->get_file_locations({debug => $debug});
+	
+	# Look for files that should be on this host.
+	my $host_uuid = $anvil->Get->host_uuid({debug => $debug});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }});
+	
+	# Sorting isn't useful really, but it ensures consistent listing run over run).
+	foreach my $file_location_file_uuid (sort {$a cmp $b} keys %{$anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}})
+	{
+		my $file_location_uuid      = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_location_file_uuid}{file_location_uuid};
+		my $file_location_file_uuid = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid};
+		my $file_location_active    = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
+		my $file_location_ready     = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_ready};
+		my $file_name               = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_name};
+		my $file_directory          = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_directory};
+		my $file_size               = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_size}; 
+		my $file_md5sum             = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_md5sum};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			's1:file_location_file_uuid' => $file_location_file_uuid, 
+			's2:file_location_uuid'      => $file_location_uuid, 
+			's3:file_location_file_uuid' => $file_location_file_uuid, 
+			's4:file_location_active'    => $file_location_active, 
+			's5:file_location_ready'     => $file_location_ready, 
+			's6:file_name'               => $file_name, 
+			's7:file_directory'          => $file_directory, 
+			's8:file_size'               => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")", 
+			's9:file_md5sum'             => $file_md5sum, 
+		}});
+		
+		my $full_path =  $file_directory."/".$file_name;
+		   $full_path =~ s/\/\//\//g;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { full_path => $full_path }});
+		
+		# If the file is not active, make sure the active is also false, regardless of anything else.
+		if (not $file_location_active)
+		{
+			if ($file_location_ready)
+			{
+				$anvil->Database->insert_or_update_file_locations({
+					debug                   => $debug, 
+					file_location_uuid      => $file_location_uuid, 
+					file_location_file_uuid => $file_location_file_uuid, 
+					file_location_host_uuid => $host_uuid, 
+					file_location_active    => $file_location_active, 
+					file_location_ready     => 0,
+				});
+			}
+		}
+		elsif (-e $full_path)
+		{
+			# It exists, what's it's size?
+			my $real_size = (stat($full_path))[7];
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				real_size => $real_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $real_size}).")", 
+			}});
+			
+			# If the size is the same as recorded, and the file is already 'ready', we're done.
+			if ($real_size == $file_size)
+			{
+				if (not $file_location_ready)
+				{
+					# Calculate the md5sum and see if it is ready now.
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0265", variables => { file => $full_path }});
+					if ($real_size > (128 * (2 ** 20)))
+					{
+						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0266", variables => { 
+							size => $anvil->Convert->bytes_to_human_readable({'bytes' => $real_size}),
+						}});
+					}
+					
+					# Update (or get) the md5sum.
+					my $real_md5sum = $anvil->Get->md5sum({debug => 2, file => $full_path});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { real_md5sum => $real_md5sum }});
+					
+					if ($real_md5sum eq $file_md5sum)
+					{
+						# It's ready now.
+						$anvil->Database->insert_or_update_file_locations({
+							debug                   => $debug, 
+							file_location_uuid      => $file_location_uuid, 
+							file_location_file_uuid => $file_location_file_uuid, 
+							file_location_host_uuid => $host_uuid, 
+							file_location_active    => $file_location_active, 
+							file_location_ready     => 1,
+						});
+					}
+				}
+			}
+			elsif ($file_location_ready)
+			{
+				# It's not ready.
+				$anvil->Database->insert_or_update_file_locations({
+					debug                   => $debug, 
+					file_location_uuid      => $file_location_uuid, 
+					file_location_file_uuid => $file_location_file_uuid, 
+					file_location_host_uuid => $host_uuid, 
+					file_location_active    => $file_location_active, 
+					file_location_ready     => 0,
+				});
+			}
+		}
+		elsif ($file_location_ready)
+		{
+			# File doesn't exist but is marked as read, mark it as not ready.
+			$anvil->Database->insert_or_update_file_locations({
+				debug                   => $debug, 
+				file_location_uuid      => $file_location_uuid, 
+				file_location_file_uuid => $file_location_file_uuid, 
+				file_location_host_uuid => $host_uuid, 
+				file_location_active    => $file_location_active, 
+				file_location_ready     => 0,
+			});
+		}
+	}
+	
+	return(0);
+}
+
+
 =head2 check_md5sums
 
 This is one half of a tool to let daemons detect when something they use has changed on disk and restart if any changes are found.
@@ -3679,6 +3816,7 @@ sub push_file
 						file_location_file_uuid => $file_uuid, 
 						file_location_host_uuid => $target_host_uuid, 
 						file_location_active    => 1, 
+						file_location_ready     => "same", 
 					});
 					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
 				}

notes

@@ -22,7 +22,7 @@ Common queries;
 * SELECT b.host_name, a.network_interface_uuid, a.network_interface_mac_address AS mac, a.network_interface_name AS name, a.network_interface_speed AS speed, a.network_interface_link_state AS link, a.network_interface_operational AS op, a.network_interface_duplex AS duplex, a.network_interface_medium AS medium, a.network_interface_bond_uuid AS bond_uuid, a.network_interface_bridge_uuid AS bridge_uuid FROM network_interfaces a, hosts b WHERE a.network_interface_host_uuid = b.host_uuid AND b.host_name LIKE 'an-a02%' AND a.network_interface_operational != 'DELETED' ORDER BY b.host_name ASC, a.network_interface_name ASC;
 * SELECT b.host_name, a.bond_uuid, a.bond_name, a.bond_mode, a.bond_mtu AS mtu, a.bond_primary_interface AS primary, a.bond_active_interface AS active, a.bond_mac_address AS mac, a.bond_operational AS op, c.bridge_name, a.modified_date FROM bonds a, hosts b, bridges c WHERE a.bond_host_uuid = b.host_uuid AND a.bond_bridge_uuid = c.bridge_uuid AND (b.host_uuid = 'b4e46faf-0ebe-e211-a0d6-00262d0ca874' OR b.host_uuid = '4ba42b4e-9bf7-e311-a889-899427029de4') ORDER BY b.host_name ASC, a.bond_name ASC;
 * SELECT b.host_name, a.bridge_uuid, a.bridge_name, a.bridge_id, a.bridge_mtu FROM bridges a, hosts b WHERE a.bridge_host_uuid = b.host_uuid AND b.host_name LIKE 'an-a02%' ORDER BY b.host_name ASC, a.bridge_name ASC;
-* SELECT a.host_name, b.file_name, c.file_location_active FROM hosts a, files b, file_locations c WHERE a.host_uuid = c.file_location_host_uuid AND b.file_uuid = c.file_location_file_uuid ORDER BY b.file_name ASC, a.host_name ASC;
+* SELECT a.host_name, b.file_name, c.file_location_active AS active, c.file_location_ready AS ready FROM hosts a, files b, file_locations c WHERE a.host_uuid = c.file_location_host_uuid AND b.file_uuid = c.file_location_file_uuid ORDER BY b.file_name ASC, a.host_name ASC;
 * SELECT b.host_name, a.health_agent_name, a.health_source_name, a.health_source_weight FROM health a, hosts b WHERE b.host_uuid = a.health_host_uuid AND b.host_name LIKE 'an-a02%' order by b.host_name ASC, a.health_agent_name ASC, a.health_source_weight ASC;
 
 

share/anvil.sql

@@ -1234,6 +1234,7 @@ CREATE TABLE file_locations (
     file_location_file_uuid    uuid                        not null,                   -- This is file to be moved to (or restored to) this machine.
     file_location_host_uuid    uuid                        not null,                   -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change.
     file_location_active       boolean                     not null    default TRUE,   -- This is set to true when the file should be on Anvil! machines, triggering rsyncs when needed. When set to false, the file will be deleted from members, if they exist.
+    file_location_ready        boolean                     not null    default FALSE,  -- This is set to true when the file is on the host with a good md5sum. If this is FALSE, any process needing this file should wait/loop until this goes TRUE
     modified_date              timestamp with time zone    not null,
     
     FOREIGN KEY(file_location_file_uuid) REFERENCES files(file_uuid), 
@@ -1247,6 +1248,7 @@ CREATE TABLE history.file_locations (
     file_location_file_uuid    text,
     file_location_host_uuid    uuid,
     file_location_active       boolean,
+    file_location_ready        boolean,
     modified_date              timestamp with time zone    not null
 );
 ALTER TABLE history.file_locations OWNER TO admin;
@@ -1262,12 +1264,14 @@ BEGIN
          file_location_file_uuid,
          file_location_host_uuid, 
          file_location_active, 
+         file_location_ready, 
          modified_date)
     VALUES
         (history_file_locations.file_location_uuid, 
          history_file_locations.file_location_file_uuid,
          history_file_locations.file_location_host_uuid, 
          history_file_locations.file_location_active, 
+         history_file_locations.file_location_ready, 
          history_file_locations.modified_date);
     RETURN NULL;
 END;

tools/anvil-daemon

@@ -544,39 +544,8 @@ sub handle_periodic_tasks
 		# Scan the local network.
 		update_state_file($anvil);
 		
-		# Make sure the shared directories exist.
+		# Check shared files.
-		foreach my $target (sort {$a cmp $b} keys %{$anvil->data->{path}{directories}{shared}})
+		check_files($anvils);
-		{
-			my $directory = $anvil->data->{path}{directories}{shared}{$target};
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
-				target    => $target, 
-				directory => $directory,
-			}});
-			if (not -e $anvil->data->{path}{directories}{shared}{$target})
-			{
-				my $failed = $anvil->Storage->make_directory({
-					directory => $directory,
-					group     => "apache", 
-					user      => "apache",
-					mode      => "0775",
-				});
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }});
-				if ($failed)
-				{
-					# Something went wrong.
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "log_0254", variables => {
-						directory => $directory,
-					}});
-				}
-				else
-				{
-					# Success
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0255", variables => {
-						directory => $directory,
-					}});
-				}
-			}
-		}
 		
 		# Check mail server config.
 		my $problem = $anvil->Email->check_config({debug => 3});
@@ -1703,6 +1672,52 @@ sub run_jobs
 	return(0);
 }
 
+#
+sub check_files
+{
+	my ($anvil) = @_;
+	
+	# Make sure the shared directories exist.
+	foreach my $target (sort {$a cmp $b} keys %{$anvil->data->{path}{directories}{shared}})
+	{
+		my $directory = $anvil->data->{path}{directories}{shared}{$target};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+			target    => $target, 
+			directory => $directory,
+		}});
+		if (not -e $anvil->data->{path}{directories}{shared}{$target})
+		{
+			my $failed = $anvil->Storage->make_directory({
+				directory => $directory,
+				group     => "apache", 
+				user      => "apache",
+				mode      => "0775",
+			});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }});
+			if ($failed)
+			{
+				# Something went wrong.
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "log_0254", variables => {
+					directory => $directory,
+				}});
+			}
+			else
+			{
+				# Success
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0255", variables => {
+					directory => $directory,
+				}});
+			}
+		}
+	}
+	
+	# Look for files on our system that are in file_locations. If they're shown as ready, make sure 
+	# they're there. If they're marked as not ready, see if they now are.
+	$anvil->Storage->check_files({debug => 2});
+	
+	return(0);
+}
+
 # This calls 'anvil-update-states' which will scan the local machine's state (hardware and software) and
 # record write it out to an HTML file 
 sub update_state_file

tools/anvil-manage-storage-groups

@@ -32,8 +32,6 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 $| = 1;
 
 my $anvil = Anvil::Tools->new();
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
 
 $anvil->Get->switches({list => [
 	"add",

tools/anvil-provision-server

@@ -31,8 +31,6 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 $| = 1;
 
 my $anvil = Anvil::Tools->new();
-$anvil->Log->level({set => 2});
-$anvil->Log->secure({set => 1});
 
 # Read switches
 $anvil->Get->switches({list => [

tools/anvil-version-changes

@@ -77,11 +77,13 @@ sub striker_checks
 	# This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release)
 	update_audits($anvil);
 	
-	### NOTE: Disabled until review complete
 	# This checks to make sure that the new dr_links table exists, and that existing anvil_dr1_host_uuid 
 	# entries are copied.
 	update_dr_links($anvil);
 	
+	# This checks to make sure that the new 'file_locations' -> 'file_location_ready' column exists.
+	update_file_location_ready($anvil);
+	
 	### TODO: Remove these later. This is here to clean up how we used to handle db_in_use and lock_request flags.
 	if (1)
 	{
@@ -241,6 +243,101 @@ CREATE TRIGGER trigger_dr_links
 	return(0);
 }
 
+# This checks to make sure that the new 'file_locations' -> 'file_location_ready' column exists.
+sub update_file_location_ready
+{
+	my ($anvil) = @_;
+	
+	
+	foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}})
+	{
+		my $query = "SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = 'public' AND table_name = 'file_locations' AND column_name = 'file_location_ready';";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+		
+		my $count = $anvil->Database->query({query => $query, uuid => $uuid, source => $THIS_FILE, line => __LINE__})->[0]->[0];
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
+		
+		if (not $count)
+		{
+			my $queries = [];
+			push @{$queries}, "ALTER TABLE public.file_locations ADD COLUMN file_location_ready boolean not null DEFAULT FALSE;";
+			push @{$queries}, "ALTER TABLE history.file_locations ADD COLUMN file_location_ready boolean;";
+			push @{$queries}, "DROP FUNCTION history_file_locations() CASCADE;";
+			push @{$queries}, q|CREATE FUNCTION history_file_locations() RETURNS trigger
+AS $$
+DECLARE
+    history_file_locations RECORD;
+BEGIN
+    SELECT INTO history_file_locations * FROM file_locations WHERE file_location_uuid = new.file_location_uuid;
+    INSERT INTO history.file_locations
+        (file_location_uuid,
+         file_location_file_uuid,
+         file_location_host_uuid, 
+         file_location_active, 
+         file_location_ready, 
+         modified_date)
+    VALUES
+        (history_file_locations.file_location_uuid, 
+         history_file_locations.file_location_file_uuid,
+         history_file_locations.file_location_host_uuid, 
+         history_file_locations.file_location_active, 
+         history_file_locations.file_location_ready, 
+         history_file_locations.modified_date);
+    RETURN NULL;
+END;
+$$
+LANGUAGE plpgsql;
+ALTER FUNCTION history_file_locations() OWNER TO admin;
+
+CREATE TRIGGER trigger_file_locations
+    AFTER INSERT OR UPDATE ON file_locations
+    FOR EACH ROW EXECUTE PROCEDURE history_file_locations();
+|;
+			foreach my $query (@{$queries})
+			{
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
+			}
+			$anvil->Database->write({debug => 2, uuid => $uuid, query => $queries, source => $THIS_FILE, line => __LINE__});
+		}
+	}
+	
+	# Now make sure that existing DR entries are copied here.
+	$anvil->Database->get_hosts({deubg => 2});
+	$anvil->Database->get_dr_links({debug => 2});
+	
+	foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
+	{
+		my $anvil_uuid          = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
+		my $anvil_dr1_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_dr1_host_uuid};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			"s1:anvil_name"          => $anvil_name,
+			"s2:anvil_uuid"          => $anvil_uuid, 
+			"s3:anvil_dr1_host_uuid" => $anvil_dr1_host_uuid, 
+		}});
+		if ($anvil_dr1_host_uuid)
+		{
+			my $dr1_host_name = $anvil->data->{hosts}{host_uuid}{$anvil_dr1_host_uuid}{short_host_name};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr1_host_name => $dr1_host_name }});
+			
+			if ((not exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid})                                          or 
+			    (not exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$anvil_dr1_host_uuid}) or 
+			    (not exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$anvil_dr1_host_uuid}{dr_link_uuid}))
+			{
+				# Add it.
+				my $dr_link_uuid = $anvil->Database->insert_or_update_dr_links({
+					debug              => 2,
+					dr_link_anvil_uuid => $anvil_uuid, 
+					dr_link_host_uuid  => $anvil_dr1_host_uuid, 
+					dr_link_note       => "auto_generated",
+				});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr1_host_name => $dr1_host_name }});
+			}
+		}
+	}
+	
+	return(0);
+}
+
 # This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release)
 sub update_audits
 {
@@ -487,6 +584,7 @@ CREATE TRIGGER trigger_file_locations
 					file_location_file_uuid => $file_uuid, 
 					file_location_host_uuid => $host_uuid, 
 					file_location_active    => 1,
+					file_location_ready     => "same",
 				});
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
 			}