* There were still references in anvil-manage-files to 'file_locations' -> 'file_location_host_uuid'. Had to rework some logic to get things working. More testing needed, but so far at least the "missing file" function is working again.

* Added missing always-available switchs in Get->switches * Create Storage->_wait_if_changing() to check to see if a file's size is changing and, if so, not return until it stops. Signed-off-by: Digimer <digimer@alteeve.ca>
2 years ago · be84a23924
parent e2a42870e5
commit be84a23924
5 changed files with 394 additions and 419 deletions
--- a/Anvil/Tools/Get.pm
+++ b/Anvil/Tools/Get.pm
@ -2297,12 +2297,14 @@ sub switches
 			next if $set_switch eq "?";
 			next if $set_switch eq "h";
 			next if $set_switch eq "help";
+			next if $set_switch eq "log-secure";
+			next if $set_switch eq "log-db-transactions";
 			next if $set_switch eq "raw";
+			next if $set_switch eq "resync-db";
 			next if $set_switch eq "v";
 			next if $set_switch eq "vv";
 			next if $set_switch eq "vvv";
 			next if $set_switch eq "vvvv";
-			next if $set_switch eq "log-secure";
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { set_switch => $set_switch }});
 			
 			my $found = 0;
--- a/Anvil/Tools/Storage.pm
+++ b/Anvil/Tools/Storage.pm
@ -46,6 +46,7 @@ my $THIS_FILE = "Storage.pm";
 # update_file
 # write_file
 # _create_rsync_wrapper
+# _wait_if_changing

 =pod

@ -5151,6 +5152,7 @@ fi";
 # Private functions                                                                                         #
 #############################################################################################################

+
 =head2

 This does the actual work of creating the C<< expect >> wrapper script and returns the path to that wrapper for C<< rsync >> calls.
@ -5227,4 +5229,104 @@ expect eof
 	return($wrapper_script);
 }

+
+=head3 _wait_if_changing
+
+This takes a full path to a file, and watches it for at specified number of seconds to see if the size is changing. If it is, this method waits until the file size stops changing. 
+
+Parameters;
+
+=head3 file (required)
+
+This is the full path to the file. If the file is not found, C<< !!error!! >> is returned.
+
+=head3 delay (optional, default '2')
+
+This is how long to wait before checking to see if the file has changed.
+
+=head3 last_size (optional)
+
+If this is set, it's the first size we compare against. If not passed, the size will be checked.
+
+=cut
+sub _wait_if_changing
+{
+	my $self      = shift;
+	my $parameter = shift;
+	my $anvil     = $self->parent;
+	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Storage->_create_rsync_wrapper()" }});
+	
+	# Check my parameters.
+	my $file      = defined $parameter->{file}      ? $parameter->{file}      : "";
+	my $delay     = defined $parameter->{delay}     ? $parameter->{delay}     : "";
+	my $last_size = defined $parameter->{last_size} ? $parameter->{last_size} : "";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		file      => $file, 
+		delay     => $delay, 
+		last_size => $last_size, 
+	}});
+	
+	if (not $delay)
+	{
+		$delay = 2;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { delay => $delay }});
+	}
+	elsif (($delay =~ /\D/) or ($delay == 0))
+	{
+		$delay = 2;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { delay => $delay }});
+	}
+	
+	if (not -e $file)
+	{
+		return("!!error!!");
+	}
+	
+	if (not $last_size)
+	{
+		$last_size = (stat($file))[7];
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			last_size => $last_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}).")", 
+		}});
+	}
+	
+	my $waiting = 1;
+	while ($waiting)
+	{
+		sleep $delay;
+		my $new_size = (stat($file))[7];
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			file      => $file,
+			last_size => $last_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}).")", 
+		}});
+		if ($new_size == $last_size)
+		{
+			# Size seems stable
+			$waiting = 0;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
+		}
+		else
+		{
+			# Might still be updating, wait.
+			my $difference = $new_size - $last_size;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0724", variables => { 
+				file             => $file,
+				old_size_bytes   => $anvil->Convert->add_commas({number => $last_size}),
+				old_size_hr      => $anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}),
+				new_size_bytes   => $anvil->Convert->add_commas({number => $new_size}),
+				new_size_hr      => $anvil->Convert->bytes_to_human_readable({'bytes' => $new_size}),
+				difference_bytes => $anvil->Convert->add_commas({number => $difference}),
+				difference_hr    => $anvil->Convert->bytes_to_human_readable({'bytes' => $difference}),
+			}});
+			
+			$last_size = $new_size;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { last_size => $last_size }});
+		}
+	}
+	
+	return(0);
+}
+
 1;
--- a/man/anvil-manage-files.8
+++ b/man/anvil-manage-files.8
@ -0,0 +1,38 @@
+.\" Manpage for the Anvil! server removal tool
+.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
+.TH anvil-manage-files "8" "August 02 2022" "Anvil! Intelligent Availability™ Platform"
+.SH NAME
+anvil-manage-files \- This program manages the files sync'ed across machines in the Anvil! cluster
+.SH SYNOPSIS
+.B anvil-manage-files 
+\fI\,<command> \/\fR[\fI\,options\/\fR]
+.SH DESCRIPTION
+This handles synchronizing files, typically ISO used to build new servers and scripts used to manage them, across Anvil! nodes.
+.TP
+.SH OPTIONS
+.TP
+\-?, \-h, \fB\-\-help\fR
+Show this man page.
+.TP
+\fB\-\-log-secure\fR
+When logging, record sensitive data, like passwords.
+.TP
+\-v, \-vv, \-vvv
+Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
+.SS "Commands:"
+.TP
+\fB\-\-delete\fR 
+This will delete the \fB\-\-file\fR </path/to/file> from the entire Anvil! cluster. 
+.TP
+This action is permanent!
+.TP
+\fB\-\-job-uuid\fR <name>
+The program is normally run as a job, with data on how to configure the host defined in the job. This switch allows the running of a specific job. If this is not set, the program will search for a job that has not yet been picked up by another process. If found, that job UUID is used automatically.
+.IP
+.SH AUTHOR
+Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
+.SH "REPORTING BUGS"
+Report bugs to users@clusterlabs.org
+
+
+", "download", "everywhere", "file", "is-script", "job-uuid", "rename", "to
--- a/share/words.xml
+++ b/share/words.xml
@ -1664,8 +1664,8 @@ The fingerprint of: [#!variable!machine!#] has changed! Updating it's entry in k
 		<key name="log_0279">The md5sum of file: [#!variable!file!#] failed to match. Discarding the downloaded file.</key>
 		<key name="log_0280">Failed to download: [#!variable!file!#] from: [#!variable!host_name!# (#!variable!ip!#). Will look on other hosts (if any left).</key>
 		<key name="log_0281">The file: [#!variable!file!#] on: [#!variable!host_name!# (#!variable!ip!#]) doesn't match the file we're looking for.
- Wanted; md5sum: [#!variable!file_md5sum!#], size: [#!variable!say_file_size!# (#!variable!file_size!# bytes)]
- Found;  md5sum: [#!variable!remote_md5sum!#], size: [#!variable!say_remote_size!# (#!variable!remote_size!# bytes)]
+- Wanted; size: [#!variable!say_file_size!# (#!variable!file_size!# bytes)]
+- Found;  size: [#!variable!say_remote_size!# (#!variable!remote_size!# bytes)]
 We will keep looking.</key>
 		<key name="log_0282">Already searched: [#!variable!host_name!# using another IP address, skipping this IP: [#!variable!ip!#].</key>
 		<key name="log_0283">Done.</key>
@ -2151,6 +2151,7 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
 		<key name="log_0721">The server: [#!variable!server!#] is ready to boot.</key>
 		<key name="log_0722">The server: [#!variable!server!#] was found to be running already, but it wasn't marked as booted. Marking it as if it just booted to handle any dependent servers.</key>
 		<key name="log_0723">The server: [#!variable!server!#] is configured to stay off, ignoring it.</key>
+		<key name="log_0724">The file: [#!variable!file!#] needs to be added to the database, but since the last scan it's size grew from: [#!variable!old_size_bytes!# (#!variables!old_size_hr!#)] to: [#!variable!new_size_bytes!# (#!variables!new_size_hr!#)]. A difference of: [#!variable!difference_bytes!# (#!variables!difference_hr!#)]. It might still be being uploaded, so we'll keep checking periodocally until the size stops changing.</key>
 		
 		<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
 		<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>
--- a/tools/anvil-manage-files
+++ b/tools/anvil-manage-files
@ -13,10 +13,9 @@
 #      - If not found anywhere, remove it from 'file_locations' and send an alert.
 #      - If found, check the size. If it differs, recalculate the md5sum.
 # - 3. If called with '--rename --file <filename> --to <newname>', rename the file and update 'files'.
-# - 4. If called with '--delete', remove from 'file_locations' and then remove from the local storage. If 
-#      also used with '--everywhere', then all copies on all systems we know about will be deleted. This is 
-#      done by registering a job against all known hosts. As such, if this is called and the target file 
-#      doesn't exist, it just clears the job and then exits.
+# - 4. If called with '--delete', remove from 'file_locations' and all copies on all systems. This is done by 
+#      registering a job against all known hosts. As such, if this is called and the target file  doesn't exist, 
+#      it just clears the job and then exits.
 # - 5. If called with '--is-script=[0|1]', mark as 'script' in the 'files' table and set/remove the executable bit.
 #
 # Exit codes;
@ -32,7 +31,7 @@
 # - 
 # 
 # NOTE:
-# - 
+# - remove unsyncs, add syncs.
 # 

 use strict;
@ -52,25 +51,11 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))

 my $anvil = Anvil::Tools->new();

-$anvil->data->{switches}{'delete'}    = "";
-$anvil->data->{switches}{download}    = "";
-$anvil->data->{switches}{everywhere}  = "";
-$anvil->data->{switches}{file}        = "";
-$anvil->data->{switches}{'is-script'} = "";
-$anvil->data->{switches}{'job-uuid'}  = "";
-$anvil->data->{switches}{'rename'}    = "";
-$anvil->data->{switches}{to}          = "";
-$anvil->Get->switches;
-$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
-	"switches::delete"     => $anvil->data->{switches}{'delete'},
-	"switches::download"   => $anvil->data->{switches}{download},
-	"switches::everywhere" => $anvil->data->{switches}{everywhere},
-	"switches::file"       => $anvil->data->{switches}{file},
-	"switches::is-script"  => $anvil->data->{switches}{'is-script'},
-	"switches::job-uuid"   => $anvil->data->{switches}{'job-uuid'},
-	"switches::rename"     => $anvil->data->{switches}{'rename'},
-	"switches::to"         => $anvil->data->{switches}{to},
-}});
+$anvil->Log->level({set => 2});
+$anvil->Log->secure({set => 1});
+
+$anvil->Get->switches({list => ["delete", "download", "file", "is-script", "job-uuid", "rename", "remove", "add", "to"], man => $THIS_FILE});
+$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});

 # Connect or die
 $anvil->Database->connect;
@ -171,31 +156,24 @@ sub find_missing_files
 	# What am I? This will impact how missing files are found.
 	my $query = "
 SELECT 
-    a.file_uuid, 
-    a.file_directory, 
-    a.file_name, 
-    a.file_size, 
-    a.file_md5sum  
+    file_uuid, 
+    file_directory, 
+    file_name, 
+    file_size, 
+    file_md5sum  
 FROM 
-    files a, 
-    hosts b, 
-    file_locations c 
+    files 
 WHERE 
-    b.host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})." 
-AND 
-    a.file_uuid = c.file_location_file_uuid 
-AND 
-    b.host_uuid = c.file_location_host_uuid 
+    file_type != 'DELETED' 
 ORDER BY 
-    a.file_directory ASC, 
-    a.file_name ASC, 
-    b.host_name ASC
+    file_directory ASC, 
+    file_name ASC 
 ;";
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
 	
 	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
 	my $count   = @{$results};
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 		results => $results, 
 		count   => $count, 
 	}});
@ -206,262 +184,105 @@ ORDER BY
 		my $file_name      =  $row->[2];
 		my $file_size      =  $row->[3];
 		my $file_md5sum    =  $row->[4];
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+		my $full_path      =  $file_directory."/".$file_name;
+		   $full_path      =~ s/\/\//\//g;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			's1:file_uuid'      => $file_uuid, 
 			's2:file_directory' => $file_directory, 
 			's3:file_name'      => $file_name, 
+			's4:file_size'      => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")", 
+			's5:file_md5sum'    => $file_md5sum, 
+			's6:full_path'      => $full_path,
 		}});
 		
-		my $test_file =  $file_directory."/".$file_name;
-		   $test_file =~ s/\/\//\//g;
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { test_file => $test_file }});
-		
-		if (not -e $test_file)
+		# If we're a striker, we want all files.
+		my $host_type = $anvil->Get->host_type();
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
+		if ($host_type eq "striker")
+		{
+			if (not -e $full_path)
 			{
 				# Missing file!
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0269", variables => { file => $test_file }});
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0269", variables => { file => $full_path }});
 				
 				# Find what target, if any, we'll the file from.
-			my ($found) = find_file($anvil, $file_uuid, $test_file, $file_size, $file_md5sum);
+				my ($found) = find_file($anvil, $file_uuid, $full_path, $file_size, $file_md5sum);
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
 			}
 		}
+	}
 	
 	### TODO: Left off here.
 	
 	return(0);
 }

-### TODO: Add sorting by preferred target
 # This looks for a file on another system. The exact order of search depends on what kind of machine we are.
 sub find_file
 {
 	my ($anvil, $file_uuid, $full_path, $file_size, $file_md5sum) = @_;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		's1:file_uuid'   => $file_uuid,
+		's2:full_path'   => $full_path, 
+		's3:file_size'   => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
+		's4:file_md5sum' => $file_md5sum, 
+	}});
 	
 	my $found = 0;
 	
-	# What are my IPs?
-	my $local_host = $anvil->Get->short_host_name();
-	$anvil->Network->get_ips();
-	foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
+	# We want to search Striker's first, DR hosts second and nodes third.
+	$anvil->Database->get_hosts;
+	my $host_order = [];
+	foreach my $type ("striker", "dr", "node")
 	{
-		next if not $anvil->data->{network}{$local_host}{interface}{$interface}{ip};
-		next if not $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask};
-		my $ip          = $anvil->data->{network}{$local_host}{interface}{$interface}{ip};
-		my $subnet_mask = $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask};
-		my $network     = $anvil->Network->get_network({ip => $ip, subnet_mask => $subnet_mask});
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
-			's1:interface'   => $interface, 
-			's2:ip'          => $ip, 
-			's3:subnet_mask' => $subnet_mask, 
-			's4:network'     => $network,
-		}});
-		
-		my $type = "other";
-		my $sort = $interface;
-		if ($interface =~ /^((?:bc|s|if)n)(\d+)/)
-		{
-			$type = $1;
-			$sort = $2;
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
-				's1:type' => $type, 
-				's2:sort' => $sort, 
-			}});
-		}
-		$anvil->data->{local_ip}{by_network}{$network}{type}        = $type;
-		$anvil->data->{local_ip}{by_network}{$network}{'sort'}      = $sort;
-		$anvil->data->{local_ip}{by_network}{$network}{interface}   = $interface;
-		$anvil->data->{local_ip}{by_network}{$network}{ip}          = $ip;
-		$anvil->data->{local_ip}{by_network}{$network}{subnet_mask} = $subnet_mask;
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
-			"s1:local_ip::by_network::${network}::type"        => $anvil->data->{local_ip}{by_network}{$network}{type}, 
-			"s1:local_ip::by_network::${network}::sort"        => $anvil->data->{local_ip}{by_network}{$network}{'sort'}, 
-			"s2:local_ip::by_network::${network}::interface"   => $anvil->data->{local_ip}{by_network}{$network}{interface}, 
-			"s3:local_ip::by_network::${network}::ip"          => $anvil->data->{local_ip}{by_network}{$network}{ip}, 
-			"s4:local_ip::by_network::${network}::subnet_mask" => $anvil->data->{local_ip}{by_network}{$network}{subnet_mask}, 
-		}});
-	}
-	
-	# Create a hash we can sort through that are on the same subnet_mask as us.
-	my $query = "
-SELECT 
-    a.host_uuid, 
-    a.host_name, 
-    a.host_type, 
-    b.file_directory, 
-    b.file_name, 
-    b.file_size, 
-    b.file_md5sum 
-FROM 
-    hosts a, 
-    files b, 
-    file_locations c 
-WHERE 
-    a.host_uuid =  c.file_location_host_uuid 
-AND 
-    b.file_uuid =  c.file_location_file_uuid 
-AND 
-    b.file_uuid =  ".$anvil->Database->quote($file_uuid)." 
-AND 
-    a.host_uuid != ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})." 
-ORDER BY 
-    file_mtime DESC;
-;";
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
-	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
-	my $count   = @{$results};
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
-		results => $results, 
-		count   => $count, 
-	}});
-	foreach my $row (@{$results})
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { type => $type }});
+		foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
 		{
-		my $host_uuid      =  $row->[0];
-		my $host_name      =  $row->[1];
-		my $host_type      =  $row->[2];
-		my $file_directory =  $row->[3];
-		my $file_name      =  $row->[4];
-		my $file_size      =  $row->[5];
-		my $file_md5sum    =  $row->[6];
-		my $test_file      =  $file_directory."/".$file_name;
-		   $test_file      =~ s/\/\//\//g;
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
-			host_uuid      => $host_uuid, 
-			host_name      => $host_name, 
-			host_type      => $host_type, 
-			file_directory => $file_directory, 
-			file_name      => $file_name, 
-			file_size      => $file_size, 
-			file_md5sum    => $file_md5sum,
-			test_file      => $test_file, 
+			my $host_uuid = $anvil->data->{sys}{hosts}{by_name}{$host_name};
+			my $host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				's1:host_name' => $host_name, 
+				's2:host_type' => $host_type, 
+				's3:host_uuid' => $host_uuid,
 			}});
 			
-		# What IP addresses are on this machine?
-		my $query = "
-SELECT 
-    ip_address_address, 
-    ip_address_subnet_mask 
-FROM 
-    ip_addresses 
-WHERE 
-    ip_address_note      != 'DELETED' 
-AND 
-    ip_address_host_uuid =  ".$anvil->Database->quote($host_uuid)."
-;";
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
-		my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
-		my $count   = @{$results};
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
-			results => $results, 
-			count   => $count, 
-		}});
-		foreach my $row (@{$results})
-		{
-			my $ip_address_address     = $row->[0];
-			my $ip_address_subnet_mask = $row->[1];
-			my $network                = $anvil->Network->get_network({ip => $ip_address_address, subnet_mask => $ip_address_subnet_mask});
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
-				ip_address_address     => $ip_address_address, 
-				ip_address_subnet_mask => $ip_address_subnet_mask, 
-				network                => $network, 
-			}});
+			next if $host_type ne $type;
+			next if $host_uuid eq $anvil->Get->host_uuid;
 			
-			# Are we on the same subnet?
-			if (exists $anvil->data->{local_ip}{by_network}{$network})
-			{
-				# We're on the same subnet!
-				my $type = $anvil->data->{local_ip}{by_network}{$network}{type};
-				my $sort = $anvil->data->{local_ip}{by_network}{$network}{'sort'};
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
-					"s1:type" => $type, 
-					"s2:sort" => $sort, 
-				}});
-				
-				# Record.
-				$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip}        = $ip_address_address;
-				$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name}      = $host_name;
-				$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid} = $host_uuid;
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
-					"s1:peer_ip::${host_type}::${type}::${sort}::name"      => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name}, 
-					"s2:peer_ip::${host_type}::${type}::${sort}::ip"        => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip}, 
-					"s3:peer_ip::${host_type}::${type}::${sort}::host_uuid" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid}, 
-				}});
-			}
+			push @{$host_order}, $host_uuid;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_uuid => $host_uuid }});
 		}
 	}
 	
-	### TODO: Do this according to what we are.
-	# Sort through what we've found.
+	# Now search.
 	my $file_found = 0;
-	my $searched_hosts = {};
-	foreach my $host_type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}})
-	{
-		last if $file_found; 
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }});
-		foreach my $type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}})
-		{
-			last if $file_found; 
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { type => $type }});
-			foreach my $sort (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}{$type}})
+	foreach my $search_host_uuid (@{$host_order})
 	{
 		last if $file_found;
-				my $ip          = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip};
-				my $name        = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name};
-				my $host_uuid   = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid};
-				my $remote_user = "admin";
-				my $password    = "";
+		my $target_host = $anvil->data->{hosts}{host_uuid}{$search_host_uuid}{short_host_name};
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-					's1:host_type'   => $host_type,
-					's2:type'        => $type, 
-					's3:sort'        => $sort,
-					's4:name'        => $name,
-					's5:ip'          => $ip,
-					's6:host_uuid'   => $host_uuid,
-					's7:remote_user' => $remote_user, 
-					's8:password'    => $password, 
+			's1:search_host_uuid' => $search_host_uuid,
+			's2:target_host'      => $target_host, 
 		}});
 		
-				if ((exists $searched_hosts->{$name}) && ($searched_hosts->{$name}))
-				{
-					# Already searched this host, this is just a different IP. Skip it.
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "log_0282", variables => {
-						host_name => $name, 
-						ip        => $ip, 
-					}});
-					next;
-				}
-				$searched_hosts->{$name} = 1;
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { "searched_hosts->{$name}" => $searched_hosts->{$name} }});
+		my $target_ip = $anvil->Network->find_target_ip({host_uuid => $search_host_uuid});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_ip => $target_ip }});
+		next if not $target_ip;
 		
-				### NOTE: There's a bug in Net::SSH2 on RHEL8 where passwordless SSH doesn't 
-				###       work. So for now, we'll manually pull in passwords from the 
-				###       anvil.conf using 'hosts::<host_name>::password' or 
-				###       'hosts::<host_uuid>::password'. This will be removed when the bug 
-				###       is fixed.
-				if ((exists $anvil->data->{hosts}{$host_uuid}{password}{$remote_user}) && ($anvil->data->{hosts}{$host_uuid}{password}{$remote_user}))
-				{
-					$password = $anvil->data->{hosts}{$host_uuid}{password}{$remote_user};
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }});
-				}
-				elsif ((exists $anvil->data->{hosts}{$name}{password}{$remote_user}) && ($anvil->data->{hosts}{$name}{password}{$remote_user}))
-				{
-					$password = $anvil->data->{hosts}{$host_uuid}{password};
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }});
-				}
+		# See if the file is on the target and, if so, if it matches.
+		### NOTE: If we want to use md5sum again, use '--with-md5sum'
+		my $shell_call = $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path.$anvil->Log->switches;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+		
+		# Test access
+		my $access = $anvil->Remote->test_access({target => $target_ip});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
+		next if not $access;
 		
-				### NOTE: This might take a while for the call to return if we're md5sum'ing
-				###       a large file like an ISO.
-				### TODO: Should we ask for the size, then follow up with the md5sum for 
-				###       files that are over a certain size? Or will this get called rarely
-				###       enough in practice that it doesn't really matter?
-				# If the file is found, we'll parse these out.
 		my $remote_size                    = 0;
-				my $remote_md5sum                  = "";
 		my ($output, $error, $return_code) = $anvil->Remote->call({
-					shell_call  => $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path." --with-md5sum".$anvil->Log->switches, 
-					remote_user => $remote_user, 
-					password    => $password,
-					target      => $ip, 
+			shell_call => $shell_call, 
+			target     => $target_ip, 
 		});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			error       => $error,
@ -470,39 +291,36 @@ AND
 		}});
 		foreach my $line (split/\n/, $output)
 		{
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
 			if ($line =~ /^size: \[(\d+)\]$/)
 			{
 				$remote_size = $1;
-						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_size => $remote_size }});
-					}
-					if ($line =~ /^md5sum: \[(.*)\]$/)
-					{
-						$remote_md5sum = $1;
-						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_md5sum => $remote_md5sum }});
+				$found       = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					remote_size => $remote_size,
+					found       => $found,
+				}});
 			}
 		}
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-					's1:remote_size'   => $remote_size,
-					's2:file_size'     => $file_size,
-					's3:remote_md5sum' => $remote_md5sum, 
-					's4:file_md5sum'   => $file_md5sum,
+			's1:found'       => $found,
+			's2:remote_size' => $remote_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}).")",
+			's3:file_size'   => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
 		}});
+		next if not $found;
 		
-				### Do I really need to match sizes if the md5sum is the same?
-				if (($remote_size eq $file_size) && ($remote_md5sum eq $file_md5sum))
+		if ($remote_size eq $file_size)
 		{
 			# Pull it over!
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0276", variables => {
 				file      => $full_path, 
-						host_name => $name, 
-						ip        => $ip, 
+				host_name => $target_host, 
+				ip        => $target_ip, 
 			}});
 			my $failed = $anvil->Storage->rsync({
 				debug       => 2, 
 				destination => $full_path,
-						password    => $password, 
-						source      => $remote_user."\@".$ip.":".$full_path,
+				source      => "root\@".$target_ip.":".$full_path,
 			});
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
 			
@ -538,8 +356,8 @@ AND
 				# Failed to rsync.
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0280", variables => { 
 					file      => $full_path, 
-							host_name => $name, 
-							ip        => $ip, 
+					host_name => $target_host, 
+					ip        => $target_ip, 
 				}});
 			}
 		}
@ -548,10 +366,8 @@ AND
 			# Doesn't match what we're looking for.
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0281", variables => { 
 				file            => $full_path, 
-						host_name       => $name, 
-						ip              => $ip, 
-						remote_md5sum   => $remote_md5sum, 
-						file_md5sum     => $file_md5sum,
+				host_name       => $target_host, 
+				ip              => $target_ip, 
 				file_size       => $file_size,
 				say_file_size   => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
 				remote_size     => $remote_size,
@ -559,11 +375,9 @@ AND
 			}});
 		}
 	}
-		}
-	}
 	
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
-	return($found);
+	return($file_found);
 }


@ -577,6 +391,7 @@ sub check_incoming
 	{
 		delete $anvil->data->{scan}{directories};
 	}
+	
 	# Read any files in '/mnt/shared/incoming'.
 	$anvil->Storage->scan_directory({
 		debug     => 3,
@ -607,6 +422,8 @@ sub check_incoming
 		my $file_mimetype   =  $anvil->data->{scan}{directories}{$full_path}{mimetype};
 		my $file_executable =  $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0;
 		my $say_mimetype    =  convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable);
+		my $full_path       =  $file_directory."/".$file_name;
+		   $full_path       =~ s/\/\//\//g;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 			file_name       => $file_name, 
 			file_size       => $file_size, 
@ -614,6 +431,7 @@ sub check_incoming
 			file_mimetype   => $file_mimetype, 
 			file_executable => $file_executable, 
 			say_mimetype    => $say_mimetype, 
+			full_path       => $full_path,
 		}});
 		
 		# Do I know about this file? If so, is the file the same size? If either is no, calculate the md5sum.
@ -629,8 +447,19 @@ sub check_incoming
 		my $file_md5sum =  $recorded_md5sum;
 		if ((not $file_uuid) or ($file_size != $recorded_size))
 		{
-			# Yes. But first, do we have a size mismatch? If so, see if we need to pull a newer 
-			# version down from elsewhere.
+			# It's possible the file is still uploading, so sleep for 2 seconds and see if the 
+			# size is still changing.
+			my $last_size = $file_size;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				last_size => $last_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}).")", 
+			}});
+			$anvil->Storage->_wait_if_changing({
+				file      => $full_path,
+				last_size => $file_size, 
+			});
+			
+			# Yes, caluclate md5sum, but first, do we have a size mismatch? If so, see if we need
+			# to pull a newer version down from elsewhere.
 			if (($file_uuid) && ($file_mtime <= $recorded_mtime))
 			{
 				# We've got an older file, we need to update.
@ -675,14 +504,6 @@ sub check_incoming
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
 		next if not $file_uuid;
 		
-		# Make sure we know about this file on this system
-		my ($file_locatiom_uuid) = $anvil->Database->insert_or_update_file_locations({
-			debug                   => 3,
-			file_location_file_uuid => $file_uuid, 
-			file_location_host_uuid => $anvil->data->{sys}{host_uuid},
-		});
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_locatiom_uuid => $file_locatiom_uuid }});
-		
 		# Are we in the incoming directory? If so, move the file.
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 			full_path          => $full_path,
@ -702,6 +523,13 @@ sub check_incoming
 				file   => $full_path,
 				target => $target, 
 			}});
+			
+			# Wait in case it's still being uploaded.
+			$anvil->Storage->_wait_if_changing({
+				file      => $full_path,
+				last_size => $file_size, 
+			});
+			
 			$anvil->Storage->move_file({
 				debug       => 3,
 				source_file => $full_path, 
@ -975,47 +803,31 @@ sub convert_mimetype
 	return($say_mimetype);
 }

-# This looks for any files with the file_type of 'DELETED' and that has an entry in file_locations for this
-# machine. Any that are found will have their file deleted and the file_locations entry deleted.
+# This looks for any files with the file_type of 'DELETED'. Any that are found will be deleted.
 sub check_for_deletes
 {
 	my ($anvil) = @_;
 	
-	my $query = "
-SELECT 
-    a.file_uuid, 
-    b.file_location_uuid, 
-    a.file_directory || '/' || a.file_name AS full_path 
-FROM 
-    files a, 
-    file_locations b 
-WHERE 
-    a.file_uuid = b.file_location_file_uuid 
-AND 
-    a.file_type = 'DELETED' 
-AND 
-    b.file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
-;";
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
+	# Get a list of files.
+	my $query = "SELECT file_uuid, file_directory || '/' || file_name AS full_path FROM files WHERE file_type = 'DELETED';";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
 	
 	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
 	my $count   = @{$results};
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 		results => $results, 
 		count   => $count, 
 	}});
 	foreach my $row (@{$results})
 	{
 		my $file_uuid = $row->[0];
-		my $file_location_uuid = $row->[1];
-		my $full_path          = $row->[2];
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+		my $full_path = $row->[1];
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			's1:file_uuid' => $file_uuid, 
-			's2:file_location_uuid' => $file_location_uuid, 
-			's3:full_path'          => $full_path, 
+			's2:full_path' => $full_path, 
 		}});
 		
-		# Get rid of it (if it actually exists) and then remove it from file_locations.
+		# Get rid of it (if it actually exists).
 		if (-e $full_path)
 		{
 			# Delete it.
@ -1038,11 +850,36 @@ AND
 			}
 		}
 		
+		# If we're a Striker, check for any file_locations that point to this file and DELETE them.
+		my $host_type = $anvil->Get->host_type();
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
+		if ($host_type eq "striker")
+		{
+			my $query = "SELECT file_location_uuid FROM file_locations WHERE file_location_file_uuid = ".$anvil->Database->quote($file_uuid).";";
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+			
+			my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
+			my $count   = @{$results};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				results => $results, 
+				count   => $count, 
+			}});
+			foreach my $row (@{$results})
+			{
+				my $file_location_uuid = $row->[0];
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
+				
 				# Delete the entry from file_locations, if needed.
+				my $query = "DELETE FROM history.file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";";
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
+				$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
+				
 				$query = "DELETE FROM file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";";
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
 				$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
 			}
+		}
+	}
 	
 	return(0);
 }
@ -1061,7 +898,7 @@ sub handle_delete
 		# Um...
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"});
 		$anvil->Job->update_progress({
-			progress => 0, 
+			progress => 100, 
 			message  => "error_0052", 
 			job_uuid => $anvil->data->{jobs}{'job-uuid'},
 		});
@ -1072,7 +909,7 @@ sub handle_delete
 		# We don't do that here...
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0053", variables => { file => $full_path }});
 		$anvil->Job->update_progress({
-			progress => 0, 
+			progress => 100, 
 			message  => "error_0053,!!file!".$full_path."!!", 
 			job_uuid => $anvil->data->{jobs}{'job-uuid'},
 		});
@ -1126,7 +963,7 @@ AND
 	# database. This way, if anything goes wrong below, future runs will try (again) to delete the file 
 	# itself. If it's only beind deleted locally, and it fails for some reason, there will be no attempt
 	# to try again.
-	if (($file_uuid) && ($anvil->data->{switches}{everywhere}))
+	if ($file_uuid)
 	{
 		# Yup.
 		($file_uuid) = $anvil->Database->insert_or_update_files({
@ -1164,18 +1001,13 @@ AND
 		}
 	}
 	
-	# Now, if I have a file_uuid, delete it from file_locations if it exists.
+	# Now, if I have a file_uuid, delete it from file_locations if it exists. It's ok if peers in an 
+	# Anvil! haven't deleted yet, as they'll see the file marked as 'DELETED' and purge their local 
+	# copies.
 	if ($file_uuid)
 	{
 		# Delete the entry from file_locations, if needed.
-		my $query = "
-DELETE FROM 
-    file_locations 
-WHERE 
-    file_location_file_uuid = ".$anvil->Database->quote($file_uuid)." 
-AND 
-    file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
-;";
+		my $query = "DELETE FROM file_locations WHERE file_location_file_uuid = ".$anvil->Database->quote($file_uuid).";";
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
 		$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
 	}