* Created Storage->move_file().

* Finished getting anvil-manage-files to find and process new files in /mnt/shared/incoming. Created a 'convert_mimetype' function to translate returned mimetype to a file type we care about.

Signed-off-by: Digimer <digimer@alteeve.ca>

Anvil/Tools.pm

@@ -1002,6 +1002,7 @@ sub _set_paths
 				logger				=>	"/usr/bin/logger",
 				md5sum				=>	"/usr/bin/md5sum",
 				'mkdir'				=>	"/usr/bin/mkdir",
+				mv				=>	"/usr/bin/mv",
 				nmcli				=>	"/bin/nmcli",
 				openssl				=>	"/usr/bin/openssl", 
 				passwd				=>	"/usr/bin/passwd",

Anvil/Tools/Database.pm

@@ -2211,6 +2211,8 @@ sub insert_or_update_file_locations
 	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_file_locations()" }});
 	
+	my $file                    = defined $parameter->{file}                    ? $parameter->{file}                    : "";
+	my $line                    = defined $parameter->{line}                    ? $parameter->{line}                    : "";
 	my $file_location_uuid      = defined $parameter->{file_location_uuid}      ? $parameter->{file_location_uuid}      : "";
 	my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : "";
 	my $file_location_host_uuid = defined $parameter->{file_location_host_uuid} ? $parameter->{file_location_host_uuid} : $anvil->data->{sys}{host_uuid};
@@ -2301,15 +2303,11 @@ INSERT INTO
     file_location_uuid, 
     file_location_file_uuid, 
     file_location_host_uuid, 
-    file_md5sum, 
-    file_type, 
     modified_date 
 ) VALUES (
     ".$anvil->data->{sys}{database}{use_handle}->quote($file_location_uuid).", 
     ".$anvil->data->{sys}{database}{use_handle}->quote($file_location_file_uuid).", 
     ".$anvil->data->{sys}{database}{use_handle}->quote($file_location_host_uuid).", 
-    ".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).", 
-    ".$anvil->data->{sys}{database}{use_handle}->quote($file_type).", 
     ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})."
 );
 ";
@@ -2322,7 +2320,7 @@ INSERT INTO
 		my $query = "
 SELECT 
     file_location_file_uuid, 
-    file_location_host_uuid, 
+    file_location_host_uuid 
 FROM 
     file_locations 
 WHERE 
@@ -2412,6 +2410,8 @@ sub insert_or_update_files
 	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_files()" }});
 	
+	my $file        = defined $parameter->{file}        ? $parameter->{file}        : "";
+	my $line        = defined $parameter->{line}        ? $parameter->{line}        : "";
 	my $file_uuid   = defined $parameter->{file_uuid}   ? $parameter->{file_uuid}   : "";
 	my $file_name   = defined $parameter->{file_name}   ? $parameter->{file_name}   : "";
 	my $file_size   = defined $parameter->{file_size}   ? $parameter->{file_size}   : "";

Anvil/Tools/Get.pm

@@ -616,13 +616,14 @@ sub md5sum
 	
 	my $sum = "";
 	my $file = defined $parameter->{file} ? $parameter->{file} : "";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file => $file }});
 	
 	if (-e $file)
 	{
 		my $shell_call = $anvil->data->{path}{exe}{md5sum}." ".$file;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
 		
-		my $return = $anvil->System->call({shell_call => $shell_call});
+		my $return = $anvil->System->call({debug => $debug, shell_call => $shell_call});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'return' => $return }});
 		
 		# split the sum off.

Anvil/Tools/Storage.pm

@@ -22,6 +22,7 @@ my $THIS_FILE = "Storage.pm";
 # copy_file
 # find
 # make_directory
+# move_file
 # read_config
 # read_file
 # read_mode
@@ -710,12 +711,16 @@ fi";
 			}});
 			if ($output->[0] eq "source file not found")
 			{
-				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { source_file => $source_file }});
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { 
+					method      => "copy_file",
+					source_file => $source_file,
+				}});
 				return(1);
 			}
 			if (($output->[0] eq "source file exists") && (not $overwrite))
 			{
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => {
+					method      => "copy_file",
 					source_file => $source_file,
 					target_file => $target_file,
 				}});
@@ -732,6 +737,7 @@ fi";
 				});
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }});
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => {
+					method      => "copy_file",
 					source_file => $source_file,
 					target_file => $target_file,
 				}});
@@ -755,7 +761,10 @@ fi";
 		# Copying locally
 		if (not -e $source_file)
 		{
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { source_file => $source_file }});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { 
+				method      => "copy_file",
+				source_file => $source_file,
+			}});
 			return(1);
 		}
 		
@@ -764,6 +773,7 @@ fi";
 		{
 			# This isn't an error.
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => {
+				method      => "copy_file",
 				source_file => $source_file,
 				target_file => $target_file,
 			}});
@@ -781,6 +791,7 @@ fi";
 			if ($failed)
 			{
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => {
+					method      => "copy_file",
 					source_file => $source_file,
 					target_file => $target_file,
 				}});
@@ -1070,6 +1081,249 @@ fi;";
 	return($failed);
 }
 
+=head2 move_file
+
+This moves a file, with a few additional checks like creating the target directory if it doesn't exist, aborting if the file already exists in the target, etc. It can move files on the local or a remote machine.
+
+As with the system copy, the target can be a directory (denoted with an ending c<< / >>), or a it can be renamed in the process (but not ending with C<< / >>).
+
+ # Example moving
+ $anvil->Storage->move_file({source_file => "/some/file", target_file => "/another/directory/"});
+
+ # Example moving with a rename at the same time
+ $anvil->Storage->move_file({source_file => "/some/file", target_file => "/another/directory/new_name"});
+
+Returns C<< 0 >> on success, otherwise C<< 1 >>.
+
+Parameters;
+
+=head3 overwrite (optional)
+
+If this is set to 'C<< 1 >>', and if the target file exists, it will be replaced.
+
+If this is not passed and the target exists, this module will return 'C<< 3 >>'.
+
+=head3 port (optional, default 22)
+
+If C<< target >> is set, this is the TCP port number used to connect to the remote machine.
+
+=head3 password (optional)
+
+If C<< target >> is set, this is the password used to log into the remote system as the C<< remote_user >>. If it is not set, an attempt to connect without a password will be made (though this will usually fail).
+
+=head3 source_file (required)
+
+This is the source file. If it isn't specified, 'C<< 1 >>' will be returned. If it doesn't exist, this method will return 'C<< 4 >>'.
+
+=head3 target (optional)
+
+If set, the file will be copied on the target machine. This must be either an IP address or a resolvable host name. 
+
+=head3 target_file (required)
+
+This is the target B<< file >>, not the directory to put it in. The target file name can be different from the source file name.
+
+if this is not specified, 'C<< 2 >>' will be returned.
+
+=head3 remote_user (optional, default root)
+
+If C<< target >> is set, this is the user account that will be used when connecting to the remote system.
+
+=cut
+sub move_file
+{
+	my $self      = shift;
+	my $parameter = shift;
+	my $anvil     = $self->parent;
+	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
+	
+	my $overwrite   = defined $parameter->{overwrite}   ? $parameter->{overwrite}   : 0;
+	my $password    = defined $parameter->{password}    ? $parameter->{password}    : "";
+	my $remote_user = defined $parameter->{remote_user} ? $parameter->{remote_user} : "root";
+	my $source_file = defined $parameter->{source_file} ? $parameter->{source_file} : "";
+	my $target_file = defined $parameter->{target_file} ? $parameter->{target_file} : "";
+	my $target      = defined $parameter->{target}      ? $parameter->{target}      : "";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		overwrite   => $overwrite,
+		password    => $anvil->Log->secure ? $password : $anvil->Words->string({key => "log_0186"}), 
+		remote_user => $remote_user, 
+		source_file => $source_file, 
+		target_file => $target_file,
+		target      => $target,
+	}});
+	
+	if (not $source_file)
+	{
+		# No source passed.
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Storage->move_file()", parameter => "source_file" }});
+		return(1);
+	}
+	if (not $target_file)
+	{
+		# No target passed.
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Storage->move_file()", parameter => "target_file" }});
+		return(2);
+	}
+	
+	# If we have a target directory, pull the file name off the source for the target checks.
+	my ($directory, $file) = ($target_file =~ /^(.*)\/(.*?)$/);
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		directory => $directory, 
+		file      => $file,
+	}});
+	if (not $file)
+	{
+		($file)      =  ($source_file =~ /^.*\/(.*?)$/);
+		$target_file .= $file;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			file        => $file,
+			target_file => $target_file, 
+		}});
+	}
+	
+	if ($target)
+	{
+		# Copying on a remote system.
+		my $proceed    = 1;
+		my $shell_call = "
+if [ -e '".$source_file."' ]; 
+    ".$anvil->data->{path}{exe}{echo}." 'source file exists'
+else
+    ".$anvil->data->{path}{exe}{echo}." 'source file not found'
+fi
+if [ -d '".$target_file."' ];
+    ".$anvil->data->{path}{exe}{echo}." 'target file exists'
+elif [ -d '".$directory."' ];
+    ".$anvil->data->{path}{exe}{echo}." 'target directory exists'
+else
+    ".$anvil->data->{path}{exe}{echo}." 'target directory not found'
+fi";
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0166", variables => { shell_call => $shell_call, target => $target, remote_user => $remote_user }});
+		my ($error, $output) = $anvil->Remote->call({
+			debug       => $debug, 
+			target      => $target,
+			user        => $remote_user, 
+			password    => $password,
+			remote_user => $remote_user, 
+			shell_call  => $shell_call,
+		});
+		if ($error)
+		{
+			# Something went wrong.
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0267", variables => { 
+				source_file => $source_file, 
+				target_file => $target_file, 
+				error       => $error,
+				output      => $output,
+				target      => $target, 
+				remote_user => $remote_user, 
+			}});
+			return(1);
+		}
+		else
+		{
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				'output->[0]' => $output->[0],
+				'output->[1]' => $output->[1],
+			}});
+			if ($output->[0] eq "source file not found")
+			{
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { 
+					method      => "move_file",
+					source_file => $source_file,
+				}});
+				return(1);
+			}
+			if (($output->[0] eq "source file exists") && (not $overwrite))
+			{
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => {
+					method      => "move_file",
+					source_file => $source_file,
+					target_file => $target_file,
+				}});
+				return(1);
+			}
+			if ($output->[1] eq "target directory not found")
+			{
+				my $failed = $anvil->Storage->make_directory({
+					debug       => $debug,
+					directory   => $directory,
+					password    => $password, 
+					remote_user => $remote_user, 
+					target      => $target,
+				});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }});
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => {
+					method      => "move_file",
+					source_file => $source_file,
+					target_file => $target_file,
+				}});
+				return(1);
+			}
+		
+			# Now backup the file.
+			my ($error, $output) = $anvil->Remote->call({
+				debug       => $debug, 
+				target      => $target,
+				user        => $remote_user, 
+				password    => $password,
+				remote_user => $remote_user, 
+				shell_call  => $anvil->data->{path}{exe}{mv}." -f ".$source_file." ".$target_file,
+			});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output }});
+		}
+	}
+	else
+	{
+		# Copying locally
+		if (not -e $source_file)
+		{
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { 
+				method      => "move_file",
+				source_file => $source_file,
+			}});
+			return(1);
+		}
+		
+		# If the target exists, abort
+		if ((-e $target_file) && (not $overwrite))
+		{
+			# This isn't an error.
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => {
+				method      => "move_file",
+				source_file => $source_file,
+				target_file => $target_file,
+			}});
+			return(1);
+		}
+		
+		# Make sure the target directory exists and create it, if not.
+		if (not -e $directory)
+		{
+			my $failed = $anvil->Storage->make_directory({
+				debug     => $debug,
+				directory => $directory,
+			});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }});
+			if ($failed)
+			{
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => {
+					method      => "move_file",
+					source_file => $source_file,
+					target_file => $target_file,
+				}});
+				return(1);
+			}
+		}
+		
+		# Now backup the file.
+		my $output = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{'mv'}." -f ".$source_file." ".$target_file});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output }});
+	}
+	
+	return(0);
+}
+
 =head2 read_config
 
 This method is used to read 'Anvil::Tools' style configuration files. These configuration files are in the format:
@@ -1783,7 +2037,13 @@ sub scan_directory
 		next if $file eq "..";
 		my $full_path                                            =  $directory."/".$file;
 		   $full_path                                            =~ s/\/\//\//g; 
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { full_path => $full_path }});
+		$anvil->data->{scan}{directories}{$full_path}{directory} =  $directory;
+		$anvil->data->{scan}{directories}{$full_path}{name}      =  $file;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			"scan::directories::${full_path}::directory" => $anvil->data->{scan}{directories}{$full_path}{directory}, 
+			"scan::directories::${full_path}::name"      => $anvil->data->{scan}{directories}{$full_path}{name}, 
+			full_path                                    => $full_path,
+		}});
 		if ((-d $full_path) && ($recursive))
 		{
 			# This is a directory, dive into it is asked.

share/anvil.sql

@@ -1181,8 +1181,8 @@ CREATE TRIGGER trigger_files
 -- This tracks which files should be on which machines.
 CREATE TABLE file_locations (
     file_location_uuid         uuid                        not null    primary key,
-    file_location_file_uuid    text                        not null,                   -- This is file to be moved to (or restored to) this machine.
+    file_location_file_uuid    uuid                        not null,                   -- This is file to be moved to (or restored to) this machine.
-    file_location_host_uuid    text                        not null,                   -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change.
+    file_location_host_uuid    uuid                        not null,                   -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change.
     modified_date              timestamp with time zone    not null,
     
     FOREIGN KEY(file_location_file_uuid) REFERENCES files(file_uuid), 

share/words.xml

@@ -299,13 +299,13 @@ It also has replacement variables: [#!variable!first!#] and [#!variable!second!#
 		<key name="log_0043"><![CDATA[[ Error ] - The method System->call() was called but 'shell_call' was not passed or was empty.]]></key>
 		<key name="log_0044">The host: [#!variable!host!#] has renewed its database lock.</key>
 		<key name="log_0045">The host: [#!variable!host!#] is requesting a database lock.</key>
-		<key name="log_0046"><![CDATA[[ Note ] - The method Storage->copy_file() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target already exists and 'overwrite' wasn't specified, skipping.]]></key>
+		<key name="log_0046"><![CDATA[[ Note ] - The method Storage->#!variable!method!#() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target already exists and 'overwrite' wasn't specified, skipping.]]></key>
 		<key name="log_0047"><![CDATA[[ Error ] - The method Log->level() was passed an invalid log level: [#!variable!set!#]. Only '0', '1', '2', '3' or '4' are valid.]]></key>
 		<key name="log_0048"><![CDATA[[ Warning ] - Testing of Anvil::Tools is beginning. This will generate warnings and alerts and are not a concern.]]></key>
 		<key name="log_0049"><![CDATA[[ Warning ] - Testing of Anvil::Tools is complete.]]></key>
 		<key name="log_0050">[ Error ] - There is a local database defined, but it does not appear to exist and we could not initialize the database server. Is 'postgresql-server' installed?</key>
 		<key name="log_0051"><![CDATA[[ Error ] - The method Storage->change_owner() was asked to change the ownership of: [#!variable!target!#] which doesn't exist.]]></key>
-		<key name="log_0052"><![CDATA[[ Error ] - The method Storage->copy_file() was called but the source file: [#!variable!source_file!#] doesn't exist.]]></key>
+		<key name="log_0052"><![CDATA[[ Error ] - The method Storage->#!variable!method!#() was called but the source file: [#!variable!source_file!#] doesn't exist.]]></key>
 		<key name="log_0053"><![CDATA[[ Error ] - The 'Database->connect()' method tried to connect to the same database twice: [#!variable!target!#].]]></key>
 		<key name="log_0054">
 Connecting to Database with configuration ID: [#!variable!uuid!#]
@@ -450,7 +450,7 @@ The database connection error was:
 		<key name="log_0167">Failed to create the directory: [#!variable!directory!#] on: [#!variable!target!#] as: [#!variable!remote_user!#]. The error (if any) was: [#!variable!error!#] and the output (if any) was: [#!variable!output!#].</key>
 		<key name="log_0168">Failed to create the directory: [#!variable!directory!#]. The error (if any) was: [#!variable!error!#].</key>
 		<key name="log_0169">Failed to copy the file: [#!variable!source_file!#] to: [#!variable!target_file!#] on the target: [#!variable!target!#] as: [#!variable!remote_user!#]. The error (if any) was: [#!variable!error!#] and the output (if any) was: [#!variable!output!#].</key>
-		<key name="log_0170"><![CDATA[[ Note ] - The method Storage->copy_file() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target's parent directory doesn't exist and we were unable to create it.]]></key>
+		<key name="log_0170"><![CDATA[[ Note ] - The method Storage->#!variable!method!#() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target's parent directory doesn't exist and we were unable to create it.]]></key>
 		<key name="log_0171"><![CDATA[[ Error ] - The method: Account->encrypt_password() tried to use the algorithm: [#!variable!algorithm!#], which is not recognized. Only 'sha256', 'sha384' and 'sha512' are currently supported. The desired algorithm can be set via 'sys::password::algorithm'.]]></key>
 		<key name="log_0172"><![CDATA[[ Error ] - Asked to validate a password for the user: [#!variable!user!#], but that user wasn't found.]]></key>
 		<key name="log_0173"><![CDATA[[ Error ] - Asked to valudate a password encoded with the algorithm: [#!variable!user_algorithm!#], which is not recognized. Only 'sha256', 'sha384' and 'sha512' are currently supported.]]></key>
@@ -558,6 +558,11 @@ Finished Downloading: [#!variable!file!#].
 		<key name="log_0261">#!variable!file!# was called, but no files where available for download in CGI. Was the variable name 'upload_file' used?</key>
 		<key name="log_0262">[ Error ] - Storage->scan_directory() was asked to scan: [#!variable!directory!#], but it doesn't exist or isn't actually a directory.</key>
 		<key name="log_0263">Now deleting the file: [#!variable!file!#].</key>
+		<key name="log_0264">Checking: [#!data!path::directories::shared::incoming!#] for new files.</key>
+		<key name="log_0265">About to calculate the md5sum for the file: [#!variable!file!#].</key>
+		<key name="log_0266">This file is large, [#!variable!size!#], this might take a bit of time...</key>
+		<key name="log_0267">Failed to move the file: [#!variable!source_file!#] to: [#!variable!target_file!#] on the target: [#!variable!target!#] as: [#!variable!remote_user!#]. The error (if any) was: [#!variable!error!#] and the output (if any) was: [#!variable!output!#].</key>
+		<key name="log_0268">The file: [#!variable!file!#] has been added to the database (if needed) moved to: [#!variable!target!#].</key>
 		
 		<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
 		<key name="t_0000">Test</key>
@@ -813,11 +818,11 @@ The update appears to have not completed successfully. The output was:
 		<key name="error_0049">A request to rename a file was made, but no file name was given.</key>
 		<key name="error_0050">A request to rename the file: [#!variable!file!#] was made, but the new name wasn't given. Was '--to X' given?</key>
 		<key name="error_0051">A request to rename the file: [#!variable!file!#] was made, but that file doesn't exist.</key>
-		<key name="error_0051">A request to rename the file: [#!variable!file!#] to: [#!variable!to!#], but there is an existing file or directory with that name.</key>
 		<key name="error_0052">A request to delete a file was made, but no file name was given.</key>
 		<key name="error_0053">A request to delete the file: [#!variable!file!#] was received, but it is not under '/mnt/shared/'. This program can only work on or under that directory.</key>
 		<key name="error_0054">Failed!</key>
 		<key name="error_0055">A request to toggle the script flag was received, but no file name was given.</key>
+		<key name="error_0056">A request to rename the file: [#!variable!file!#] to: [#!variable!to!#], but there is an existing file or directory with that name.</key>
 		
 		<!-- These are units, words and so on used when displaying information. -->
 		<key name="unit_0001">Yes</key>

tools/anvil-manage-files

@@ -52,6 +52,8 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
 }
 
 my $anvil = Anvil::Tools->new();
+$anvil->Log->level({set => 2});
+$anvil->Log->secure({set => 1});
 
 $anvil->data->{switches}{'job-uuid'}  = "";
 $anvil->data->{switches}{'rename'}    = "";
@@ -61,7 +63,7 @@ $anvil->data->{switches}{to}          = "";
 $anvil->data->{switches}{'delete'}    = "";
 $anvil->data->{switches}{everywhere}  = "";
 $anvil->Get->switches;
-$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 	"switches::job-uuid"   => $anvil->data->{switches}{'job-uuid'},
 	"switches::rename"     => $anvil->data->{switches}{'rename'},
 	"switches::is-script"  => $anvil->data->{switches}{'is-script'},
@@ -115,15 +117,22 @@ if ($anvil->data->{switches}{'rename'})
 {
 	handle_rename($anvil);
 }
-elsif ($anvil->data->{switches}{'delete'}))
+elsif ($anvil->data->{switches}{'delete'})
 {
 	handle_delete($anvil);
 }
-elsif ($anvil->data->{switches}{'is-script'}))
+elsif ($anvil->data->{switches}{'is-script'})
 {
 	handle_script($anvil);
 }
+else
+{
+	# Check for new files
+	check_incoming($anvil);
 	
+	# Check for files we should have but don't yet have.
+	find_missing_files($anvil);
+}
 
 # We're done
 $anvil->nice_exit({exit_code => 0});
@@ -133,6 +142,157 @@ $anvil->nice_exit({exit_code => 0});
 # Private functions.                                                                                        #
 #############################################################################################################
 
+# This looks to see if there are any entries in 'file_locations' for us that, the pointed to file, doesn't 
+# exist on this machine. For those entries, we will search for the file on other machines. The one exception
+# is server definition files. For those, we write the file out from the server's 'definition' table entry.
+# 
+# When a missing entry is found, and an entry doesn't exist, the file will be found (if possible) and copied
+# to this houst. Only machines on the same subnet are searched. The search pattern is; 
+# 
+# Nodes;   1. Check for the file on the peer.
+#          2. Check for the file on Strikers, in alphabetical order.
+#          3. Check for the file on DR host, if available.
+#          4. Check other nodes, in alphabetical order.
+#          5. Check other DR hosts, in alphabetical order.
+# Striker; 1. Check for the file on other Strikers, in alphabetical order.
+#          2. Check for the file on DR hosts, if available
+#          3. Check for the file on Anvil! nodes.
+# DR Host; 1. Check for the file on Strikers, in alphabetical order.
+#          2. Check for the file on Anvil! nodes.
+# * If a file can't be found, it will try again every so often until it is found.
+# * When a file is found, it is copied to '/mnt/shared/incoming'. Only when the file has arrived and the 
+#   md5sum matches. At this point, it is moved into the proper directory.
+sub find_missing_files
+{
+	my ($anvil) = @_;
+	
+	# What am I? This will impact how missing files are found.
+	my $query = "
+SELECT 
+    file_location_file_uuid 
+FROM 
+    file_locations 
+WHERE 
+    file_location_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{host_uuid})." 
+;";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
+		
+		my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
+		my $count   = @{$results};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			results => $results, 
+			count   => $count, 
+		}});
+		foreach my $row (@{$results})
+		{
+			my $file_location_file_uuid = $row->[0];
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				file_location_file_uuid => $file_location_file_uuid, 
+			}});
+			
+			### TODO: How to handle when the file with the same name exists on 2+ machines with 
+			###       different md5sums. Use the most recent mtime?
+			# Read in the file details.
+		}
+	
+	# Read in any entries from 'file_locations'.
+	
+	return(0);
+}
+
+# This handles deleting a file. 
+sub check_incoming
+{
+	my ($anvil) = @_;
+	
+	# This hash shouldn't exist, but lets be safe...
+	if (exists $anvil->data->{scan}{directories})
+	{
+		delete $anvil->data->{scan}{directories};
+	}
+	# Read any files in '/mnt/shared/incoming'.
+	$anvil->Storage->scan_directory({
+		debug     => 3,
+		directory => $anvil->data->{path}{directories}{shared}{incoming},
+		recursive => 0,
+	});
+	
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0264"});
+	foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}})
+	{
+		# Is this a file?
+		my $file_name       = $anvil->data->{scan}{directories}{$full_path}{name};
+		my $file_type       = $anvil->data->{scan}{directories}{$full_path}{type};
+		my $file_size       = $anvil->data->{scan}{directories}{$full_path}{size};
+		my $file_mimetype   = $anvil->data->{scan}{directories}{$full_path}{mimetype};
+		my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			full_path       => $full_path,
+			file_name       => $file_name, 
+			file_type       => $file_type, 
+			file_size       => $file_size, 
+			file_mimetype   => $file_mimetype, 
+			file_executable => $file_executable, 
+		}});
+		next if $file_type ne "file";
+		
+		# If this file is over 128 MiB, warn the user that it might take a second
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0265", variables => { file => $full_path }});
+		if ($file_size > (128 * (2 ** 20)))
+		{
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0266", variables => { 
+				size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
+			}});
+		}
+		my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable);
+		my $file_md5sum  = $anvil->Get->md5sum({debug => 2, file => $full_path});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			say_mimetype => $say_mimetype, 
+			file_md5sum  => $file_md5sum,
+		}});
+		
+		# Do we know about this file? If not, file_uuid will be blank when we call the 
+		# insert_or_update. If we do, it will update the file name, if needed.
+		my ($file_uuid) = $anvil->Database->insert_or_update_files({
+			debug       => 2,
+			file_uuid   => get_file_uuid($anvil, $file_md5sum, $file_name), 
+			file_name   => $file_name, 
+			file_size   => $file_size, 
+			file_md5sum => $file_md5sum, 
+			file_type   => $say_mimetype, 
+		});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
+		
+		# Make sure we know about this file on this system
+		my ($file_locatiom_uuid) = $anvil->Database->insert_or_update_file_locations({
+			debug                   => 2,
+			file_location_file_uuid => $file_uuid, 
+			file_location_host_uuid => $anvil->data->{sys}{host_uuid},
+		});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_locatiom_uuid => $file_locatiom_uuid }});
+		
+		# Not move it. If it's a definition file, we'll move it to 
+		# 'path::directories::shared::definitions', otherwise we'll move it to 
+		# 'path::directories::shared::files'.
+		my $target =  $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files};
+		   $target .= "/";
+		   $target =~ s/\/\//\//g;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }});
+		
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0268", variables => { 
+			file   => $full_path,
+			target => $target, 
+		}});
+		$anvil->Storage->move_file({
+			debug       => 2,
+			source_file => $full_path, 
+			target_file => $target,
+		});
+	}
+	
+	return(0);
+}
+
 # This gets the file_uuid for a given file name and/or md5sum. If the file isn't found, an empty string is
 # returned.
 sub get_file_uuid
@@ -191,7 +351,6 @@ sub handle_script
 	{
 		# Um...
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0055"});
-		if ($anvil->data->{jobs}{'job-uuid'})
 		$anvil->Job->update_progress({
 			progress => 0, 
 			message  => "error_0055", 
@@ -205,12 +364,14 @@ sub handle_script
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
 	
 	# Toggle the executable bits.
+	my $executable = 0;
 	if ($anvil->data->{switches}{'is-script'})
 	{
 		# Is it already executable?
 		if (-x $anvil->data->{switches}{file})
 		{
 			# Switch it on
+			$executable = 1;
 			$anvil->Storage->change_mode({target => $anvil->data->{switches}{file}, mode => "a+x"});
 		}
 		else
@@ -224,6 +385,7 @@ sub handle_script
 		if (-x $anvil->data->{switches}{file})
 		{
 			# Switch it off.
+			$executable = 1;
 			$anvil->Storage->change_mode({target => $anvil->data->{switches}{file}, mode => "a-x"});
 		}
 		else
@@ -249,7 +411,7 @@ WHERE
 ;";
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
 		
-		my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
+		my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results }});
 		my $file_name   = $results->[0]->[0];
 		my $file_size   = $results->[0]->[1];
@@ -264,22 +426,10 @@ WHERE
 		if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'}))
 		{
 			# Figure out what the file type is and update.
-			my $mimetype = mimetype($full_path);
+			my $mimetype = mimetype($file_name);
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mimetype => $mimetype }});
 			
-			my $say_mimetype = "other";
+			my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, $executable);
-			if ($mimetype =~ /cd-image/)
-			{
-				$say_mimetype = "iso";
-			}
-			elsif ($mimetype =~ /rpm$/)
-			{
-				$say_mimetype = "rpm";
-			}
-			elsif ($mimetype =~ /disk-image/)
-			{
-				$say_mimetype = "disk-image";
-			}
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }});
 			$anvil->Database->insert_or_update_files({
 				file_uuid   => $file_uuid, 
@@ -288,11 +438,22 @@ WHERE
 				file_md5sum => $file_md5sum, 
 				file_type   => $say_mimetype, 
 			});
+			
+			# Change the file tpye to "say_mimetype".
+			$anvil->Database->insert_or_update_files({
+				debug       => 2,
+				file_uuid   => $file_uuid, 
+				file_name   => $anvil->data->{switches}{file}, 
+				file_size   => $file_size, 
+				file_md5sum => $file_md5sum, 
+				file_type   => $say_mimetype, 
+			});
 		}
-		elsif ($file_type ne "script") && ($anvil->data->{switches}{'is-script'}))
+		elsif (($file_type ne "script") && ($anvil->data->{switches}{'is-script'}))
 		{
 			# Change the file tpye to "script"
 			$anvil->Database->insert_or_update_files({
+				debug       => 2,
 				file_uuid   => $file_uuid, 
 				file_name   => $anvil->data->{switches}{file}, 
 				file_size   => $file_size, 
@@ -305,6 +466,55 @@ WHERE
 	return(0);
 }
 
+# This takes the mimetype as reported by the 'mimetype' method and returns a file type we care about.
+sub convert_mimetype
+{
+	my ($anvil, $mimetype, $file, $executable) = @_;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		mimetype   => $mimetype,
+		file       => $file,
+		executable => $executable, 
+	}});
+	
+	my $say_mimetype = "other";
+	if ($mimetype)
+	{
+		if ($mimetype =~ /cd-image/)
+		{
+			$say_mimetype = "iso";
+		}
+		elsif ($mimetype =~ /xml/)
+		{
+			# This might be a definition, but look inside it to be sure.
+			my $is_domain = $anvil->System->call({
+				debug      => 2, 
+				shell_call => "if \$(".$anvil->data->{path}{exe}{'grep'}." -q '</domain>' ".$file."); then ".$anvil->data->{path}{exe}{echo}." 1; else ".$anvil->data->{path}{exe}{echo}." 0; fi", 
+			});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { is_domain => $is_domain }});
+
+			if ($is_domain)
+			{
+				$say_mimetype = "definition";
+			}
+		}
+		elsif ($mimetype =~ /rpm$/)
+		{
+			$say_mimetype = "rpm";
+		}
+		elsif ($mimetype =~ /disk-image/)
+		{
+			$say_mimetype = "disk-image";
+		}
+		elsif ($executable)
+		{
+			$say_mimetype = "script";
+		}
+	}
+	
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }});
+	return($say_mimetype);
+}
+
 # This handles deleting a file. If the requested deletion target doesn't exist, we'll just clear the 
 # database. If that doesn't exist either, we still don't error. This is to handle broad requests to delete a
 # file everywhere. If we're asked to delete it everywhere, then we'll register a job against all hosts.
@@ -316,7 +526,6 @@ sub handle_delete
 	{
 		# Um...
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"});
-		if ($anvil->data->{jobs}{'job-uuid'})
 		$anvil->Job->update_progress({
 			progress => 0, 
 			message  => "error_0052", 
@@ -403,10 +612,10 @@ sub handle_rename
 	elsif (-e $anvil->data->{switches}{to})
 	{
 		# There's already a file (or directory or something) with that name.
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052", variables => { file => $anvil->data->{switches}{file}, to => $anvil->data->{switches}{to} }});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0056", variables => { file => $anvil->data->{switches}{file}, to => $anvil->data->{switches}{to} }});
 		$anvil->Job->update_progress({
 			progress => 0, 
-			message  => "error_0052,!!file!".$anvil->data->{switches}{file}."!!,!!to!".$anvil->data->{switches}{to}."!!", 
+			message  => "error_0056,!!file!".$anvil->data->{switches}{file}."!!,!!to!".$anvil->data->{switches}{to}."!!", 
 			job_uuid => $anvil->data->{jobs}{'job-uuid'},
 		});
 		$anvil->nice_exit({exit_code => 6});