diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm
index 76d5193b..65ccf372 100644
--- a/Anvil/Tools.pm
+++ b/Anvil/Tools.pm
@@ -1002,6 +1002,7 @@ sub _set_paths
logger => "/usr/bin/logger",
md5sum => "/usr/bin/md5sum",
'mkdir' => "/usr/bin/mkdir",
+ mv => "/usr/bin/mv",
nmcli => "/bin/nmcli",
openssl => "/usr/bin/openssl",
passwd => "/usr/bin/passwd",
diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm
index 7466ee25..245f883d 100644
--- a/Anvil/Tools/Database.pm
+++ b/Anvil/Tools/Database.pm
@@ -2211,6 +2211,8 @@ sub insert_or_update_file_locations
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_file_locations()" }});
+ my $file = defined $parameter->{file} ? $parameter->{file} : "";
+ my $line = defined $parameter->{line} ? $parameter->{line} : "";
my $file_location_uuid = defined $parameter->{file_location_uuid} ? $parameter->{file_location_uuid} : "";
my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : "";
my $file_location_host_uuid = defined $parameter->{file_location_host_uuid} ? $parameter->{file_location_host_uuid} : $anvil->data->{sys}{host_uuid};
@@ -2301,15 +2303,11 @@ INSERT INTO
file_location_uuid,
file_location_file_uuid,
file_location_host_uuid,
- file_md5sum,
- file_type,
modified_date
) VALUES (
".$anvil->data->{sys}{database}{use_handle}->quote($file_location_uuid).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_location_file_uuid).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_location_host_uuid).",
- ".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).",
- ".$anvil->data->{sys}{database}{use_handle}->quote($file_type).",
".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})."
);
";
@@ -2322,7 +2320,7 @@ INSERT INTO
my $query = "
SELECT
file_location_file_uuid,
- file_location_host_uuid,
+ file_location_host_uuid
FROM
file_locations
WHERE
@@ -2412,6 +2410,8 @@ sub insert_or_update_files
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_files()" }});
+ my $file = defined $parameter->{file} ? $parameter->{file} : "";
+ my $line = defined $parameter->{line} ? $parameter->{line} : "";
my $file_uuid = defined $parameter->{file_uuid} ? $parameter->{file_uuid} : "";
my $file_name = defined $parameter->{file_name} ? $parameter->{file_name} : "";
my $file_size = defined $parameter->{file_size} ? $parameter->{file_size} : "";
diff --git a/Anvil/Tools/Get.pm b/Anvil/Tools/Get.pm
index ebb922fa..c50d22d3 100644
--- a/Anvil/Tools/Get.pm
+++ b/Anvil/Tools/Get.pm
@@ -616,13 +616,14 @@ sub md5sum
my $sum = "";
my $file = defined $parameter->{file} ? $parameter->{file} : "";
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file => $file }});
if (-e $file)
{
my $shell_call = $anvil->data->{path}{exe}{md5sum}." ".$file;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
- my $return = $anvil->System->call({shell_call => $shell_call});
+ my $return = $anvil->System->call({debug => $debug, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'return' => $return }});
# split the sum off.
diff --git a/Anvil/Tools/Storage.pm b/Anvil/Tools/Storage.pm
index 259de959..45fe5172 100644
--- a/Anvil/Tools/Storage.pm
+++ b/Anvil/Tools/Storage.pm
@@ -22,6 +22,7 @@ my $THIS_FILE = "Storage.pm";
# copy_file
# find
# make_directory
+# move_file
# read_config
# read_file
# read_mode
@@ -710,12 +711,16 @@ fi";
}});
if ($output->[0] eq "source file not found")
{
- $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { source_file => $source_file }});
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => {
+ method => "copy_file",
+ source_file => $source_file,
+ }});
return(1);
}
if (($output->[0] eq "source file exists") && (not $overwrite))
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => {
+ method => "copy_file",
source_file => $source_file,
target_file => $target_file,
}});
@@ -732,6 +737,7 @@ fi";
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => {
+ method => "copy_file",
source_file => $source_file,
target_file => $target_file,
}});
@@ -755,7 +761,10 @@ fi";
# Copying locally
if (not -e $source_file)
{
- $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { source_file => $source_file }});
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => {
+ method => "copy_file",
+ source_file => $source_file,
+ }});
return(1);
}
@@ -764,6 +773,7 @@ fi";
{
# This isn't an error.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => {
+ method => "copy_file",
source_file => $source_file,
target_file => $target_file,
}});
@@ -781,6 +791,7 @@ fi";
if ($failed)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => {
+ method => "copy_file",
source_file => $source_file,
target_file => $target_file,
}});
@@ -1070,6 +1081,249 @@ fi;";
return($failed);
}
+=head2 move_file
+
+This moves a file, with a few additional checks like creating the target directory if it doesn't exist, aborting if the file already exists in the target, etc. It can move files on the local or a remote machine.
+
+As with the system copy, the target can be a directory (denoted with an ending c<< / >>), or a it can be renamed in the process (but not ending with C<< / >>).
+
+ # Example moving
+ $anvil->Storage->move_file({source_file => "/some/file", target_file => "/another/directory/"});
+
+ # Example moving with a rename at the same time
+ $anvil->Storage->move_file({source_file => "/some/file", target_file => "/another/directory/new_name"});
+
+Returns C<< 0 >> on success, otherwise C<< 1 >>.
+
+Parameters;
+
+=head3 overwrite (optional)
+
+If this is set to 'C<< 1 >>', and if the target file exists, it will be replaced.
+
+If this is not passed and the target exists, this module will return 'C<< 3 >>'.
+
+=head3 port (optional, default 22)
+
+If C<< target >> is set, this is the TCP port number used to connect to the remote machine.
+
+=head3 password (optional)
+
+If C<< target >> is set, this is the password used to log into the remote system as the C<< remote_user >>. If it is not set, an attempt to connect without a password will be made (though this will usually fail).
+
+=head3 source_file (required)
+
+This is the source file. If it isn't specified, 'C<< 1 >>' will be returned. If it doesn't exist, this method will return 'C<< 4 >>'.
+
+=head3 target (optional)
+
+If set, the file will be copied on the target machine. This must be either an IP address or a resolvable host name.
+
+=head3 target_file (required)
+
+This is the target B<< file >>, not the directory to put it in. The target file name can be different from the source file name.
+
+if this is not specified, 'C<< 2 >>' will be returned.
+
+=head3 remote_user (optional, default root)
+
+If C<< target >> is set, this is the user account that will be used when connecting to the remote system.
+
+=cut
+sub move_file
+{
+ my $self = shift;
+ my $parameter = shift;
+ my $anvil = $self->parent;
+ my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
+
+ my $overwrite = defined $parameter->{overwrite} ? $parameter->{overwrite} : 0;
+ my $password = defined $parameter->{password} ? $parameter->{password} : "";
+ my $remote_user = defined $parameter->{remote_user} ? $parameter->{remote_user} : "root";
+ my $source_file = defined $parameter->{source_file} ? $parameter->{source_file} : "";
+ my $target_file = defined $parameter->{target_file} ? $parameter->{target_file} : "";
+ my $target = defined $parameter->{target} ? $parameter->{target} : "";
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
+ overwrite => $overwrite,
+ password => $anvil->Log->secure ? $password : $anvil->Words->string({key => "log_0186"}),
+ remote_user => $remote_user,
+ source_file => $source_file,
+ target_file => $target_file,
+ target => $target,
+ }});
+
+ if (not $source_file)
+ {
+ # No source passed.
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Storage->move_file()", parameter => "source_file" }});
+ return(1);
+ }
+ if (not $target_file)
+ {
+ # No target passed.
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Storage->move_file()", parameter => "target_file" }});
+ return(2);
+ }
+
+ # If we have a target directory, pull the file name off the source for the target checks.
+ my ($directory, $file) = ($target_file =~ /^(.*)\/(.*?)$/);
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
+ directory => $directory,
+ file => $file,
+ }});
+ if (not $file)
+ {
+ ($file) = ($source_file =~ /^.*\/(.*?)$/);
+ $target_file .= $file;
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
+ file => $file,
+ target_file => $target_file,
+ }});
+ }
+
+ if ($target)
+ {
+ # Copying on a remote system.
+ my $proceed = 1;
+ my $shell_call = "
+if [ -e '".$source_file."' ];
+ ".$anvil->data->{path}{exe}{echo}." 'source file exists'
+else
+ ".$anvil->data->{path}{exe}{echo}." 'source file not found'
+fi
+if [ -d '".$target_file."' ];
+ ".$anvil->data->{path}{exe}{echo}." 'target file exists'
+elif [ -d '".$directory."' ];
+ ".$anvil->data->{path}{exe}{echo}." 'target directory exists'
+else
+ ".$anvil->data->{path}{exe}{echo}." 'target directory not found'
+fi";
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0166", variables => { shell_call => $shell_call, target => $target, remote_user => $remote_user }});
+ my ($error, $output) = $anvil->Remote->call({
+ debug => $debug,
+ target => $target,
+ user => $remote_user,
+ password => $password,
+ remote_user => $remote_user,
+ shell_call => $shell_call,
+ });
+ if ($error)
+ {
+ # Something went wrong.
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0267", variables => {
+ source_file => $source_file,
+ target_file => $target_file,
+ error => $error,
+ output => $output,
+ target => $target,
+ remote_user => $remote_user,
+ }});
+ return(1);
+ }
+ else
+ {
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
+ 'output->[0]' => $output->[0],
+ 'output->[1]' => $output->[1],
+ }});
+ if ($output->[0] eq "source file not found")
+ {
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => {
+ method => "move_file",
+ source_file => $source_file,
+ }});
+ return(1);
+ }
+ if (($output->[0] eq "source file exists") && (not $overwrite))
+ {
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => {
+ method => "move_file",
+ source_file => $source_file,
+ target_file => $target_file,
+ }});
+ return(1);
+ }
+ if ($output->[1] eq "target directory not found")
+ {
+ my $failed = $anvil->Storage->make_directory({
+ debug => $debug,
+ directory => $directory,
+ password => $password,
+ remote_user => $remote_user,
+ target => $target,
+ });
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }});
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => {
+ method => "move_file",
+ source_file => $source_file,
+ target_file => $target_file,
+ }});
+ return(1);
+ }
+
+ # Now backup the file.
+ my ($error, $output) = $anvil->Remote->call({
+ debug => $debug,
+ target => $target,
+ user => $remote_user,
+ password => $password,
+ remote_user => $remote_user,
+ shell_call => $anvil->data->{path}{exe}{mv}." -f ".$source_file." ".$target_file,
+ });
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output }});
+ }
+ }
+ else
+ {
+ # Copying locally
+ if (not -e $source_file)
+ {
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => {
+ method => "move_file",
+ source_file => $source_file,
+ }});
+ return(1);
+ }
+
+ # If the target exists, abort
+ if ((-e $target_file) && (not $overwrite))
+ {
+ # This isn't an error.
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => {
+ method => "move_file",
+ source_file => $source_file,
+ target_file => $target_file,
+ }});
+ return(1);
+ }
+
+ # Make sure the target directory exists and create it, if not.
+ if (not -e $directory)
+ {
+ my $failed = $anvil->Storage->make_directory({
+ debug => $debug,
+ directory => $directory,
+ });
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }});
+ if ($failed)
+ {
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => {
+ method => "move_file",
+ source_file => $source_file,
+ target_file => $target_file,
+ }});
+ return(1);
+ }
+ }
+
+ # Now backup the file.
+ my $output = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{'mv'}." -f ".$source_file." ".$target_file});
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output }});
+ }
+
+ return(0);
+}
+
=head2 read_config
This method is used to read 'Anvil::Tools' style configuration files. These configuration files are in the format:
@@ -1781,9 +2035,15 @@ sub scan_directory
{
next if $file eq ".";
next if $file eq "..";
- my $full_path = $directory."/".$file;
- $full_path =~ s/\/\//\//g;
- $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { full_path => $full_path }});
+ my $full_path = $directory."/".$file;
+ $full_path =~ s/\/\//\//g;
+ $anvil->data->{scan}{directories}{$full_path}{directory} = $directory;
+ $anvil->data->{scan}{directories}{$full_path}{name} = $file;
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
+ "scan::directories::${full_path}::directory" => $anvil->data->{scan}{directories}{$full_path}{directory},
+ "scan::directories::${full_path}::name" => $anvil->data->{scan}{directories}{$full_path}{name},
+ full_path => $full_path,
+ }});
if ((-d $full_path) && ($recursive))
{
# This is a directory, dive into it is asked.
diff --git a/share/anvil.sql b/share/anvil.sql
index a9a724e1..cd5f2dce 100644
--- a/share/anvil.sql
+++ b/share/anvil.sql
@@ -1181,8 +1181,8 @@ CREATE TRIGGER trigger_files
-- This tracks which files should be on which machines.
CREATE TABLE file_locations (
file_location_uuid uuid not null primary key,
- file_location_file_uuid text not null, -- This is file to be moved to (or restored to) this machine.
- file_location_host_uuid text not null, -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change.
+ file_location_file_uuid uuid not null, -- This is file to be moved to (or restored to) this machine.
+ file_location_host_uuid uuid not null, -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change.
modified_date timestamp with time zone not null,
FOREIGN KEY(file_location_file_uuid) REFERENCES files(file_uuid),
diff --git a/share/words.xml b/share/words.xml
index 2558aea7..fd1bd3e4 100644
--- a/share/words.xml
+++ b/share/words.xml
@@ -299,13 +299,13 @@ It also has replacement variables: [#!variable!first!#] and [#!variable!second!#
call() was called but 'shell_call' was not passed or was empty.]]>
The host: [#!variable!host!#] has renewed its database lock.
The host: [#!variable!host!#] is requesting a database lock.
- copy_file() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target already exists and 'overwrite' wasn't specified, skipping.]]>
+ #!variable!method!#() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target already exists and 'overwrite' wasn't specified, skipping.]]>
level() was passed an invalid log level: [#!variable!set!#]. Only '0', '1', '2', '3' or '4' are valid.]]>
[ Error ] - There is a local database defined, but it does not appear to exist and we could not initialize the database server. Is 'postgresql-server' installed?
change_owner() was asked to change the ownership of: [#!variable!target!#] which doesn't exist.]]>
- copy_file() was called but the source file: [#!variable!source_file!#] doesn't exist.]]>
+ #!variable!method!#() was called but the source file: [#!variable!source_file!#] doesn't exist.]]>
connect()' method tried to connect to the same database twice: [#!variable!target!#].]]>
Connecting to Database with configuration ID: [#!variable!uuid!#]
@@ -450,7 +450,7 @@ The database connection error was:
Failed to create the directory: [#!variable!directory!#] on: [#!variable!target!#] as: [#!variable!remote_user!#]. The error (if any) was: [#!variable!error!#] and the output (if any) was: [#!variable!output!#].
Failed to create the directory: [#!variable!directory!#]. The error (if any) was: [#!variable!error!#].
Failed to copy the file: [#!variable!source_file!#] to: [#!variable!target_file!#] on the target: [#!variable!target!#] as: [#!variable!remote_user!#]. The error (if any) was: [#!variable!error!#] and the output (if any) was: [#!variable!output!#].
- copy_file() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target's parent directory doesn't exist and we were unable to create it.]]>
+ #!variable!method!#() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target's parent directory doesn't exist and we were unable to create it.]]>
encrypt_password() tried to use the algorithm: [#!variable!algorithm!#], which is not recognized. Only 'sha256', 'sha384' and 'sha512' are currently supported. The desired algorithm can be set via 'sys::password::algorithm'.]]>
@@ -558,6 +558,11 @@ Finished Downloading: [#!variable!file!#].
#!variable!file!# was called, but no files where available for download in CGI. Was the variable name 'upload_file' used?
[ Error ] - Storage->scan_directory() was asked to scan: [#!variable!directory!#], but it doesn't exist or isn't actually a directory.
Now deleting the file: [#!variable!file!#].
+ Checking: [#!data!path::directories::shared::incoming!#] for new files.
+ About to calculate the md5sum for the file: [#!variable!file!#].
+ This file is large, [#!variable!size!#], this might take a bit of time...
+ Failed to move the file: [#!variable!source_file!#] to: [#!variable!target_file!#] on the target: [#!variable!target!#] as: [#!variable!remote_user!#]. The error (if any) was: [#!variable!error!#] and the output (if any) was: [#!variable!output!#].
+ The file: [#!variable!file!#] has been added to the database (if needed) moved to: [#!variable!target!#].
Test
@@ -813,11 +818,11 @@ The update appears to have not completed successfully. The output was:
A request to rename a file was made, but no file name was given.
A request to rename the file: [#!variable!file!#] was made, but the new name wasn't given. Was '--to X' given?
A request to rename the file: [#!variable!file!#] was made, but that file doesn't exist.
- A request to rename the file: [#!variable!file!#] to: [#!variable!to!#], but there is an existing file or directory with that name.
A request to delete a file was made, but no file name was given.
A request to delete the file: [#!variable!file!#] was received, but it is not under '/mnt/shared/'. This program can only work on or under that directory.
Failed!
A request to toggle the script flag was received, but no file name was given.
+ A request to rename the file: [#!variable!file!#] to: [#!variable!to!#], but there is an existing file or directory with that name.
Yes
diff --git a/tools/anvil-manage-files b/tools/anvil-manage-files
index 6aa3a32c..1fef8b7a 100755
--- a/tools/anvil-manage-files
+++ b/tools/anvil-manage-files
@@ -52,6 +52,8 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
}
my $anvil = Anvil::Tools->new();
+$anvil->Log->level({set => 2});
+$anvil->Log->secure({set => 1});
$anvil->data->{switches}{'job-uuid'} = "";
$anvil->data->{switches}{'rename'} = "";
@@ -61,7 +63,7 @@ $anvil->data->{switches}{to} = "";
$anvil->data->{switches}{'delete'} = "";
$anvil->data->{switches}{everywhere} = "";
$anvil->Get->switches;
-$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"switches::job-uuid" => $anvil->data->{switches}{'job-uuid'},
"switches::rename" => $anvil->data->{switches}{'rename'},
"switches::is-script" => $anvil->data->{switches}{'is-script'},
@@ -115,15 +117,22 @@ if ($anvil->data->{switches}{'rename'})
{
handle_rename($anvil);
}
-elsif ($anvil->data->{switches}{'delete'}))
+elsif ($anvil->data->{switches}{'delete'})
{
handle_delete($anvil);
}
-elsif ($anvil->data->{switches}{'is-script'}))
+elsif ($anvil->data->{switches}{'is-script'})
{
handle_script($anvil);
}
-
+else
+{
+ # Check for new files
+ check_incoming($anvil);
+
+ # Check for files we should have but don't yet have.
+ find_missing_files($anvil);
+}
# We're done
$anvil->nice_exit({exit_code => 0});
@@ -133,6 +142,157 @@ $anvil->nice_exit({exit_code => 0});
# Private functions. #
#############################################################################################################
+# This looks to see if there are any entries in 'file_locations' for us that, the pointed to file, doesn't
+# exist on this machine. For those entries, we will search for the file on other machines. The one exception
+# is server definition files. For those, we write the file out from the server's 'definition' table entry.
+#
+# When a missing entry is found, and an entry doesn't exist, the file will be found (if possible) and copied
+# to this houst. Only machines on the same subnet are searched. The search pattern is;
+#
+# Nodes; 1. Check for the file on the peer.
+# 2. Check for the file on Strikers, in alphabetical order.
+# 3. Check for the file on DR host, if available.
+# 4. Check other nodes, in alphabetical order.
+# 5. Check other DR hosts, in alphabetical order.
+# Striker; 1. Check for the file on other Strikers, in alphabetical order.
+# 2. Check for the file on DR hosts, if available
+# 3. Check for the file on Anvil! nodes.
+# DR Host; 1. Check for the file on Strikers, in alphabetical order.
+# 2. Check for the file on Anvil! nodes.
+# * If a file can't be found, it will try again every so often until it is found.
+# * When a file is found, it is copied to '/mnt/shared/incoming'. Only when the file has arrived and the
+# md5sum matches. At this point, it is moved into the proper directory.
+sub find_missing_files
+{
+ my ($anvil) = @_;
+
+ # What am I? This will impact how missing files are found.
+ my $query = "
+SELECT
+ file_location_file_uuid
+FROM
+ file_locations
+WHERE
+ file_location_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{host_uuid})."
+;";
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
+
+ my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
+ my $count = @{$results};
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
+ results => $results,
+ count => $count,
+ }});
+ foreach my $row (@{$results})
+ {
+ my $file_location_file_uuid = $row->[0];
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
+ file_location_file_uuid => $file_location_file_uuid,
+ }});
+
+ ### TODO: How to handle when the file with the same name exists on 2+ machines with
+ ### different md5sums. Use the most recent mtime?
+ # Read in the file details.
+ }
+
+ # Read in any entries from 'file_locations'.
+
+ return(0);
+}
+
+# This handles deleting a file.
+sub check_incoming
+{
+ my ($anvil) = @_;
+
+ # This hash shouldn't exist, but lets be safe...
+ if (exists $anvil->data->{scan}{directories})
+ {
+ delete $anvil->data->{scan}{directories};
+ }
+ # Read any files in '/mnt/shared/incoming'.
+ $anvil->Storage->scan_directory({
+ debug => 3,
+ directory => $anvil->data->{path}{directories}{shared}{incoming},
+ recursive => 0,
+ });
+
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0264"});
+ foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}})
+ {
+ # Is this a file?
+ my $file_name = $anvil->data->{scan}{directories}{$full_path}{name};
+ my $file_type = $anvil->data->{scan}{directories}{$full_path}{type};
+ my $file_size = $anvil->data->{scan}{directories}{$full_path}{size};
+ my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype};
+ my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0;
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+ full_path => $full_path,
+ file_name => $file_name,
+ file_type => $file_type,
+ file_size => $file_size,
+ file_mimetype => $file_mimetype,
+ file_executable => $file_executable,
+ }});
+ next if $file_type ne "file";
+
+ # If this file is over 128 MiB, warn the user that it might take a second
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0265", variables => { file => $full_path }});
+ if ($file_size > (128 * (2 ** 20)))
+ {
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0266", variables => {
+ size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
+ }});
+ }
+ my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable);
+ my $file_md5sum = $anvil->Get->md5sum({debug => 2, file => $full_path});
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+ say_mimetype => $say_mimetype,
+ file_md5sum => $file_md5sum,
+ }});
+
+ # Do we know about this file? If not, file_uuid will be blank when we call the
+ # insert_or_update. If we do, it will update the file name, if needed.
+ my ($file_uuid) = $anvil->Database->insert_or_update_files({
+ debug => 2,
+ file_uuid => get_file_uuid($anvil, $file_md5sum, $file_name),
+ file_name => $file_name,
+ file_size => $file_size,
+ file_md5sum => $file_md5sum,
+ file_type => $say_mimetype,
+ });
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
+
+ # Make sure we know about this file on this system
+ my ($file_locatiom_uuid) = $anvil->Database->insert_or_update_file_locations({
+ debug => 2,
+ file_location_file_uuid => $file_uuid,
+ file_location_host_uuid => $anvil->data->{sys}{host_uuid},
+ });
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_locatiom_uuid => $file_locatiom_uuid }});
+
+ # Not move it. If it's a definition file, we'll move it to
+ # 'path::directories::shared::definitions', otherwise we'll move it to
+ # 'path::directories::shared::files'.
+ my $target = $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files};
+ $target .= "/";
+ $target =~ s/\/\//\//g;
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }});
+
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0268", variables => {
+ file => $full_path,
+ target => $target,
+ }});
+ $anvil->Storage->move_file({
+ debug => 2,
+ source_file => $full_path,
+ target_file => $target,
+ });
+ }
+
+ return(0);
+}
+
# This gets the file_uuid for a given file name and/or md5sum. If the file isn't found, an empty string is
# returned.
sub get_file_uuid
@@ -191,7 +351,6 @@ sub handle_script
{
# Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0055"});
- if ($anvil->data->{jobs}{'job-uuid'})
$anvil->Job->update_progress({
progress => 0,
message => "error_0055",
@@ -205,12 +364,14 @@ sub handle_script
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
# Toggle the executable bits.
+ my $executable = 0;
if ($anvil->data->{switches}{'is-script'})
{
# Is it already executable?
if (-x $anvil->data->{switches}{file})
{
# Switch it on
+ $executable = 1;
$anvil->Storage->change_mode({target => $anvil->data->{switches}{file}, mode => "a+x"});
}
else
@@ -224,6 +385,7 @@ sub handle_script
if (-x $anvil->data->{switches}{file})
{
# Switch it off.
+ $executable = 1;
$anvil->Storage->change_mode({target => $anvil->data->{switches}{file}, mode => "a-x"});
}
else
@@ -249,7 +411,7 @@ WHERE
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
- my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
+ my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results }});
my $file_name = $results->[0]->[0];
my $file_size = $results->[0]->[1];
@@ -264,22 +426,10 @@ WHERE
if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'}))
{
# Figure out what the file type is and update.
- my $mimetype = mimetype($full_path);
+ my $mimetype = mimetype($file_name);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mimetype => $mimetype }});
- my $say_mimetype = "other";
- if ($mimetype =~ /cd-image/)
- {
- $say_mimetype = "iso";
- }
- elsif ($mimetype =~ /rpm$/)
- {
- $say_mimetype = "rpm";
- }
- elsif ($mimetype =~ /disk-image/)
- {
- $say_mimetype = "disk-image";
- }
+ my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, $executable);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }});
$anvil->Database->insert_or_update_files({
file_uuid => $file_uuid,
@@ -288,11 +438,22 @@ WHERE
file_md5sum => $file_md5sum,
file_type => $say_mimetype,
});
+
+ # Change the file tpye to "say_mimetype".
+ $anvil->Database->insert_or_update_files({
+ debug => 2,
+ file_uuid => $file_uuid,
+ file_name => $anvil->data->{switches}{file},
+ file_size => $file_size,
+ file_md5sum => $file_md5sum,
+ file_type => $say_mimetype,
+ });
}
- elsif ($file_type ne "script") && ($anvil->data->{switches}{'is-script'}))
+ elsif (($file_type ne "script") && ($anvil->data->{switches}{'is-script'}))
{
# Change the file tpye to "script"
$anvil->Database->insert_or_update_files({
+ debug => 2,
file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file},
file_size => $file_size,
@@ -305,6 +466,55 @@ WHERE
return(0);
}
+# This takes the mimetype as reported by the 'mimetype' method and returns a file type we care about.
+sub convert_mimetype
+{
+ my ($anvil, $mimetype, $file, $executable) = @_;
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+ mimetype => $mimetype,
+ file => $file,
+ executable => $executable,
+ }});
+
+ my $say_mimetype = "other";
+ if ($mimetype)
+ {
+ if ($mimetype =~ /cd-image/)
+ {
+ $say_mimetype = "iso";
+ }
+ elsif ($mimetype =~ /xml/)
+ {
+ # This might be a definition, but look inside it to be sure.
+ my $is_domain = $anvil->System->call({
+ debug => 2,
+ shell_call => "if \$(".$anvil->data->{path}{exe}{'grep'}." -q '' ".$file."); then ".$anvil->data->{path}{exe}{echo}." 1; else ".$anvil->data->{path}{exe}{echo}." 0; fi",
+ });
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { is_domain => $is_domain }});
+
+ if ($is_domain)
+ {
+ $say_mimetype = "definition";
+ }
+ }
+ elsif ($mimetype =~ /rpm$/)
+ {
+ $say_mimetype = "rpm";
+ }
+ elsif ($mimetype =~ /disk-image/)
+ {
+ $say_mimetype = "disk-image";
+ }
+ elsif ($executable)
+ {
+ $say_mimetype = "script";
+ }
+ }
+
+ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }});
+ return($say_mimetype);
+}
+
# This handles deleting a file. If the requested deletion target doesn't exist, we'll just clear the
# database. If that doesn't exist either, we still don't error. This is to handle broad requests to delete a
# file everywhere. If we're asked to delete it everywhere, then we'll register a job against all hosts.
@@ -316,7 +526,6 @@ sub handle_delete
{
# Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"});
- if ($anvil->data->{jobs}{'job-uuid'})
$anvil->Job->update_progress({
progress => 0,
message => "error_0052",
@@ -403,10 +612,10 @@ sub handle_rename
elsif (-e $anvil->data->{switches}{to})
{
# There's already a file (or directory or something) with that name.
- $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052", variables => { file => $anvil->data->{switches}{file}, to => $anvil->data->{switches}{to} }});
+ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0056", variables => { file => $anvil->data->{switches}{file}, to => $anvil->data->{switches}{to} }});
$anvil->Job->update_progress({
progress => 0,
- message => "error_0052,!!file!".$anvil->data->{switches}{file}."!!,!!to!".$anvil->data->{switches}{to}."!!",
+ message => "error_0056,!!file!".$anvil->data->{switches}{file}."!!,!!to!".$anvil->data->{switches}{to}."!!",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 6});