seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Created Storage->move_file().

Browse Source 
* Finished getting anvil-manage-files to find and process new files in /mnt/shared/incoming. Created a 'convert_mimetype' function to translate returned mimetype to a file type we care about.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 6 years ago
parent e3d11fd938
commit d9e9884e53
7 changed files with 517 additions and 41 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      Anvil/Tools.pm
  2. 10
      Anvil/Tools/Database.pm
  3. 3
      Anvil/Tools/Get.pm
  4. 270
      Anvil/Tools/Storage.pm
  5. 4
      share/anvil.sql
  6. 13
      share/words.xml
  7. 257
      tools/anvil-manage-files

1
Anvil/Tools.pm
Unescape View File

@ -1002,6 +1002,7 @@ sub _set_paths
logger => "/usr/bin/logger",
md5sum => "/usr/bin/md5sum",
'mkdir' => "/usr/bin/mkdir",
mv => "/usr/bin/mv",
nmcli => "/bin/nmcli",
openssl => "/usr/bin/openssl",
passwd => "/usr/bin/passwd",

10
Anvil/Tools/Database.pm
Unescape View File

@ -2211,6 +2211,8 @@ sub insert_or_update_file_locations
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_file_locations()" }});
my $file = defined $parameter->{file} ? $parameter->{file} : "";
my $line = defined $parameter->{line} ? $parameter->{line} : "";
my $file_location_uuid = defined $parameter->{file_location_uuid} ? $parameter->{file_location_uuid} : "";
my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : "";
my $file_location_host_uuid = defined $parameter->{file_location_host_uuid} ? $parameter->{file_location_host_uuid} : $anvil->data->{sys}{host_uuid};
@ -2301,15 +2303,11 @@ INSERT INTO
file_location_uuid,
file_location_file_uuid,
file_location_host_uuid,
file_md5sum,
file_type,
modified_date
) VALUES (
".$anvil->data->{sys}{database}{use_handle}->quote($file_location_uuid).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_location_file_uuid).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_location_host_uuid).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_type).",
".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})."
);
";
@ -2322,7 +2320,7 @@ INSERT INTO
my $query = "
SELECT
file_location_file_uuid,
file_location_host_uuid,
file_location_host_uuid
FROM
file_locations
WHERE
@ -2412,6 +2410,8 @@ sub insert_or_update_files
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_files()" }});
my $file = defined $parameter->{file} ? $parameter->{file} : "";
my $line = defined $parameter->{line} ? $parameter->{line} : "";
my $file_uuid = defined $parameter->{file_uuid} ? $parameter->{file_uuid} : "";
my $file_name = defined $parameter->{file_name} ? $parameter->{file_name} : "";
my $file_size = defined $parameter->{file_size} ? $parameter->{file_size} : "";

3
Anvil/Tools/Get.pm
Unescape View File

@ -616,13 +616,14 @@ sub md5sum
my $sum = "";
my $file = defined $parameter->{file} ? $parameter->{file} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file => $file }});
if (-e $file)
{
my $shell_call = $anvil->data->{path}{exe}{md5sum}." ".$file;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
my $return = $anvil->System->call({shell_call => $shell_call});
my $return = $anvil->System->call({debug => $debug, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'return' => $return }});
# split the sum off.

270
Anvil/Tools/Storage.pm
Unescape View File

@ -22,6 +22,7 @@ my $THIS_FILE = "Storage.pm";
# copy_file
# find
# make_directory
# move_file
# read_config
# read_file
# read_mode
@ -710,12 +711,16 @@ fi";
}});
if ($output->[0] eq "source file not found")
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { source_file => $source_file }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => {
method => "copy_file",
source_file => $source_file,
}});
return(1);
}
if (($output->[0] eq "source file exists") && (not $overwrite))
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => {
method => "copy_file",
source_file => $source_file,
target_file => $target_file,
}});
@ -732,6 +737,7 @@ fi";
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => {
method => "copy_file",
source_file => $source_file,
target_file => $target_file,
}});
@ -755,7 +761,10 @@ fi";
# Copying locally
if (not -e $source_file)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { source_file => $source_file }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => {
method => "copy_file",
source_file => $source_file,
}});
return(1);
}
@ -764,6 +773,7 @@ fi";
{
# This isn't an error.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => {
method => "copy_file",
source_file => $source_file,
target_file => $target_file,
}});
@ -781,6 +791,7 @@ fi";
if ($failed)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => {
method => "copy_file",
source_file => $source_file,
target_file => $target_file,
}});
@ -1070,6 +1081,249 @@ fi;";
return($failed);
}
=head2 move_file
This moves a file, with a few additional checks like creating the target directory if it doesn't exist, aborting if the file already exists in the target, etc. It can move files on the local or a remote machine.
As with the system copy, the target can be a directory (denoted with an ending c<< / >>), or a it can be renamed in the process (but not ending with C<< / >>).
# Example moving
$anvil->Storage->move_file({source_file => "/some/file", target_file => "/another/directory/"});
# Example moving with a rename at the same time
$anvil->Storage->move_file({source_file => "/some/file", target_file => "/another/directory/new_name"});
Returns C<< 0 >> on success, otherwise C<< 1 >>.
Parameters;
=head3 overwrite (optional)
If this is set to 'C<< 1 >>', and if the target file exists, it will be replaced.
If this is not passed and the target exists, this module will return 'C<< 3 >>'.
=head3 port (optional, default 22)
If C<< target >> is set, this is the TCP port number used to connect to the remote machine.
=head3 password (optional)
If C<< target >> is set, this is the password used to log into the remote system as the C<< remote_user >>. If it is not set, an attempt to connect without a password will be made (though this will usually fail).
=head3 source_file (required)
This is the source file. If it isn't specified, 'C<< 1 >>' will be returned. If it doesn't exist, this method will return 'C<< 4 >>'.
=head3 target (optional)
If set, the file will be copied on the target machine. This must be either an IP address or a resolvable host name.
=head3 target_file (required)
This is the target B<< file >>, not the directory to put it in. The target file name can be different from the source file name.
if this is not specified, 'C<< 2 >>' will be returned.
=head3 remote_user (optional, default root)
If C<< target >> is set, this is the user account that will be used when connecting to the remote system.
=cut
sub move_file
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
my $overwrite = defined $parameter->{overwrite} ? $parameter->{overwrite} : 0;
my $password = defined $parameter->{password} ? $parameter->{password} : "";
my $remote_user = defined $parameter->{remote_user} ? $parameter->{remote_user} : "root";
my $source_file = defined $parameter->{source_file} ? $parameter->{source_file} : "";
my $target_file = defined $parameter->{target_file} ? $parameter->{target_file} : "";
my $target = defined $parameter->{target} ? $parameter->{target} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
overwrite => $overwrite,
password => $anvil->Log->secure ? $password : $anvil->Words->string({key => "log_0186"}),
remote_user => $remote_user,
source_file => $source_file,
target_file => $target_file,
target => $target,
}});
if (not $source_file)
{
# No source passed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Storage->move_file()", parameter => "source_file" }});
return(1);
}
if (not $target_file)
{
# No target passed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Storage->move_file()", parameter => "target_file" }});
return(2);
}
# If we have a target directory, pull the file name off the source for the target checks.
my ($directory, $file) = ($target_file =~ /^(.*)\/(.*?)$/);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
directory => $directory,
file => $file,
}});
if (not $file)
{
($file) = ($source_file =~ /^.*\/(.*?)$/);
$target_file .= $file;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file => $file,
target_file => $target_file,
}});
}
if ($target)
{
# Copying on a remote system.
my $proceed = 1;
my $shell_call = "
if [ -e '".$source_file."' ];
".$anvil->data->{path}{exe}{echo}." 'source file exists'
else
".$anvil->data->{path}{exe}{echo}." 'source file not found'
fi
if [ -d '".$target_file."' ];
".$anvil->data->{path}{exe}{echo}." 'target file exists'
elif [ -d '".$directory."' ];
".$anvil->data->{path}{exe}{echo}." 'target directory exists'
else
".$anvil->data->{path}{exe}{echo}." 'target directory not found'
fi";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0166", variables => { shell_call => $shell_call, target => $target, remote_user => $remote_user }});
my ($error, $output) = $anvil->Remote->call({
debug => $debug,
target => $target,
user => $remote_user,
password => $password,
remote_user => $remote_user,
shell_call => $shell_call,
});
if ($error)
{
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0267", variables => {
source_file => $source_file,
target_file => $target_file,
error => $error,
output => $output,
target => $target,
remote_user => $remote_user,
}});
return(1);
}
else
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
'output->[0]' => $output->[0],
'output->[1]' => $output->[1],
}});
if ($output->[0] eq "source file not found")
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => {
method => "move_file",
source_file => $source_file,
}});
return(1);
}
if (($output->[0] eq "source file exists") && (not $overwrite))
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => {
method => "move_file",
source_file => $source_file,
target_file => $target_file,
}});
return(1);
}
if ($output->[1] eq "target directory not found")
{
my $failed = $anvil->Storage->make_directory({
debug => $debug,
directory => $directory,
password => $password,
remote_user => $remote_user,
target => $target,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => {
method => "move_file",
source_file => $source_file,
target_file => $target_file,
}});
return(1);
}
# Now backup the file.
my ($error, $output) = $anvil->Remote->call({
debug => $debug,
target => $target,
user => $remote_user,
password => $password,
remote_user => $remote_user,
shell_call => $anvil->data->{path}{exe}{mv}." -f ".$source_file." ".$target_file,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output }});
}
}
else
{
# Copying locally
if (not -e $source_file)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => {
method => "move_file",
source_file => $source_file,
}});
return(1);
}
# If the target exists, abort
if ((-e $target_file) && (not $overwrite))
{
# This isn't an error.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => {
method => "move_file",
source_file => $source_file,
target_file => $target_file,
}});
return(1);
}
# Make sure the target directory exists and create it, if not.
if (not -e $directory)
{
my $failed = $anvil->Storage->make_directory({
debug => $debug,
directory => $directory,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }});
if ($failed)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => {
method => "move_file",
source_file => $source_file,
target_file => $target_file,
}});
return(1);
}
}
# Now backup the file.
my $output = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{'mv'}." -f ".$source_file." ".$target_file});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output }});
}
return(0);
}
=head2 read_config
This method is used to read 'Anvil::Tools' style configuration files. These configuration files are in the format:
@ -1781,9 +2035,15 @@ sub scan_directory
{
next if $file eq ".";
next if $file eq "..";
my $full_path = $directory."/".$file;
$full_path =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { full_path => $full_path }});
my $full_path = $directory."/".$file;
$full_path =~ s/\/\//\//g;
$anvil->data->{scan}{directories}{$full_path}{directory} = $directory;
$anvil->data->{scan}{directories}{$full_path}{name} = $file;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"scan::directories::${full_path}::directory" => $anvil->data->{scan}{directories}{$full_path}{directory},
"scan::directories::${full_path}::name" => $anvil->data->{scan}{directories}{$full_path}{name},
full_path => $full_path,
}});
if ((-d $full_path) && ($recursive))
{
# This is a directory, dive into it is asked.

4
share/anvil.sql
Unescape View File

@ -1181,8 +1181,8 @@ CREATE TRIGGER trigger_files
-- This tracks which files should be on which machines.
CREATE TABLE file_locations (
file_location_uuid uuid not null primary key,
file_location_file_uuid text not null, -- This is file to be moved to (or restored to) this machine.
file_location_host_uuid text not null, -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change.
file_location_file_uuid uuid not null, -- This is file to be moved to (or restored to) this machine.
file_location_host_uuid uuid not null, -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change.
modified_date timestamp with time zone not null,
FOREIGN KEY(file_location_file_uuid) REFERENCES files(file_uuid),

13
share/words.xml
Unescape View File

@ -299,13 +299,13 @@ It also has replacement variables: [#!variable!first!#] and [#!variable!second!#
<key name="log_0043"><![CDATA[[ Error ] - The method System->call() was called but 'shell_call' was not passed or was empty.]]></key>
<key name="log_0044">The host: [#!variable!host!#] has renewed its database lock.</key>
<key name="log_0045">The host: [#!variable!host!#] is requesting a database lock.</key>
<key name="log_0046"><![CDATA[[ Note ] - The method Storage->copy_file() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target already exists and 'overwrite' wasn't specified, skipping.]]></key>
<key name="log_0046"><![CDATA[[ Note ] - The method Storage->#!variable!method!#() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target already exists and 'overwrite' wasn't specified, skipping.]]></key>
<key name="log_0047"><![CDATA[[ Error ] - The method Log->level() was passed an invalid log level: [#!variable!set!#]. Only '0', '1', '2', '3' or '4' are valid.]]></key>
<key name="log_0048"><![CDATA[[ Warning ] - Testing of Anvil::Tools is beginning. This will generate warnings and alerts and are not a concern.]]></key>
<key name="log_0049"><![CDATA[[ Warning ] - Testing of Anvil::Tools is complete.]]></key>
<key name="log_0050">[ Error ] - There is a local database defined, but it does not appear to exist and we could not initialize the database server. Is 'postgresql-server' installed?</key>
<key name="log_0051"><![CDATA[[ Error ] - The method Storage->change_owner() was asked to change the ownership of: [#!variable!target!#] which doesn't exist.]]></key>
<key name="log_0052"><![CDATA[[ Error ] - The method Storage->copy_file() was called but the source file: [#!variable!source_file!#] doesn't exist.]]></key>
<key name="log_0052"><![CDATA[[ Error ] - The method Storage->#!variable!method!#() was called but the source file: [#!variable!source_file!#] doesn't exist.]]></key>
<key name="log_0053"><![CDATA[[ Error ] - The 'Database->connect()' method tried to connect to the same database twice: [#!variable!target!#].]]></key>
<key name="log_0054">
Connecting to Database with configuration ID: [#!variable!uuid!#]
@ -450,7 +450,7 @@ The database connection error was:
<key name="log_0167">Failed to create the directory: [#!variable!directory!#] on: [#!variable!target!#] as: [#!variable!remote_user!#]. The error (if any) was: [#!variable!error!#] and the output (if any) was: [#!variable!output!#].</key>
<key name="log_0168">Failed to create the directory: [#!variable!directory!#]. The error (if any) was: [#!variable!error!#].</key>
<key name="log_0169">Failed to copy the file: [#!variable!source_file!#] to: [#!variable!target_file!#] on the target: [#!variable!target!#] as: [#!variable!remote_user!#]. The error (if any) was: [#!variable!error!#] and the output (if any) was: [#!variable!output!#].</key>
<key name="log_0170"><![CDATA[[ Note ] - The method Storage->copy_file() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target's parent directory doesn't exist and we were unable to create it.]]></key>
<key name="log_0170"><![CDATA[[ Note ] - The method Storage->#!variable!method!#() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target's parent directory doesn't exist and we were unable to create it.]]></key>
<key name="log_0171"><![CDATA[[ Error ] - The method: Account->encrypt_password() tried to use the algorithm: [#!variable!algorithm!#], which is not recognized. Only 'sha256', 'sha384' and 'sha512' are currently supported. The desired algorithm can be set via 'sys::password::algorithm'.]]></key>
<key name="log_0172"><![CDATA[[ Error ] - Asked to validate a password for the user: [#!variable!user!#], but that user wasn't found.]]></key>
<key name="log_0173"><![CDATA[[ Error ] - Asked to valudate a password encoded with the algorithm: [#!variable!user_algorithm!#], which is not recognized. Only 'sha256', 'sha384' and 'sha512' are currently supported.]]></key>
@ -558,6 +558,11 @@ Finished Downloading: [#!variable!file!#].
<key name="log_0261">#!variable!file!# was called, but no files where available for download in CGI. Was the variable name 'upload_file' used?</key>
<key name="log_0262">[ Error ] - Storage->scan_directory() was asked to scan: [#!variable!directory!#], but it doesn't exist or isn't actually a directory.</key>
<key name="log_0263">Now deleting the file: [#!variable!file!#].</key>
<key name="log_0264">Checking: [#!data!path::directories::shared::incoming!#] for new files.</key>
<key name="log_0265">About to calculate the md5sum for the file: [#!variable!file!#].</key>
<key name="log_0266">This file is large, [#!variable!size!#], this might take a bit of time...</key>
<key name="log_0267">Failed to move the file: [#!variable!source_file!#] to: [#!variable!target_file!#] on the target: [#!variable!target!#] as: [#!variable!remote_user!#]. The error (if any) was: [#!variable!error!#] and the output (if any) was: [#!variable!output!#].</key>
<key name="log_0268">The file: [#!variable!file!#] has been added to the database (if needed) moved to: [#!variable!target!#].</key>
<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
<key name="t_0000">Test</key>
@ -813,11 +818,11 @@ The update appears to have not completed successfully. The output was:
<key name="error_0049">A request to rename a file was made, but no file name was given.</key>
<key name="error_0050">A request to rename the file: [#!variable!file!#] was made, but the new name wasn't given. Was '--to X' given?</key>
<key name="error_0051">A request to rename the file: [#!variable!file!#] was made, but that file doesn't exist.</key>
<key name="error_0051">A request to rename the file: [#!variable!file!#] to: [#!variable!to!#], but there is an existing file or directory with that name.</key>
<key name="error_0052">A request to delete a file was made, but no file name was given.</key>
<key name="error_0053">A request to delete the file: [#!variable!file!#] was received, but it is not under '/mnt/shared/'. This program can only work on or under that directory.</key>
<key name="error_0054">Failed!</key>
<key name="error_0055">A request to toggle the script flag was received, but no file name was given.</key>
<key name="error_0056">A request to rename the file: [#!variable!file!#] to: [#!variable!to!#], but there is an existing file or directory with that name.</key>
<!-- These are units, words and so on used when displaying information. -->
<key name="unit_0001">Yes</key>

257
tools/anvil-manage-files
Unescape View File

@ -52,6 +52,8 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
}
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
$anvil->data->{switches}{'job-uuid'} = "";
$anvil->data->{switches}{'rename'} = "";
@ -61,7 +63,7 @@ $anvil->data->{switches}{to} = "";
$anvil->data->{switches}{'delete'} = "";
$anvil->data->{switches}{everywhere} = "";
$anvil->Get->switches;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"switches::job-uuid" => $anvil->data->{switches}{'job-uuid'},
"switches::rename" => $anvil->data->{switches}{'rename'},
"switches::is-script" => $anvil->data->{switches}{'is-script'},
@ -115,15 +117,22 @@ if ($anvil->data->{switches}{'rename'})
{
handle_rename($anvil);
}
elsif ($anvil->data->{switches}{'delete'}))
elsif ($anvil->data->{switches}{'delete'})
{
handle_delete($anvil);
}
elsif ($anvil->data->{switches}{'is-script'}))
elsif ($anvil->data->{switches}{'is-script'})
{
handle_script($anvil);
}
else
{
# Check for new files
check_incoming($anvil);
# Check for files we should have but don't yet have.
find_missing_files($anvil);
}
# We're done
$anvil->nice_exit({exit_code => 0});
@ -133,6 +142,157 @@ $anvil->nice_exit({exit_code => 0});
# Private functions. #
#############################################################################################################
# This looks to see if there are any entries in 'file_locations' for us that, the pointed to file, doesn't
# exist on this machine. For those entries, we will search for the file on other machines. The one exception
# is server definition files. For those, we write the file out from the server's 'definition' table entry.
#
# When a missing entry is found, and an entry doesn't exist, the file will be found (if possible) and copied
# to this houst. Only machines on the same subnet are searched. The search pattern is;
#
# Nodes; 1. Check for the file on the peer.
# 2. Check for the file on Strikers, in alphabetical order.
# 3. Check for the file on DR host, if available.
# 4. Check other nodes, in alphabetical order.
# 5. Check other DR hosts, in alphabetical order.
# Striker; 1. Check for the file on other Strikers, in alphabetical order.
# 2. Check for the file on DR hosts, if available
# 3. Check for the file on Anvil! nodes.
# DR Host; 1. Check for the file on Strikers, in alphabetical order.
# 2. Check for the file on Anvil! nodes.
# * If a file can't be found, it will try again every so often until it is found.
# * When a file is found, it is copied to '/mnt/shared/incoming'. Only when the file has arrived and the
# md5sum matches. At this point, it is moved into the proper directory.
sub find_missing_files
{
my ($anvil) = @_;
# What am I? This will impact how missing files are found.
my $query = "
SELECT
file_location_file_uuid
FROM
file_locations
WHERE
file_location_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{host_uuid})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $file_location_file_uuid = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_location_file_uuid => $file_location_file_uuid,
}});
### TODO: How to handle when the file with the same name exists on 2+ machines with
### different md5sums. Use the most recent mtime?
# Read in the file details.
}
# Read in any entries from 'file_locations'.
return(0);
}
# This handles deleting a file.
sub check_incoming
{
my ($anvil) = @_;
# This hash shouldn't exist, but lets be safe...
if (exists $anvil->data->{scan}{directories})
{
delete $anvil->data->{scan}{directories};
}
# Read any files in '/mnt/shared/incoming'.
$anvil->Storage->scan_directory({
debug => 3,
directory => $anvil->data->{path}{directories}{shared}{incoming},
recursive => 0,
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0264"});
foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}})
{
# Is this a file?
my $file_name = $anvil->data->{scan}{directories}{$full_path}{name};
my $file_type = $anvil->data->{scan}{directories}{$full_path}{type};
my $file_size = $anvil->data->{scan}{directories}{$full_path}{size};
my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype};
my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
full_path => $full_path,
file_name => $file_name,
file_type => $file_type,
file_size => $file_size,
file_mimetype => $file_mimetype,
file_executable => $file_executable,
}});
next if $file_type ne "file";
# If this file is over 128 MiB, warn the user that it might take a second
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0265", variables => { file => $full_path }});
if ($file_size > (128 * (2 ** 20)))
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0266", variables => {
size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
}});
}
my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable);
my $file_md5sum = $anvil->Get->md5sum({debug => 2, file => $full_path});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
say_mimetype => $say_mimetype,
file_md5sum => $file_md5sum,
}});
# Do we know about this file? If not, file_uuid will be blank when we call the
# insert_or_update. If we do, it will update the file name, if needed.
my ($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => get_file_uuid($anvil, $file_md5sum, $file_name),
file_name => $file_name,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => $say_mimetype,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
# Make sure we know about this file on this system
my ($file_locatiom_uuid) = $anvil->Database->insert_or_update_file_locations({
debug => 2,
file_location_file_uuid => $file_uuid,
file_location_host_uuid => $anvil->data->{sys}{host_uuid},
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_locatiom_uuid => $file_locatiom_uuid }});
# Not move it. If it's a definition file, we'll move it to
# 'path::directories::shared::definitions', otherwise we'll move it to
# 'path::directories::shared::files'.
my $target = $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files};
$target .= "/";
$target =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0268", variables => {
file => $full_path,
target => $target,
}});
$anvil->Storage->move_file({
debug => 2,
source_file => $full_path,
target_file => $target,
});
}
return(0);
}
# This gets the file_uuid for a given file name and/or md5sum. If the file isn't found, an empty string is
# returned.
sub get_file_uuid
@ -191,7 +351,6 @@ sub handle_script
{
# Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0055"});
if ($anvil->data->{jobs}{'job-uuid'})
$anvil->Job->update_progress({
progress => 0,
message => "error_0055",
@ -205,12 +364,14 @@ sub handle_script
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
# Toggle the executable bits.
my $executable = 0;
if ($anvil->data->{switches}{'is-script'})
{
# Is it already executable?
if (-x $anvil->data->{switches}{file})
{
# Switch it on
$executable = 1;
$anvil->Storage->change_mode({target => $anvil->data->{switches}{file}, mode => "a+x"});
}
else
@ -224,6 +385,7 @@ sub handle_script
if (-x $anvil->data->{switches}{file})
{
# Switch it off.
$executable = 1;
$anvil->Storage->change_mode({target => $anvil->data->{switches}{file}, mode => "a-x"});
}
else
@ -249,7 +411,7 @@ WHERE
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results }});
my $file_name = $results->[0]->[0];
my $file_size = $results->[0]->[1];
@ -264,22 +426,10 @@ WHERE
if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'}))
{
# Figure out what the file type is and update.
my $mimetype = mimetype($full_path);
my $mimetype = mimetype($file_name);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mimetype => $mimetype }});
my $say_mimetype = "other";
if ($mimetype =~ /cd-image/)
{
$say_mimetype = "iso";
}
elsif ($mimetype =~ /rpm$/)
{
$say_mimetype = "rpm";
}
elsif ($mimetype =~ /disk-image/)
{
$say_mimetype = "disk-image";
}
my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, $executable);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }});
$anvil->Database->insert_or_update_files({
file_uuid => $file_uuid,
@ -288,11 +438,22 @@ WHERE
file_md5sum => $file_md5sum,
file_type => $say_mimetype,
});
# Change the file tpye to "say_mimetype".
$anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file},
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => $say_mimetype,
});
}
elsif ($file_type ne "script") && ($anvil->data->{switches}{'is-script'}))
elsif (($file_type ne "script") && ($anvil->data->{switches}{'is-script'}))
{
# Change the file tpye to "script"
$anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file},
file_size => $file_size,
@ -305,6 +466,55 @@ WHERE
return(0);
}
# This takes the mimetype as reported by the 'mimetype' method and returns a file type we care about.
sub convert_mimetype
{
my ($anvil, $mimetype, $file, $executable) = @_;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
mimetype => $mimetype,
file => $file,
executable => $executable,
}});
my $say_mimetype = "other";
if ($mimetype)
{
if ($mimetype =~ /cd-image/)
{
$say_mimetype = "iso";
}
elsif ($mimetype =~ /xml/)
{
# This might be a definition, but look inside it to be sure.
my $is_domain = $anvil->System->call({
debug => 2,
shell_call => "if \$(".$anvil->data->{path}{exe}{'grep'}." -q '</domain>' ".$file."); then ".$anvil->data->{path}{exe}{echo}." 1; else ".$anvil->data->{path}{exe}{echo}." 0; fi",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { is_domain => $is_domain }});
if ($is_domain)
{
$say_mimetype = "definition";
}
}
elsif ($mimetype =~ /rpm$/)
{
$say_mimetype = "rpm";
}
elsif ($mimetype =~ /disk-image/)
{
$say_mimetype = "disk-image";
}
elsif ($executable)
{
$say_mimetype = "script";
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }});
return($say_mimetype);
}
# This handles deleting a file. If the requested deletion target doesn't exist, we'll just clear the
# database. If that doesn't exist either, we still don't error. This is to handle broad requests to delete a
# file everywhere. If we're asked to delete it everywhere, then we'll register a job against all hosts.
@ -316,7 +526,6 @@ sub handle_delete
{
# Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"});
if ($anvil->data->{jobs}{'job-uuid'})
$anvil->Job->update_progress({
progress => 0,
message => "error_0052",
@ -403,10 +612,10 @@ sub handle_rename
elsif (-e $anvil->data->{switches}{to})
{
# There's already a file (or directory or something) with that name.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052", variables => { file => $anvil->data->{switches}{file}, to => $anvil->data->{switches}{to} }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0056", variables => { file => $anvil->data->{switches}{file}, to => $anvil->data->{switches}{to} }});
$anvil->Job->update_progress({
progress => 0,
message => "error_0052,!!file!".$anvil->data->{switches}{file}."!!,!!to!".$anvil->data->{switches}{to}."!!",
message => "error_0056,!!file!".$anvil->data->{switches}{file}."!!,!!to!".$anvil->data->{switches}{to}."!!",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 6});

Write Preview
Loading…
Cancel
Save