From d9e9884e531d4c7734cd66de7e0e252f18401dfa Mon Sep 17 00:00:00 2001 From: Digimer Date: Thu, 21 Feb 2019 01:37:51 -0500 Subject: [PATCH] * Created Storage->move_file(). * Finished getting anvil-manage-files to find and process new files in /mnt/shared/incoming. Created a 'convert_mimetype' function to translate returned mimetype to a file type we care about. Signed-off-by: Digimer --- Anvil/Tools.pm | 1 + Anvil/Tools/Database.pm | 10 +- Anvil/Tools/Get.pm | 3 +- Anvil/Tools/Storage.pm | 270 ++++++++++++++++++++++++++++++++++++++- share/anvil.sql | 4 +- share/words.xml | 13 +- tools/anvil-manage-files | 257 +++++++++++++++++++++++++++++++++---- 7 files changed, 517 insertions(+), 41 deletions(-) diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index 76d5193b..65ccf372 100644 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -1002,6 +1002,7 @@ sub _set_paths logger => "/usr/bin/logger", md5sum => "/usr/bin/md5sum", 'mkdir' => "/usr/bin/mkdir", + mv => "/usr/bin/mv", nmcli => "/bin/nmcli", openssl => "/usr/bin/openssl", passwd => "/usr/bin/passwd", diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 7466ee25..245f883d 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -2211,6 +2211,8 @@ sub insert_or_update_file_locations my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_file_locations()" }}); + my $file = defined $parameter->{file} ? $parameter->{file} : ""; + my $line = defined $parameter->{line} ? $parameter->{line} : ""; my $file_location_uuid = defined $parameter->{file_location_uuid} ? $parameter->{file_location_uuid} : ""; my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : ""; my $file_location_host_uuid = defined $parameter->{file_location_host_uuid} ? $parameter->{file_location_host_uuid} : $anvil->data->{sys}{host_uuid}; @@ -2301,15 +2303,11 @@ INSERT INTO file_location_uuid, file_location_file_uuid, file_location_host_uuid, - file_md5sum, - file_type, modified_date ) VALUES ( ".$anvil->data->{sys}{database}{use_handle}->quote($file_location_uuid).", ".$anvil->data->{sys}{database}{use_handle}->quote($file_location_file_uuid).", ".$anvil->data->{sys}{database}{use_handle}->quote($file_location_host_uuid).", - ".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).", - ".$anvil->data->{sys}{database}{use_handle}->quote($file_type).", ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})." ); "; @@ -2322,7 +2320,7 @@ INSERT INTO my $query = " SELECT file_location_file_uuid, - file_location_host_uuid, + file_location_host_uuid FROM file_locations WHERE @@ -2412,6 +2410,8 @@ sub insert_or_update_files my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_files()" }}); + my $file = defined $parameter->{file} ? $parameter->{file} : ""; + my $line = defined $parameter->{line} ? $parameter->{line} : ""; my $file_uuid = defined $parameter->{file_uuid} ? $parameter->{file_uuid} : ""; my $file_name = defined $parameter->{file_name} ? $parameter->{file_name} : ""; my $file_size = defined $parameter->{file_size} ? $parameter->{file_size} : ""; diff --git a/Anvil/Tools/Get.pm b/Anvil/Tools/Get.pm index ebb922fa..c50d22d3 100644 --- a/Anvil/Tools/Get.pm +++ b/Anvil/Tools/Get.pm @@ -616,13 +616,14 @@ sub md5sum my $sum = ""; my $file = defined $parameter->{file} ? $parameter->{file} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file => $file }}); if (-e $file) { my $shell_call = $anvil->data->{path}{exe}{md5sum}." ".$file; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }}); - my $return = $anvil->System->call({shell_call => $shell_call}); + my $return = $anvil->System->call({debug => $debug, shell_call => $shell_call}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'return' => $return }}); # split the sum off. diff --git a/Anvil/Tools/Storage.pm b/Anvil/Tools/Storage.pm index 259de959..45fe5172 100644 --- a/Anvil/Tools/Storage.pm +++ b/Anvil/Tools/Storage.pm @@ -22,6 +22,7 @@ my $THIS_FILE = "Storage.pm"; # copy_file # find # make_directory +# move_file # read_config # read_file # read_mode @@ -710,12 +711,16 @@ fi"; }}); if ($output->[0] eq "source file not found") { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { source_file => $source_file }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { + method => "copy_file", + source_file => $source_file, + }}); return(1); } if (($output->[0] eq "source file exists") && (not $overwrite)) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => { + method => "copy_file", source_file => $source_file, target_file => $target_file, }}); @@ -732,6 +737,7 @@ fi"; }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => { + method => "copy_file", source_file => $source_file, target_file => $target_file, }}); @@ -755,7 +761,10 @@ fi"; # Copying locally if (not -e $source_file) { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { source_file => $source_file }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { + method => "copy_file", + source_file => $source_file, + }}); return(1); } @@ -764,6 +773,7 @@ fi"; { # This isn't an error. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => { + method => "copy_file", source_file => $source_file, target_file => $target_file, }}); @@ -781,6 +791,7 @@ fi"; if ($failed) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => { + method => "copy_file", source_file => $source_file, target_file => $target_file, }}); @@ -1070,6 +1081,249 @@ fi;"; return($failed); } +=head2 move_file + +This moves a file, with a few additional checks like creating the target directory if it doesn't exist, aborting if the file already exists in the target, etc. It can move files on the local or a remote machine. + +As with the system copy, the target can be a directory (denoted with an ending c<< / >>), or a it can be renamed in the process (but not ending with C<< / >>). + + # Example moving + $anvil->Storage->move_file({source_file => "/some/file", target_file => "/another/directory/"}); + + # Example moving with a rename at the same time + $anvil->Storage->move_file({source_file => "/some/file", target_file => "/another/directory/new_name"}); + +Returns C<< 0 >> on success, otherwise C<< 1 >>. + +Parameters; + +=head3 overwrite (optional) + +If this is set to 'C<< 1 >>', and if the target file exists, it will be replaced. + +If this is not passed and the target exists, this module will return 'C<< 3 >>'. + +=head3 port (optional, default 22) + +If C<< target >> is set, this is the TCP port number used to connect to the remote machine. + +=head3 password (optional) + +If C<< target >> is set, this is the password used to log into the remote system as the C<< remote_user >>. If it is not set, an attempt to connect without a password will be made (though this will usually fail). + +=head3 source_file (required) + +This is the source file. If it isn't specified, 'C<< 1 >>' will be returned. If it doesn't exist, this method will return 'C<< 4 >>'. + +=head3 target (optional) + +If set, the file will be copied on the target machine. This must be either an IP address or a resolvable host name. + +=head3 target_file (required) + +This is the target B<< file >>, not the directory to put it in. The target file name can be different from the source file name. + +if this is not specified, 'C<< 2 >>' will be returned. + +=head3 remote_user (optional, default root) + +If C<< target >> is set, this is the user account that will be used when connecting to the remote system. + +=cut +sub move_file +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + + my $overwrite = defined $parameter->{overwrite} ? $parameter->{overwrite} : 0; + my $password = defined $parameter->{password} ? $parameter->{password} : ""; + my $remote_user = defined $parameter->{remote_user} ? $parameter->{remote_user} : "root"; + my $source_file = defined $parameter->{source_file} ? $parameter->{source_file} : ""; + my $target_file = defined $parameter->{target_file} ? $parameter->{target_file} : ""; + my $target = defined $parameter->{target} ? $parameter->{target} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + overwrite => $overwrite, + password => $anvil->Log->secure ? $password : $anvil->Words->string({key => "log_0186"}), + remote_user => $remote_user, + source_file => $source_file, + target_file => $target_file, + target => $target, + }}); + + if (not $source_file) + { + # No source passed. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Storage->move_file()", parameter => "source_file" }}); + return(1); + } + if (not $target_file) + { + # No target passed. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Storage->move_file()", parameter => "target_file" }}); + return(2); + } + + # If we have a target directory, pull the file name off the source for the target checks. + my ($directory, $file) = ($target_file =~ /^(.*)\/(.*?)$/); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + directory => $directory, + file => $file, + }}); + if (not $file) + { + ($file) = ($source_file =~ /^.*\/(.*?)$/); + $target_file .= $file; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + file => $file, + target_file => $target_file, + }}); + } + + if ($target) + { + # Copying on a remote system. + my $proceed = 1; + my $shell_call = " +if [ -e '".$source_file."' ]; + ".$anvil->data->{path}{exe}{echo}." 'source file exists' +else + ".$anvil->data->{path}{exe}{echo}." 'source file not found' +fi +if [ -d '".$target_file."' ]; + ".$anvil->data->{path}{exe}{echo}." 'target file exists' +elif [ -d '".$directory."' ]; + ".$anvil->data->{path}{exe}{echo}." 'target directory exists' +else + ".$anvil->data->{path}{exe}{echo}." 'target directory not found' +fi"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0166", variables => { shell_call => $shell_call, target => $target, remote_user => $remote_user }}); + my ($error, $output) = $anvil->Remote->call({ + debug => $debug, + target => $target, + user => $remote_user, + password => $password, + remote_user => $remote_user, + shell_call => $shell_call, + }); + if ($error) + { + # Something went wrong. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0267", variables => { + source_file => $source_file, + target_file => $target_file, + error => $error, + output => $output, + target => $target, + remote_user => $remote_user, + }}); + return(1); + } + else + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 'output->[0]' => $output->[0], + 'output->[1]' => $output->[1], + }}); + if ($output->[0] eq "source file not found") + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { + method => "move_file", + source_file => $source_file, + }}); + return(1); + } + if (($output->[0] eq "source file exists") && (not $overwrite)) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => { + method => "move_file", + source_file => $source_file, + target_file => $target_file, + }}); + return(1); + } + if ($output->[1] eq "target directory not found") + { + my $failed = $anvil->Storage->make_directory({ + debug => $debug, + directory => $directory, + password => $password, + remote_user => $remote_user, + target => $target, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => { + method => "move_file", + source_file => $source_file, + target_file => $target_file, + }}); + return(1); + } + + # Now backup the file. + my ($error, $output) = $anvil->Remote->call({ + debug => $debug, + target => $target, + user => $remote_user, + password => $password, + remote_user => $remote_user, + shell_call => $anvil->data->{path}{exe}{mv}." -f ".$source_file." ".$target_file, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output }}); + } + } + else + { + # Copying locally + if (not -e $source_file) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0052", variables => { + method => "move_file", + source_file => $source_file, + }}); + return(1); + } + + # If the target exists, abort + if ((-e $target_file) && (not $overwrite)) + { + # This isn't an error. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0046", variables => { + method => "move_file", + source_file => $source_file, + target_file => $target_file, + }}); + return(1); + } + + # Make sure the target directory exists and create it, if not. + if (not -e $directory) + { + my $failed = $anvil->Storage->make_directory({ + debug => $debug, + directory => $directory, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }}); + if ($failed) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0170", variables => { + method => "move_file", + source_file => $source_file, + target_file => $target_file, + }}); + return(1); + } + } + + # Now backup the file. + my $output = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{'mv'}." -f ".$source_file." ".$target_file}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output }}); + } + + return(0); +} + =head2 read_config This method is used to read 'Anvil::Tools' style configuration files. These configuration files are in the format: @@ -1781,9 +2035,15 @@ sub scan_directory { next if $file eq "."; next if $file eq ".."; - my $full_path = $directory."/".$file; - $full_path =~ s/\/\//\//g; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { full_path => $full_path }}); + my $full_path = $directory."/".$file; + $full_path =~ s/\/\//\//g; + $anvil->data->{scan}{directories}{$full_path}{directory} = $directory; + $anvil->data->{scan}{directories}{$full_path}{name} = $file; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "scan::directories::${full_path}::directory" => $anvil->data->{scan}{directories}{$full_path}{directory}, + "scan::directories::${full_path}::name" => $anvil->data->{scan}{directories}{$full_path}{name}, + full_path => $full_path, + }}); if ((-d $full_path) && ($recursive)) { # This is a directory, dive into it is asked. diff --git a/share/anvil.sql b/share/anvil.sql index a9a724e1..cd5f2dce 100644 --- a/share/anvil.sql +++ b/share/anvil.sql @@ -1181,8 +1181,8 @@ CREATE TRIGGER trigger_files -- This tracks which files should be on which machines. CREATE TABLE file_locations ( file_location_uuid uuid not null primary key, - file_location_file_uuid text not null, -- This is file to be moved to (or restored to) this machine. - file_location_host_uuid text not null, -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change. + file_location_file_uuid uuid not null, -- This is file to be moved to (or restored to) this machine. + file_location_host_uuid uuid not null, -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change. modified_date timestamp with time zone not null, FOREIGN KEY(file_location_file_uuid) REFERENCES files(file_uuid), diff --git a/share/words.xml b/share/words.xml index 2558aea7..fd1bd3e4 100644 --- a/share/words.xml +++ b/share/words.xml @@ -299,13 +299,13 @@ It also has replacement variables: [#!variable!first!#] and [#!variable!second!# call() was called but 'shell_call' was not passed or was empty.]]> The host: [#!variable!host!#] has renewed its database lock. The host: [#!variable!host!#] is requesting a database lock. - copy_file() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target already exists and 'overwrite' wasn't specified, skipping.]]> + #!variable!method!#() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target already exists and 'overwrite' wasn't specified, skipping.]]> level() was passed an invalid log level: [#!variable!set!#]. Only '0', '1', '2', '3' or '4' are valid.]]> [ Error ] - There is a local database defined, but it does not appear to exist and we could not initialize the database server. Is 'postgresql-server' installed? change_owner() was asked to change the ownership of: [#!variable!target!#] which doesn't exist.]]> - copy_file() was called but the source file: [#!variable!source_file!#] doesn't exist.]]> + #!variable!method!#() was called but the source file: [#!variable!source_file!#] doesn't exist.]]> connect()' method tried to connect to the same database twice: [#!variable!target!#].]]> Connecting to Database with configuration ID: [#!variable!uuid!#] @@ -450,7 +450,7 @@ The database connection error was: Failed to create the directory: [#!variable!directory!#] on: [#!variable!target!#] as: [#!variable!remote_user!#]. The error (if any) was: [#!variable!error!#] and the output (if any) was: [#!variable!output!#]. Failed to create the directory: [#!variable!directory!#]. The error (if any) was: [#!variable!error!#]. Failed to copy the file: [#!variable!source_file!#] to: [#!variable!target_file!#] on the target: [#!variable!target!#] as: [#!variable!remote_user!#]. The error (if any) was: [#!variable!error!#] and the output (if any) was: [#!variable!output!#]. - copy_file() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target's parent directory doesn't exist and we were unable to create it.]]> + #!variable!method!#() was asked to copy: [#!variable!source_file!#] to: [#!variable!target_file!#], but the target's parent directory doesn't exist and we were unable to create it.]]> encrypt_password() tried to use the algorithm: [#!variable!algorithm!#], which is not recognized. Only 'sha256', 'sha384' and 'sha512' are currently supported. The desired algorithm can be set via 'sys::password::algorithm'.]]> @@ -558,6 +558,11 @@ Finished Downloading: [#!variable!file!#]. #!variable!file!# was called, but no files where available for download in CGI. Was the variable name 'upload_file' used? [ Error ] - Storage->scan_directory() was asked to scan: [#!variable!directory!#], but it doesn't exist or isn't actually a directory. Now deleting the file: [#!variable!file!#]. + Checking: [#!data!path::directories::shared::incoming!#] for new files. + About to calculate the md5sum for the file: [#!variable!file!#]. + This file is large, [#!variable!size!#], this might take a bit of time... + Failed to move the file: [#!variable!source_file!#] to: [#!variable!target_file!#] on the target: [#!variable!target!#] as: [#!variable!remote_user!#]. The error (if any) was: [#!variable!error!#] and the output (if any) was: [#!variable!output!#]. + The file: [#!variable!file!#] has been added to the database (if needed) moved to: [#!variable!target!#]. Test @@ -813,11 +818,11 @@ The update appears to have not completed successfully. The output was: A request to rename a file was made, but no file name was given. A request to rename the file: [#!variable!file!#] was made, but the new name wasn't given. Was '--to X' given? A request to rename the file: [#!variable!file!#] was made, but that file doesn't exist. - A request to rename the file: [#!variable!file!#] to: [#!variable!to!#], but there is an existing file or directory with that name. A request to delete a file was made, but no file name was given. A request to delete the file: [#!variable!file!#] was received, but it is not under '/mnt/shared/'. This program can only work on or under that directory. Failed! A request to toggle the script flag was received, but no file name was given. + A request to rename the file: [#!variable!file!#] to: [#!variable!to!#], but there is an existing file or directory with that name. Yes diff --git a/tools/anvil-manage-files b/tools/anvil-manage-files index 6aa3a32c..1fef8b7a 100755 --- a/tools/anvil-manage-files +++ b/tools/anvil-manage-files @@ -52,6 +52,8 @@ if (($running_directory =~ /^\./) && ($ENV{PWD})) } my $anvil = Anvil::Tools->new(); +$anvil->Log->level({set => 2}); +$anvil->Log->secure({set => 1}); $anvil->data->{switches}{'job-uuid'} = ""; $anvil->data->{switches}{'rename'} = ""; @@ -61,7 +63,7 @@ $anvil->data->{switches}{to} = ""; $anvil->data->{switches}{'delete'} = ""; $anvil->data->{switches}{everywhere} = ""; $anvil->Get->switches; -$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'}, "switches::rename" => $anvil->data->{switches}{'rename'}, "switches::is-script" => $anvil->data->{switches}{'is-script'}, @@ -115,15 +117,22 @@ if ($anvil->data->{switches}{'rename'}) { handle_rename($anvil); } -elsif ($anvil->data->{switches}{'delete'})) +elsif ($anvil->data->{switches}{'delete'}) { handle_delete($anvil); } -elsif ($anvil->data->{switches}{'is-script'})) +elsif ($anvil->data->{switches}{'is-script'}) { handle_script($anvil); } - +else +{ + # Check for new files + check_incoming($anvil); + + # Check for files we should have but don't yet have. + find_missing_files($anvil); +} # We're done $anvil->nice_exit({exit_code => 0}); @@ -133,6 +142,157 @@ $anvil->nice_exit({exit_code => 0}); # Private functions. # ############################################################################################################# +# This looks to see if there are any entries in 'file_locations' for us that, the pointed to file, doesn't +# exist on this machine. For those entries, we will search for the file on other machines. The one exception +# is server definition files. For those, we write the file out from the server's 'definition' table entry. +# +# When a missing entry is found, and an entry doesn't exist, the file will be found (if possible) and copied +# to this houst. Only machines on the same subnet are searched. The search pattern is; +# +# Nodes; 1. Check for the file on the peer. +# 2. Check for the file on Strikers, in alphabetical order. +# 3. Check for the file on DR host, if available. +# 4. Check other nodes, in alphabetical order. +# 5. Check other DR hosts, in alphabetical order. +# Striker; 1. Check for the file on other Strikers, in alphabetical order. +# 2. Check for the file on DR hosts, if available +# 3. Check for the file on Anvil! nodes. +# DR Host; 1. Check for the file on Strikers, in alphabetical order. +# 2. Check for the file on Anvil! nodes. +# * If a file can't be found, it will try again every so often until it is found. +# * When a file is found, it is copied to '/mnt/shared/incoming'. Only when the file has arrived and the +# md5sum matches. At this point, it is moved into the proper directory. +sub find_missing_files +{ + my ($anvil) = @_; + + # What am I? This will impact how missing files are found. + my $query = " +SELECT + file_location_file_uuid +FROM + file_locations +WHERE + file_location_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{host_uuid})." +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + + my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $file_location_file_uuid = $row->[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + file_location_file_uuid => $file_location_file_uuid, + }}); + + ### TODO: How to handle when the file with the same name exists on 2+ machines with + ### different md5sums. Use the most recent mtime? + # Read in the file details. + } + + # Read in any entries from 'file_locations'. + + return(0); +} + +# This handles deleting a file. +sub check_incoming +{ + my ($anvil) = @_; + + # This hash shouldn't exist, but lets be safe... + if (exists $anvil->data->{scan}{directories}) + { + delete $anvil->data->{scan}{directories}; + } + # Read any files in '/mnt/shared/incoming'. + $anvil->Storage->scan_directory({ + debug => 3, + directory => $anvil->data->{path}{directories}{shared}{incoming}, + recursive => 0, + }); + + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0264"}); + foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}}) + { + # Is this a file? + my $file_name = $anvil->data->{scan}{directories}{$full_path}{name}; + my $file_type = $anvil->data->{scan}{directories}{$full_path}{type}; + my $file_size = $anvil->data->{scan}{directories}{$full_path}{size}; + my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype}; + my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + full_path => $full_path, + file_name => $file_name, + file_type => $file_type, + file_size => $file_size, + file_mimetype => $file_mimetype, + file_executable => $file_executable, + }}); + next if $file_type ne "file"; + + # If this file is over 128 MiB, warn the user that it might take a second + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0265", variables => { file => $full_path }}); + if ($file_size > (128 * (2 ** 20))) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0266", variables => { + size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}), + }}); + } + my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable); + my $file_md5sum = $anvil->Get->md5sum({debug => 2, file => $full_path}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + say_mimetype => $say_mimetype, + file_md5sum => $file_md5sum, + }}); + + # Do we know about this file? If not, file_uuid will be blank when we call the + # insert_or_update. If we do, it will update the file name, if needed. + my ($file_uuid) = $anvil->Database->insert_or_update_files({ + debug => 2, + file_uuid => get_file_uuid($anvil, $file_md5sum, $file_name), + file_name => $file_name, + file_size => $file_size, + file_md5sum => $file_md5sum, + file_type => $say_mimetype, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); + + # Make sure we know about this file on this system + my ($file_locatiom_uuid) = $anvil->Database->insert_or_update_file_locations({ + debug => 2, + file_location_file_uuid => $file_uuid, + file_location_host_uuid => $anvil->data->{sys}{host_uuid}, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_locatiom_uuid => $file_locatiom_uuid }}); + + # Not move it. If it's a definition file, we'll move it to + # 'path::directories::shared::definitions', otherwise we'll move it to + # 'path::directories::shared::files'. + my $target = $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files}; + $target .= "/"; + $target =~ s/\/\//\//g; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }}); + + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0268", variables => { + file => $full_path, + target => $target, + }}); + $anvil->Storage->move_file({ + debug => 2, + source_file => $full_path, + target_file => $target, + }); + } + + return(0); +} + # This gets the file_uuid for a given file name and/or md5sum. If the file isn't found, an empty string is # returned. sub get_file_uuid @@ -191,7 +351,6 @@ sub handle_script { # Um... $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0055"}); - if ($anvil->data->{jobs}{'job-uuid'}) $anvil->Job->update_progress({ progress => 0, message => "error_0055", @@ -205,12 +364,14 @@ sub handle_script $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); # Toggle the executable bits. + my $executable = 0; if ($anvil->data->{switches}{'is-script'}) { # Is it already executable? if (-x $anvil->data->{switches}{file}) { # Switch it on + $executable = 1; $anvil->Storage->change_mode({target => $anvil->data->{switches}{file}, mode => "a+x"}); } else @@ -224,6 +385,7 @@ sub handle_script if (-x $anvil->data->{switches}{file}) { # Switch it off. + $executable = 1; $anvil->Storage->change_mode({target => $anvil->data->{switches}{file}, mode => "a-x"}); } else @@ -249,7 +411,7 @@ WHERE ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); - my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results }}); my $file_name = $results->[0]->[0]; my $file_size = $results->[0]->[1]; @@ -264,22 +426,10 @@ WHERE if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'})) { # Figure out what the file type is and update. - my $mimetype = mimetype($full_path); + my $mimetype = mimetype($file_name); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mimetype => $mimetype }}); - my $say_mimetype = "other"; - if ($mimetype =~ /cd-image/) - { - $say_mimetype = "iso"; - } - elsif ($mimetype =~ /rpm$/) - { - $say_mimetype = "rpm"; - } - elsif ($mimetype =~ /disk-image/) - { - $say_mimetype = "disk-image"; - } + my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, $executable); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }}); $anvil->Database->insert_or_update_files({ file_uuid => $file_uuid, @@ -288,11 +438,22 @@ WHERE file_md5sum => $file_md5sum, file_type => $say_mimetype, }); + + # Change the file tpye to "say_mimetype". + $anvil->Database->insert_or_update_files({ + debug => 2, + file_uuid => $file_uuid, + file_name => $anvil->data->{switches}{file}, + file_size => $file_size, + file_md5sum => $file_md5sum, + file_type => $say_mimetype, + }); } - elsif ($file_type ne "script") && ($anvil->data->{switches}{'is-script'})) + elsif (($file_type ne "script") && ($anvil->data->{switches}{'is-script'})) { # Change the file tpye to "script" $anvil->Database->insert_or_update_files({ + debug => 2, file_uuid => $file_uuid, file_name => $anvil->data->{switches}{file}, file_size => $file_size, @@ -305,6 +466,55 @@ WHERE return(0); } +# This takes the mimetype as reported by the 'mimetype' method and returns a file type we care about. +sub convert_mimetype +{ + my ($anvil, $mimetype, $file, $executable) = @_; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + mimetype => $mimetype, + file => $file, + executable => $executable, + }}); + + my $say_mimetype = "other"; + if ($mimetype) + { + if ($mimetype =~ /cd-image/) + { + $say_mimetype = "iso"; + } + elsif ($mimetype =~ /xml/) + { + # This might be a definition, but look inside it to be sure. + my $is_domain = $anvil->System->call({ + debug => 2, + shell_call => "if \$(".$anvil->data->{path}{exe}{'grep'}." -q '' ".$file."); then ".$anvil->data->{path}{exe}{echo}." 1; else ".$anvil->data->{path}{exe}{echo}." 0; fi", + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { is_domain => $is_domain }}); + + if ($is_domain) + { + $say_mimetype = "definition"; + } + } + elsif ($mimetype =~ /rpm$/) + { + $say_mimetype = "rpm"; + } + elsif ($mimetype =~ /disk-image/) + { + $say_mimetype = "disk-image"; + } + elsif ($executable) + { + $say_mimetype = "script"; + } + } + + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }}); + return($say_mimetype); +} + # This handles deleting a file. If the requested deletion target doesn't exist, we'll just clear the # database. If that doesn't exist either, we still don't error. This is to handle broad requests to delete a # file everywhere. If we're asked to delete it everywhere, then we'll register a job against all hosts. @@ -316,7 +526,6 @@ sub handle_delete { # Um... $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"}); - if ($anvil->data->{jobs}{'job-uuid'}) $anvil->Job->update_progress({ progress => 0, message => "error_0052", @@ -403,10 +612,10 @@ sub handle_rename elsif (-e $anvil->data->{switches}{to}) { # There's already a file (or directory or something) with that name. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052", variables => { file => $anvil->data->{switches}{file}, to => $anvil->data->{switches}{to} }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0056", variables => { file => $anvil->data->{switches}{file}, to => $anvil->data->{switches}{to} }}); $anvil->Job->update_progress({ progress => 0, - message => "error_0052,!!file!".$anvil->data->{switches}{file}."!!,!!to!".$anvil->data->{switches}{to}."!!", + message => "error_0056,!!file!".$anvil->data->{switches}{file}."!!,!!to!".$anvil->data->{switches}{to}."!!", job_uuid => $anvil->data->{jobs}{'job-uuid'}, }); $anvil->nice_exit({exit_code => 6});