From e7cf8ac789bdaff1b6dfce1050144e349df19658 Mon Sep 17 00:00:00 2001 From: Digimer Date: Tue, 9 Aug 2022 00:08:19 -0400 Subject: [PATCH] * Got more work done on anvil-manage-files. It now picks up new files on nodes/dr hosts in an Anvil! and downloads them if needed. * Updated anvil-daemon to call anvil-manage-files on a per-minute basis to handle files added outside of the WebUI. Signed-off-by: Digimer --- Anvil/Tools.pm | 1 + Anvil/Tools/Database.pm | 4 +-- share/words.xml | 2 +- tools/anvil-daemon | 78 ++++++++-------------------------------- tools/anvil-manage-files | 71 ++++++++++++++++++++++++++++++++---- 5 files changed, 83 insertions(+), 73 deletions(-) diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index f10dd624..aecb7394 100644 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -1111,6 +1111,7 @@ sub _set_paths 'anvil-join-anvil' => "/usr/sbin/anvil-join-anvil", 'anvil-maintenance-mode' => "/usr/sbin/anvil-maintenance-mode", 'anvil-manage-dr' => "/usr/sbin/anvil-manage-dr", + 'anvil-manage-files' => "/usr/sbin/anvil-manage-files", 'anvil-manage-firewall' => "/usr/sbin/anvil-manage-firewall", 'anvil-manage-keys' => "/usr/sbin/anvil-manage-keys", 'anvil-manage-power' => "/usr/sbin/anvil-manage-power", diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 1d41a8ab..217cf7ab 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -381,7 +381,7 @@ sub backup_database =head2 check_file_locations -This method checks to see that there is a corresponding entry in C<< file_locations >> for all Anvil! systems and files in the database. Any that are found to be missing will be set to C<< file_location_active >> -> c<< false >>. +This method checks to see that there is a corresponding entry in C<< file_locations >> for all Anvil! systems and files in the database. Any that are found to be missing will be set to C<< file_location_active >> -> c<< true >>. This method takes no parameters. @@ -420,7 +420,7 @@ sub check_file_locations debug => $debug, file_location_file_uuid => $file_uuid, file_location_anvil_uuid => $anvil_uuid, - file_location_active => 0, + file_location_active => 1, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); } diff --git a/share/words.xml b/share/words.xml index f71947ee..2bec97df 100644 --- a/share/words.xml +++ b/share/words.xml @@ -1670,7 +1670,7 @@ We will keep looking. Already searched: [#!variable!host_name!# using another IP address, skipping this IP: [#!variable!ip!#]. Done. [ Error ] - Failed to remove the file: [#!variable!file!#]! Please check the permissions or for SELinux denials. - As requested by another machine, we will now delete the file: [#!variable!file!#]. + The file: [#!variable!file!#] is marked as not sync'ed to this Anvil!, removing it now. [ Error ] - The URL: [#!variable!url!#] to download appears to be invalid. [ Error ] - The requested URL: [#!variable!url!#] was not found on the remote server. [ Error ] - The requested URL: [#!variable!url!#] does not resolve to a known domain. diff --git a/tools/anvil-daemon b/tools/anvil-daemon index e6a2524c..bcac6dd9 100755 --- a/tools/anvil-daemon +++ b/tools/anvil-daemon @@ -582,7 +582,7 @@ sub handle_periodic_tasks my $problem = $anvil->Email->check_config({debug => 3}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }}); - # Check if any files have been uploaded to /mnt/shared/incoming on striker + # Check if anything is needed to be done in /mnt/shared. check_incoming($anvil); # Check for stale db_in_use states. @@ -823,73 +823,23 @@ AND return(0); } -# On dashboards, this checks to see if any files are in /mnt/shared/incoming and, if so, that they've been processed. +# This checks to see if any files in /mnt/shared need to be dealt with, like incorporating files in +# /mnt/shared/incoming, etc. sub check_incoming { my ($anvil) = @_; - my $system_type = $anvil->Get->host_type(); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { system_type => $system_type }}); - if ($system_type eq "striker") - { - # Look for files in /mnt/shared/incoming that are not yet in the database. - my $directory = $anvil->data->{path}{directories}{shared}{incoming}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { directory => $directory }}); - - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { directory => $directory }}); - local(*DIRECTORY); - opendir(DIRECTORY, $directory); - while(my $file = readdir(DIRECTORY)) - { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file => $file }}); - next if $file eq "."; - next if $file eq ".."; - next if $file =~ /^\./; # This is files being rsync'ed still - my $full_path = $directory."/".$file; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { full_path => $full_path }}); - - # Skip anything that is not a file. - next if not -f $full_path; - - # Is this file already in the DB? - my $query = "SELECT file_uuid FROM files WHERE file_name = ".$anvil->Database->quote($file).";"; - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); - my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); - my $count = @{$results}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - results => $results, - count => $count, - }}); - if (not $count) - { - # Add it to the database. - my $size = (stat($full_path))[7]; - my $say_size_human = $anvil->Convert->bytes_to_human_readable({'bytes' => $size}); - my $say_size_comma = $anvil->Convert->add_commas({number => $size}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - size => $size, - say_size_human => $say_size_human, - say_size_comma => $say_size_comma, - }}); - - # Register a job to call anvil-sync-shared - my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ - file => $THIS_FILE, - line => __LINE__, - job_command => $anvil->data->{path}{exe}{'anvil-sync-shared'}, - job_data => "file=".$full_path, - job_name => "storage::move_incoming", - job_title => "job_0132", - job_description => "job_0133", - job_progress => 0, - job_host_uuid => $anvil->data->{sys}{host_uuid}, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); - } - - } - closedir(DIRECTORY); - } + my $shell_call = $anvil->data->{path}{exe}{'anvil-manage-files'}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); + my ($output, $return_code) = $anvil->System->call({ + shell_call => $shell_call, + source => $THIS_FILE, + line => __LINE__, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + output => $output, + return_code => $return_code, + }}); return(0); } diff --git a/tools/anvil-manage-files b/tools/anvil-manage-files index 9197cbed..31fe37fc 100755 --- a/tools/anvil-manage-files +++ b/tools/anvil-manage-files @@ -51,9 +51,6 @@ if (($running_directory =~ /^\./) && ($ENV{PWD})) my $anvil = Anvil::Tools->new(); -$anvil->Log->level({set => 2}); -$anvil->Log->secure({set => 1}); - $anvil->Get->switches({list => ["delete", "download", "file", "is-script", "job-uuid", "rename", "remove", "add", "to"], man => $THIS_FILE}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); @@ -154,6 +151,8 @@ sub find_missing_files my ($anvil) = @_; # What am I? This will impact how missing files are found. + $anvil->Database->get_anvils(); + my $query = " SELECT file_uuid, @@ -209,11 +208,68 @@ ORDER BY my ($found) = find_file($anvil, $file_uuid, $full_path, $file_size, $file_md5sum); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }}); } + + # If we're a Striker, see if any Anvil! systems exist that don't know about this file. + $anvil->Database->check_file_locations({debug => 2}); + } + else + { + # Check to see if we're supposed to have this file. + $anvil->Database->get_file_locations(); + my $anvil_uuid = $anvil->Cluster->get_anvil_uuid(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); + + # Nothing to do if we're not in an Anvil! yet. + next if not $anvil_uuid; + + # Do we have a file_location_uuid? If not, there will be soon but nothing to do until + # then. + my $file_location_uuid = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }}); + next if not $file_location_uuid; + + # Should we have it? + my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_active => $file_location_active }}); + + if ($file_location_active) + { + # Make sure it exists, and add it if not. + if (not -e $full_path) + { + # Find and copy it. + my ($found) = find_file($anvil, $file_uuid, $full_path, $file_size, $file_md5sum); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }}); + } + } + else + { + # If it exists, remove it. + if (-e $full_path) + { + # Delete it. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0285", variables => { file => $full_path }}); + + unlink $full_path; + + # Sleep and verify + sleep 1; + if (-e $full_path) + { + # Failed to delete... + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0284", variables => { file => $full_path }}); + return(""); + } + else + { + # Deleted successfully. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0283"}); + } + } + } } } - ### TODO: Left off here. - return(0); } @@ -400,7 +456,7 @@ sub check_incoming }); my $incoming_directory = $anvil->data->{path}{directories}{shared}{incoming}; - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0264"}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0264"}); foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}}) { # Skip this if it's under '/mnt/shared/temp' (that's used for files being downloaded, etc) @@ -511,6 +567,9 @@ sub check_incoming }}); if ($full_path =~ /^$incoming_directory/) { + # Ignore files starting with '.', they're often in-progress rsync's. + next if $file_name =~ /^\./; + # If it's a definition file, we'll move it to # 'path::directories::shared::definitions', otherwise we'll move it to # 'path::directories::shared::files'.