* Got more work done on anvil-manage-files. It now picks up new files on nodes/dr hosts in an Anvil! and downloads them if needed.

* Updated anvil-daemon to call anvil-manage-files on a per-minute basis to handle files added outside of the WebUI.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 2 years ago
parent be84a23924
commit e7cf8ac789
  1. 1
      Anvil/Tools.pm
  2. 4
      Anvil/Tools/Database.pm
  3. 2
      share/words.xml
  4. 78
      tools/anvil-daemon
  5. 71
      tools/anvil-manage-files

@ -1111,6 +1111,7 @@ sub _set_paths
'anvil-join-anvil' => "/usr/sbin/anvil-join-anvil",
'anvil-maintenance-mode' => "/usr/sbin/anvil-maintenance-mode",
'anvil-manage-dr' => "/usr/sbin/anvil-manage-dr",
'anvil-manage-files' => "/usr/sbin/anvil-manage-files",
'anvil-manage-firewall' => "/usr/sbin/anvil-manage-firewall",
'anvil-manage-keys' => "/usr/sbin/anvil-manage-keys",
'anvil-manage-power' => "/usr/sbin/anvil-manage-power",

@ -381,7 +381,7 @@ sub backup_database
=head2 check_file_locations
This method checks to see that there is a corresponding entry in C<< file_locations >> for all Anvil! systems and files in the database. Any that are found to be missing will be set to C<< file_location_active >> -> c<< false >>.
This method checks to see that there is a corresponding entry in C<< file_locations >> for all Anvil! systems and files in the database. Any that are found to be missing will be set to C<< file_location_active >> -> c<< true >>.
This method takes no parameters.
@ -420,7 +420,7 @@ sub check_file_locations
debug => $debug,
file_location_file_uuid => $file_uuid,
file_location_anvil_uuid => $anvil_uuid,
file_location_active => 0,
file_location_active => 1,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
}

@ -1670,7 +1670,7 @@ We will keep looking.</key>
<key name="log_0282">Already searched: [#!variable!host_name!# using another IP address, skipping this IP: [#!variable!ip!#].</key>
<key name="log_0283">Done.</key>
<key name="log_0284">[ Error ] - Failed to remove the file: [#!variable!file!#]! Please check the permissions or for SELinux denials.</key>
<key name="log_0285">As requested by another machine, we will now delete the file: [#!variable!file!#].</key>
<key name="log_0285">The file: [#!variable!file!#] is marked as not sync'ed to this Anvil!, removing it now.</key>
<key name="log_0286">[ Error ] - The URL: [#!variable!url!#] to download appears to be invalid.</key>
<key name="log_0287">[ Error ] - The requested URL: [#!variable!url!#] was not found on the remote server.</key>
<key name="log_0288">[ Error ] - The requested URL: [#!variable!url!#] does not resolve to a known domain.</key>

@ -582,7 +582,7 @@ sub handle_periodic_tasks
my $problem = $anvil->Email->check_config({debug => 3});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
# Check if any files have been uploaded to /mnt/shared/incoming on striker
# Check if anything is needed to be done in /mnt/shared.
check_incoming($anvil);
# Check for stale db_in_use states.
@ -823,73 +823,23 @@ AND
return(0);
}
# On dashboards, this checks to see if any files are in /mnt/shared/incoming and, if so, that they've been processed.
# This checks to see if any files in /mnt/shared need to be dealt with, like incorporating files in
# /mnt/shared/incoming, etc.
sub check_incoming
{
my ($anvil) = @_;
my $system_type = $anvil->Get->host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { system_type => $system_type }});
if ($system_type eq "striker")
{
# Look for files in /mnt/shared/incoming that are not yet in the database.
my $directory = $anvil->data->{path}{directories}{shared}{incoming};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { directory => $directory }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { directory => $directory }});
local(*DIRECTORY);
opendir(DIRECTORY, $directory);
while(my $file = readdir(DIRECTORY))
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file => $file }});
next if $file eq ".";
next if $file eq "..";
next if $file =~ /^\./; # This is files being rsync'ed still
my $full_path = $directory."/".$file;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { full_path => $full_path }});
# Skip anything that is not a file.
next if not -f $full_path;
# Is this file already in the DB?
my $query = "SELECT file_uuid FROM files WHERE file_name = ".$anvil->Database->quote($file).";";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
if (not $count)
{
# Add it to the database.
my $size = (stat($full_path))[7];
my $say_size_human = $anvil->Convert->bytes_to_human_readable({'bytes' => $size});
my $say_size_comma = $anvil->Convert->add_commas({number => $size});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
size => $size,
say_size_human => $say_size_human,
say_size_comma => $say_size_comma,
}});
# Register a job to call anvil-sync-shared
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
file => $THIS_FILE,
line => __LINE__,
job_command => $anvil->data->{path}{exe}{'anvil-sync-shared'},
job_data => "file=".$full_path,
job_name => "storage::move_incoming",
job_title => "job_0132",
job_description => "job_0133",
job_progress => 0,
job_host_uuid => $anvil->data->{sys}{host_uuid},
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
}
}
closedir(DIRECTORY);
}
my $shell_call = $anvil->data->{path}{exe}{'anvil-manage-files'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({
shell_call => $shell_call,
source => $THIS_FILE,
line => __LINE__,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
return(0);
}

@ -51,9 +51,6 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
$anvil->Get->switches({list => ["delete", "download", "file", "is-script", "job-uuid", "rename", "remove", "add", "to"], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
@ -154,6 +151,8 @@ sub find_missing_files
my ($anvil) = @_;
# What am I? This will impact how missing files are found.
$anvil->Database->get_anvils();
my $query = "
SELECT
file_uuid,
@ -209,10 +208,67 @@ ORDER BY
my ($found) = find_file($anvil, $file_uuid, $full_path, $file_size, $file_md5sum);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
}
# If we're a Striker, see if any Anvil! systems exist that don't know about this file.
$anvil->Database->check_file_locations({debug => 2});
}
}
else
{
# Check to see if we're supposed to have this file.
$anvil->Database->get_file_locations();
my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
# Nothing to do if we're not in an Anvil! yet.
next if not $anvil_uuid;
# Do we have a file_location_uuid? If not, there will be soon but nothing to do until
# then.
my $file_location_uuid = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
next if not $file_location_uuid;
# Should we have it?
my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_active => $file_location_active }});
if ($file_location_active)
{
# Make sure it exists, and add it if not.
if (not -e $full_path)
{
# Find and copy it.
my ($found) = find_file($anvil, $file_uuid, $full_path, $file_size, $file_md5sum);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
}
}
else
{
# If it exists, remove it.
if (-e $full_path)
{
# Delete it.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0285", variables => { file => $full_path }});
unlink $full_path;
### TODO: Left off here.
# Sleep and verify
sleep 1;
if (-e $full_path)
{
# Failed to delete...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0284", variables => { file => $full_path }});
return("");
}
else
{
# Deleted successfully.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0283"});
}
}
}
}
}
return(0);
}
@ -400,7 +456,7 @@ sub check_incoming
});
my $incoming_directory = $anvil->data->{path}{directories}{shared}{incoming};
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0264"});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0264"});
foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}})
{
# Skip this if it's under '/mnt/shared/temp' (that's used for files being downloaded, etc)
@ -511,6 +567,9 @@ sub check_incoming
}});
if ($full_path =~ /^$incoming_directory/)
{
# Ignore files starting with '.', they're often in-progress rsync's.
next if $file_name =~ /^\./;
# If it's a definition file, we'll move it to
# 'path::directories::shared::definitions', otherwise we'll move it to
# 'path::directories::shared::files'.

Loading…
Cancel
Save