|
|
|
#!/usr/bin/perl
|
|
|
|
#
|
|
|
|
# This handles moving around and managing files on Anvil! nodes, DR hosts and Striker dashboards.
|
|
|
|
#
|
|
|
|
# When this is called (periodically by the daemon of after an upload / ISO generation);
|
|
|
|
# - 1. Check 'incoming/' for files. For any found, generate an md5sum and see if the file name and sum match
|
|
|
|
# anything in the database from any host;
|
|
|
|
# - If so, update/add an entry in 'file_locations'
|
|
|
|
# - If not, create a new entry in 'files' and then add the first entry in 'file_locations'
|
|
|
|
# - 2. Check 'file_locations' for any files on this system, and verify they exist still.
|
|
|
|
# - If not, check the other files for one with a matching md5sum. If found, handle as a rename.
|
|
|
|
# - If not found at all, search for the file according to the search rules and copy to here if found.
|
|
|
|
# - If not found anywhere, remove it from 'file_locations' and send an alert.
|
|
|
|
# - If found, check the size. If it differs, recalculate the md5sum.
|
|
|
|
# - 3. If called with '--rename --file <filename> --to <newname>', rename the file and update 'files'.
|
|
|
|
# - 4. If called with '--delete', remove from 'file_locations' and all copies on all systems. This is done by
|
|
|
|
# registering a job against all known hosts. As such, if this is called and the target file doesn't exist,
|
|
|
|
# it just clears the job and then exits.
|
|
|
|
# - 5. If called with '--is-script=[0|1]', mark as 'script' in the 'files' table and set/remove the executable bit.
|
|
|
|
#
|
|
|
|
# Exit codes;
|
|
|
|
# 0 = Normal exit or md5sum of this program changed and it exited to reload.
|
|
|
|
# 1 = No databases available.
|
|
|
|
# 2 = Another process that is still running has picked up this job.
|
|
|
|
# 3 = No filename specified when needed by another switch.
|
|
|
|
# 4 = Request to change the file name but the new name wasn't given.
|
|
|
|
# 5 = The file specified was not found.
|
|
|
|
# 6 = The file to delete is not under '/mnt/shared/'.
|
|
|
|
#
|
|
|
|
# TODO:
|
|
|
|
# - If two Strikers have the same file name, but different sizes, we get into a yo-yo of updating the two
|
|
|
|
# sides. If this happens, we need to rsync the larger one over the smaller one.
|
|
|
|
#
|
|
|
|
# - Create a ".done.<file_name>" when the upload completes so that we know it's time to add it to the database.
|
|
|
|
#
|
|
|
|
# NOTE:
|
|
|
|
# - remove unsyncs, add syncs.
|
|
|
|
#
|
|
|
|
|
|
|
|
use strict;
|
|
|
|
use warnings;
|
|
|
|
use Anvil::Tools;
|
|
|
|
use Data::Dumper;
|
|
|
|
|
|
|
|
# Disable buffering
|
|
|
|
$| = 1;
|
|
|
|
|
|
|
|
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
|
|
|
|
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
|
|
|
|
if (($running_directory =~ /^\./) && ($ENV{PWD}))
|
|
|
|
{
|
|
|
|
$running_directory =~ s/^\./$ENV{PWD}/;
|
|
|
|
}
|
|
|
|
|
|
|
|
my $anvil = Anvil::Tools->new();
|
|
|
|
$anvil->Get->switches({list => ["add", "check", "delete", "download", "file", "is-script", "rename", "remove", "to"], man => $THIS_FILE});
|
|
|
|
|
|
|
|
### TODO: Remove this
|
|
|
|
$anvil->Log->level({set => 2});
|
|
|
|
$anvil->Log->secure({set => 1});
|
|
|
|
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
|
|
|
|
|
|
|
|
# Connect or die
|
|
|
|
$anvil->Database->connect;
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
|
|
|
|
if (not $anvil->data->{sys}{database}{connections})
|
|
|
|
{
|
|
|
|
# No databases, exit.
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0003"});
|
|
|
|
$anvil->nice_exit({exit_code => 1});
|
|
|
|
}
|
|
|
|
|
|
|
|
# If we have a job_uuid, pick it up.
|
|
|
|
if ($anvil->data->{switches}{'job-uuid'})
|
|
|
|
{
|
|
|
|
my $return = $anvil->Job->get_job_details({check => 1, job_uuid => $anvil->data->{switches}{'job-uuid'}});
|
|
|
|
if ($return == 1)
|
|
|
|
{
|
|
|
|
# It's not a UUID.
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
|
|
|
|
$anvil->nice_exit({exit_code => 2});
|
|
|
|
}
|
|
|
|
if ($return == 2)
|
|
|
|
{
|
|
|
|
# This job is being handled by another process that is still alive.
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
|
|
|
|
$anvil->nice_exit({exit_code => 3});
|
|
|
|
}
|
|
|
|
|
|
|
|
# If there's a progress, clear it.
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'job::job_progress' => $anvil->data->{job}{job_progress} }});
|
|
|
|
if ($anvil->data->{job}{job_progress})
|
|
|
|
{
|
|
|
|
$anvil->Job->update_progress({
|
|
|
|
progress => 1,
|
|
|
|
message => "clear",
|
|
|
|
job_uuid => $anvil->data->{jobs}{'job-uuid'},
|
|
|
|
});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# What are we doing?
|
|
|
|
if ($anvil->data->{switches}{'rename'})
|
|
|
|
{
|
|
|
|
handle_rename($anvil);
|
|
|
|
}
|
|
|
|
elsif ($anvil->data->{switches}{'delete'})
|
|
|
|
{
|
|
|
|
handle_delete($anvil);
|
|
|
|
}
|
|
|
|
elsif ($anvil->data->{switches}{'is-script'})
|
|
|
|
{
|
|
|
|
handle_script($anvil);
|
|
|
|
}
|
|
|
|
elsif ($anvil->data->{switches}{check})
|
|
|
|
{
|
|
|
|
# Check for files scheduled for deletion.
|
|
|
|
check_for_deletes($anvil);
|
|
|
|
|
|
|
|
# Check for new files
|
|
|
|
check_incoming($anvil);
|
|
|
|
|
|
|
|
# Check for files we should have but don't yet have.
|
|
|
|
find_missing_files($anvil);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
# Show the list of all files we know about.
|
|
|
|
show_files($anvil);
|
|
|
|
}
|
|
|
|
|
|
|
|
# We're done
|
|
|
|
$anvil->nice_exit({exit_code => 0});
|
|
|
|
|
|
|
|
|
|
|
|
#############################################################################################################
|
|
|
|
# Private functions. #
|
|
|
|
#############################################################################################################
|
|
|
|
|
|
|
|
# This lists all files.
|
|
|
|
sub show_files
|
|
|
|
{
|
|
|
|
my ($anvil) = @_;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return(0);
|
|
|
|
}
|
|
|
|
|
|
|
|
# This looks to see if there are any entries in 'file_locations' for us that, the pointed to file, doesn't
|
|
|
|
# exist on this machine. For those entries, we will search for the file on other machines. The one exception
|
|
|
|
# is server definition files. For those, we write the file out from the server's 'definition' table entry.
|
|
|
|
#
|
|
|
|
# When a missing entry is found, and an entry doesn't exist, the file will be found (if possible) and copied
|
|
|
|
# to this host. Only machines on the same subnet are searched. The search pattern is;
|
|
|
|
#
|
|
|
|
# Nodes; 1. Check for the file on the peer.
|
|
|
|
# 2. Check for the file on Strikers, in alphabetical order.
|
|
|
|
# 3. Check for the file on DR host, if available.
|
|
|
|
# 4. Check other nodes, in alphabetical order.
|
|
|
|
# 5. Check other DR hosts, in alphabetical order.
|
|
|
|
# Striker; 1. Check for the file on other Strikers, in alphabetical order.
|
|
|
|
# 2. Check for the file on DR hosts, if available
|
|
|
|
# 3. Check for the file on Anvil! nodes.
|
|
|
|
# DR Host; 1. Check for the file on Strikers, in alphabetical order.
|
|
|
|
# 2. Check for the file on Anvil! nodes.
|
|
|
|
# * If a file can't be found, it will try again every so often until it is found.
|
|
|
|
# * When a file is found, it is copied to '/mnt/shared/incoming'. Only when the file has arrived and the
|
|
|
|
# md5sum matches. At this point, it is moved into the proper directory.
|
|
|
|
sub find_missing_files
|
|
|
|
{
|
|
|
|
my ($anvil) = @_;
|
|
|
|
|
|
|
|
# What am I? This will impact how missing files are found.
|
|
|
|
$anvil->Database->get_anvils();
|
|
|
|
|
|
|
|
my $host_uuid = $anvil->Get->host_uuid();
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_uuid => $host_uuid }});
|
|
|
|
|
|
|
|
my $query = "
|
|
|
|
SELECT
|
|
|
|
file_uuid,
|
|
|
|
file_directory,
|
|
|
|
file_name,
|
|
|
|
file_size,
|
|
|
|
file_md5sum
|
|
|
|
FROM
|
|
|
|
files
|
|
|
|
WHERE
|
|
|
|
file_type != 'DELETED'
|
|
|
|
ORDER BY
|
|
|
|
file_directory ASC,
|
|
|
|
file_name ASC
|
|
|
|
;";
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
|
|
|
|
|
|
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
|
|
|
|
my $count = @{$results};
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
results => $results,
|
|
|
|
count => $count,
|
|
|
|
}});
|
|
|
|
foreach my $row (@{$results})
|
|
|
|
{
|
|
|
|
my $file_uuid = $row->[0];
|
|
|
|
my $file_directory = $row->[1];
|
|
|
|
my $file_name = $row->[2];
|
|
|
|
my $file_size = $row->[3];
|
|
|
|
my $file_md5sum = $row->[4];
|
|
|
|
my $full_path = $file_directory."/".$file_name;
|
|
|
|
$full_path =~ s/\/\//\//g;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
's1:file_uuid' => $file_uuid,
|
|
|
|
's2:file_directory' => $file_directory,
|
|
|
|
's3:file_name' => $file_name,
|
|
|
|
's4:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
|
|
|
|
's5:file_md5sum' => $file_md5sum,
|
|
|
|
's6:full_path' => $full_path,
|
|
|
|
}});
|
|
|
|
|
|
|
|
# If we're a striker, we want all files.
|
|
|
|
my $host_type = $anvil->Get->host_type();
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
|
|
|
|
if ($host_type eq "striker")
|
|
|
|
{
|
|
|
|
if (not -e $full_path)
|
|
|
|
{
|
|
|
|
# Missing file!
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0269", variables => { file => $full_path }});
|
|
|
|
|
|
|
|
# Find what target, if any, we'll the file from.
|
|
|
|
my ($found) = find_file($anvil, $file_uuid, $full_path, $file_size, $file_md5sum);
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
|
|
|
|
}
|
|
|
|
|
|
|
|
# If we're a Striker, see if any Anvil! systems exist that don't know about this file.
|
|
|
|
$anvil->Database->check_file_locations({debug => 2});
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
# Check to see if we're supposed to have this file.
|
|
|
|
my $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
|
|
|
|
next if not $file_location_uuid;
|
|
|
|
|
|
|
|
# Should we have it?
|
|
|
|
my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_active => $file_location_active }});
|
|
|
|
|
|
|
|
if ($file_location_active)
|
|
|
|
{
|
|
|
|
# Make sure it exists, and add it if not.
|
|
|
|
if (not -e $full_path)
|
|
|
|
{
|
|
|
|
# Find and copy it.
|
|
|
|
my ($found) = find_file($anvil, $file_uuid, $full_path, $file_size, $file_md5sum);
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
# If it exists, remove it.
|
|
|
|
if (-e $full_path)
|
|
|
|
{
|
|
|
|
# Delete it.
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0285", variables => { file => $full_path }});
|
|
|
|
|
|
|
|
unlink $full_path;
|
|
|
|
|
|
|
|
# Sleep and verify
|
|
|
|
sleep 1;
|
|
|
|
if (-e $full_path)
|
|
|
|
{
|
|
|
|
# Failed to delete...
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0284", variables => { file => $full_path }});
|
|
|
|
return("");
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
# Deleted successfully.
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0283"});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return(0);
|
|
|
|
}
|
|
|
|
|
|
|
|
# This looks for a file on another system. The exact order of search depends on what kind of machine we are.
|
|
|
|
sub find_file
|
|
|
|
{
|
|
|
|
my ($anvil, $file_uuid, $full_path, $file_size, $file_md5sum) = @_;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
's1:file_uuid' => $file_uuid,
|
|
|
|
's2:full_path' => $full_path,
|
|
|
|
's3:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
|
|
|
|
's4:file_md5sum' => $file_md5sum,
|
|
|
|
}});
|
|
|
|
|
|
|
|
# Before we do anything, see if the file is shown as being in a directory other than 'files' in the
|
|
|
|
# database, but actually in files on disk. If so, update the database.
|
|
|
|
my $host_type = $anvil->Get->host_type();
|
|
|
|
my $file_directory = $anvil->data->{files}{file_uuid}{$file_uuid}{file_directory};
|
|
|
|
my $file_name = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name};
|
|
|
|
my $files_full_path = $anvil->data->{path}{directories}{shared}{files}."/".$file_name;
|
|
|
|
$files_full_path =~ s/\/\//\//g;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
's1:file_directory' => $file_directory,
|
|
|
|
's2:file_name' => $file_name,
|
|
|
|
's3:files_full_path' => $files_full_path,
|
|
|
|
}});
|
|
|
|
|
|
|
|
if (($full_path ne $files_full_path) && (-e $files_full_path) && ($host_type eq "striker"))
|
|
|
|
{
|
|
|
|
### TODO: Add checks to see if anything other than the directory needs updating.
|
|
|
|
# It's in 'files' now, update the database.
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0725", variables => {
|
|
|
|
file => $full_path,
|
|
|
|
directory => $anvil->data->{path}{directories}{shared}{files},
|
|
|
|
}});
|
|
|
|
$anvil->Database->insert_or_update_files({
|
|
|
|
debug => 2,
|
|
|
|
file_uuid => $file_uuid,
|
|
|
|
file_name => $anvil->data->{files}{file_uuid}{$file_uuid}{file_name},
|
|
|
|
file_directory => $anvil->data->{path}{directories}{shared}{files},
|
|
|
|
file_size => $anvil->data->{files}{file_uuid}{$file_uuid}{file_size},
|
|
|
|
file_md5sum => $anvil->data->{files}{file_uuid}{$file_uuid}{file_md5sum},
|
|
|
|
file_mtime => $anvil->data->{files}{file_uuid}{$file_uuid}{file_mtime},
|
|
|
|
file_type => $anvil->data->{files}{file_uuid}{$file_uuid}{file_type},
|
|
|
|
});
|
|
|
|
|
|
|
|
return(1);
|
|
|
|
}
|
|
|
|
|
|
|
|
# We want to search Striker's first, DR hosts second and nodes third.
|
|
|
|
$anvil->Database->get_hosts;
|
|
|
|
my $host_order = [];
|
|
|
|
foreach my $type ("striker", "dr", "node")
|
|
|
|
{
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { type => $type }});
|
|
|
|
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
|
|
|
|
{
|
|
|
|
my $host_uuid = $anvil->data->{sys}{hosts}{by_name}{$host_name};
|
|
|
|
my $host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type};
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
's1:host_name' => $host_name,
|
|
|
|
's2:host_type' => $host_type,
|
|
|
|
's3:host_uuid' => $host_uuid,
|
|
|
|
}});
|
|
|
|
|
|
|
|
next if $host_type ne $type;
|
|
|
|
next if $host_uuid eq $anvil->Get->host_uuid;
|
|
|
|
|
|
|
|
push @{$host_order}, $host_uuid;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_uuid => $host_uuid }});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# Now search.
|
|
|
|
my $found_on_host = 0; # Found the file on a target
|
|
|
|
my $file_found = 0; # Actually have the file locally.
|
|
|
|
foreach my $search_host_uuid (@{$host_order})
|
|
|
|
{
|
|
|
|
last if $file_found;
|
|
|
|
my $target_host = $anvil->data->{hosts}{host_uuid}{$search_host_uuid}{short_host_name};
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
's1:target_host' => $target_host,
|
|
|
|
's2:search_host_uuid' => $search_host_uuid,
|
|
|
|
}});
|
|
|
|
|
|
|
|
my $target_ip = $anvil->Network->find_target_ip({host_uuid => $search_host_uuid});
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_ip => $target_ip }});
|
|
|
|
next if not $target_ip;
|
|
|
|
|
|
|
|
# See if the file is on the target and, if so, if it matches.
|
|
|
|
### NOTE: If we want to use md5sum again, use '--with-md5sum'
|
|
|
|
my $shell_call = $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path.$anvil->Log->switches;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
|
|
|
|
|
|
|
|
# Test access
|
|
|
|
my $access = $anvil->Remote->test_access({target => $target_ip});
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
|
|
|
|
next if not $access;
|
|
|
|
|
|
|
|
my $remote_size = 0;
|
|
|
|
my ($output, $error, $return_code) = $anvil->Remote->call({
|
|
|
|
shell_call => $shell_call,
|
|
|
|
target => $target_ip,
|
|
|
|
});
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
error => $error,
|
|
|
|
output => $output,
|
|
|
|
return_code => $return_code,
|
|
|
|
}});
|
|
|
|
foreach my $line (split/\n/, $output)
|
|
|
|
{
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
|
|
|
|
if ($line =~ /^size: \[(\d+)\]$/)
|
|
|
|
{
|
|
|
|
$remote_size = $1;
|
|
|
|
$found_on_host = 1;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
remote_size => $remote_size,
|
|
|
|
found_on_host => $found_on_host,
|
|
|
|
}});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
's1:found_on_host' => $found_on_host,
|
|
|
|
's2:remote_size' => $remote_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}).")",
|
|
|
|
's3:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
|
|
|
|
}});
|
|
|
|
next if not $found_on_host;
|
|
|
|
|
|
|
|
if ($remote_size eq $file_size)
|
|
|
|
{
|
|
|
|
# Pull it over!
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0276", variables => {
|
|
|
|
file => $full_path,
|
|
|
|
host_name => $target_host,
|
|
|
|
ip => $target_ip,
|
|
|
|
}});
|
|
|
|
my $failed = $anvil->Storage->rsync({
|
|
|
|
debug => 2,
|
|
|
|
destination => $full_path,
|
|
|
|
source => "root\@".$target_ip.":".$full_path,
|
|
|
|
});
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
|
|
|
|
|
|
|
|
if (-f $full_path)
|
|
|
|
{
|
|
|
|
# Got it!
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0277", variables => { file => $full_path }});
|
|
|
|
|
|
|
|
# Verify the md5sum.
|
|
|
|
my $local_md5sum = $anvil->Get->md5sum({file => $full_path});
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
local_md5sum => $local_md5sum,
|
|
|
|
file_md5sum => $file_md5sum,
|
|
|
|
}});
|
|
|
|
if ($local_md5sum eq $file_md5sum)
|
|
|
|
{
|
|
|
|
# Success!
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0278", variables => { file => $full_path }});
|
|
|
|
|
|
|
|
$file_found = 1;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_found => $file_found }});
|
|
|
|
last;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
# Failed. :(
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0279", variables => { file => $full_path }});
|
|
|
|
unlink $full_path;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
# Failed to rsync.
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0280", variables => {
|
|
|
|
file => $full_path,
|
|
|
|
host_name => $target_host,
|
|
|
|
ip => $target_ip,
|
|
|
|
}});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
# Doesn't match what we're looking for.
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0281", variables => {
|
|
|
|
file => $full_path,
|
|
|
|
host_name => $target_host,
|
|
|
|
ip => $target_ip,
|
|
|
|
file_size => $file_size,
|
|
|
|
say_file_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
|
|
|
|
remote_size => $remote_size,
|
|
|
|
say_remote_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}),
|
|
|
|
}});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_found => $file_found }});
|
|
|
|
return($file_found);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
# This handles deleting a file.
|
|
|
|
sub check_incoming
|
|
|
|
{
|
|
|
|
my ($anvil) = @_;
|
|
|
|
|
|
|
|
# This hash shouldn't exist, but lets be safe...
|
|
|
|
if (exists $anvil->data->{scan}{directories})
|
|
|
|
{
|
|
|
|
delete $anvil->data->{scan}{directories};
|
|
|
|
}
|
|
|
|
|
|
|
|
# Read any files in '/mnt/shared/incoming'.
|
|
|
|
$anvil->Storage->scan_directory({
|
|
|
|
debug => 2,
|
|
|
|
directory => $anvil->data->{path}{directories}{shared}{base},
|
|
|
|
recursive => 1,
|
|
|
|
});
|
|
|
|
|
|
|
|
my $archives_directory = $anvil->data->{path}{directories}{shared}{archives};
|
|
|
|
my $definitions_directory = $anvil->data->{path}{directories}{shared}{definitions};
|
|
|
|
my $incoming_directory = $anvil->data->{path}{directories}{shared}{incoming};
|
|
|
|
my $provision_directory = $anvil->data->{path}{directories}{shared}{provision};
|
|
|
|
my $temp_directory = $anvil->data->{path}{directories}{shared}{temp};
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
archives_directory => $archives_directory,
|
|
|
|
definitions_directory => $definitions_directory,
|
|
|
|
incoming_directory => $incoming_directory,
|
|
|
|
provision_directory => $provision_directory,
|
|
|
|
temp_directory => $temp_directory,
|
|
|
|
}});
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0264"});
|
|
|
|
foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}})
|
|
|
|
{
|
|
|
|
# Skip this if it's under a directory managed elsewhere, or that we don't care about.
|
|
|
|
next if $full_path =~ /^$temp_directory/;
|
|
|
|
next if $full_path =~ /^$provision_directory/;
|
|
|
|
next if $full_path =~ /^$definitions_directory/;
|
|
|
|
|
|
|
|
# Skip if this isn't a file.
|
|
|
|
my $file_type = $anvil->data->{scan}{directories}{$full_path}{type};
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
full_path => $full_path,
|
|
|
|
file_type => $file_type,
|
|
|
|
}});
|
|
|
|
next if $file_type ne "file";
|
|
|
|
|
|
|
|
my $file_name = $anvil->data->{scan}{directories}{$full_path}{name};
|
|
|
|
my $file_directory = $anvil->data->{scan}{directories}{$full_path}{directory};
|
|
|
|
my $file_size = $anvil->data->{scan}{directories}{$full_path}{size};
|
|
|
|
my $file_mtime = $anvil->data->{scan}{directories}{$full_path}{mtime};
|
|
|
|
my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype};
|
|
|
|
my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0;
|
|
|
|
my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable);
|
|
|
|
my $full_path = $file_directory."/".$file_name;
|
|
|
|
$full_path =~ s/\/\//\//g;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
file_name => $file_name,
|
|
|
|
file_size => $file_size,
|
|
|
|
file_mtime => $file_mtime,
|
|
|
|
file_mimetype => $file_mimetype,
|
|
|
|
file_executable => $file_executable,
|
|
|
|
say_mimetype => $say_mimetype,
|
|
|
|
full_path => $full_path,
|
|
|
|
}});
|
|
|
|
|
|
|
|
# Skip dot-files, they're usually files being uploaded.
|
|
|
|
next if $file_name =~ /^\./;
|
|
|
|
|
|
|
|
# Do I know about this file? If so, is the file the same size? If either is no, calculate the md5sum.
|
|
|
|
my ($file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum) = get_file_db_info($anvil, "", $file_name);
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
file_uuid => $file_uuid,
|
|
|
|
recorded_size => $recorded_size,
|
|
|
|
recorded_mtime => $recorded_mtime,
|
|
|
|
recorded_md5sum => $recorded_md5sum,
|
|
|
|
}});
|
|
|
|
|
|
|
|
# Calculate the md5sum?
|
|
|
|
my $added_to_db = 0;
|
|
|
|
my $file_md5sum = $recorded_md5sum;
|
|
|
|
if ((not $file_uuid) or ($file_size != $recorded_size))
|
|
|
|
{
|
This commit has more changes than I would normally like, but it's all linked to changing file uploads to rsync serially.
* To update file handling for the new DR host linking mechanism, file_locations -> file_location_anvil_uuid was changed to file_location_host_uuid.
This required a fair number of changes elsewhere to handle this, with a particular noted change to Database->get_anvils() to look at host_uuid's for the subnodes in an Anvil! and, if either is marked as needing a file, make sure the peer is as well. Similarly, any linked DRs are set to have the file as well.
* Created a new Network->find_access that simply takes a target host name or UUID, and it returns a list of networks and IPs that the target can be accessed by.
* Updated Network->load_ips() to find the network interface being used for traffic so that things like the interface speed can be recorded, even when an IP is on a bridge or bond.
Unrelated, but in this commit, is a restoration of calling scan agents with a timeout now that the virsh hang issue has been resolved.
Signed-off-by: digimer <mkelly@alteeve.ca>
2 years ago
|
|
|
# It's possible the file is still uploading, so sleep for a bit and see if the
|
|
|
|
# size is still changing.
|
|
|
|
my $last_size = $file_size;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
last_size => $last_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}).")",
|
|
|
|
}});
|
|
|
|
$anvil->Storage->_wait_if_changing({
|
|
|
|
file => $full_path,
|
|
|
|
last_size => $file_size,
|
|
|
|
});
|
|
|
|
|
|
|
|
# Yes, caluclate md5sum, but first, do we have a size mismatch? If so, see if we need
|
|
|
|
# to pull a newer version down from elsewhere.
|
|
|
|
if (($file_uuid) && ($file_mtime <= $recorded_mtime))
|
|
|
|
{
|
|
|
|
# We've got an older file, we need to update.
|
|
|
|
pull_file($anvil, $file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum);
|
|
|
|
|
|
|
|
# TODO: Now see if it exists and, if it does, re-stat it. If not, loop to the
|
|
|
|
# next file and skip this one.
|
|
|
|
}
|
|
|
|
|
|
|
|
# Now generate the md5sum. If this file is over 128 MiB, warn the user that it might
|
|
|
|
# take a while.
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0265", variables => { file => $full_path }});
|
|
|
|
if ($file_size > (128 * (2 ** 20)))
|
|
|
|
{
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0266", variables => {
|
|
|
|
size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
|
|
|
|
}});
|
|
|
|
}
|
|
|
|
|
|
|
|
# Update (or get) the md5sum.
|
|
|
|
$file_md5sum = $anvil->Get->md5sum({debug => 2, file => $full_path});
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
say_mimetype => $say_mimetype,
|
|
|
|
file_md5sum => $file_md5sum,
|
|
|
|
}});
|
|
|
|
|
|
|
|
# Insert or update the files entry.
|
|
|
|
($file_uuid) = $anvil->Database->insert_or_update_files({
|
|
|
|
debug => 2,
|
|
|
|
file_uuid => $file_uuid,
|
|
|
|
file_name => $file_name,
|
|
|
|
file_directory => $file_directory,
|
|
|
|
file_size => $file_size,
|
|
|
|
file_md5sum => $file_md5sum,
|
|
|
|
file_mtime => $file_mtime,
|
|
|
|
file_type => $say_mimetype,
|
|
|
|
});
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
|
|
|
|
|
|
|
|
if ($file_uuid)
|
|
|
|
{
|
|
|
|
$added_to_db = 1;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { added_to_db => $added_to_db }});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# If we still don't have a file UUID for some reason, skip this file.
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
|
|
|
|
next if not $file_uuid;
|
|
|
|
|
|
|
|
# Are we in the incoming directory? If so, move the file.
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
full_path => $full_path,
|
|
|
|
incoming_directory => $incoming_directory,
|
|
|
|
}});
|
|
|
|
if ($full_path =~ /^$incoming_directory/)
|
|
|
|
{
|
|
|
|
# Ignore files starting with '.', they're often in-progress rsync's.
|
|
|
|
next if $file_name =~ /^\./;
|
|
|
|
|
|
|
|
# If it's a definition file, we'll move it to
|
|
|
|
# 'path::directories::shared::definitions', otherwise we'll move it to
|
|
|
|
# 'path::directories::shared::files'.
|
|
|
|
my $target = $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files};
|
|
|
|
$target .= "/";
|
|
|
|
$target =~ s/\/\//\//g;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }});
|
|
|
|
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0268", variables => {
|
|
|
|
file => $full_path,
|
|
|
|
target => $target,
|
|
|
|
}});
|
|
|
|
|
|
|
|
my $problem = $anvil->Storage->move_file({
|
|
|
|
debug => 2,
|
|
|
|
source_file => $full_path,
|
|
|
|
target_file => $target,
|
|
|
|
});
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
|
|
|
|
if (not $problem)
|
|
|
|
{
|
|
|
|
$full_path = $target."/".$file_name;
|
|
|
|
$file_directory = $target;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
file_directory => $file_directory,
|
|
|
|
full_path => $full_path,
|
|
|
|
}});
|
|
|
|
}
|
|
|
|
|
|
|
|
# Update the file_directory.
|
|
|
|
$target =~ s/\/$//;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }});
|
|
|
|
($file_uuid) = $anvil->Database->insert_or_update_files({
|
|
|
|
debug => 2,
|
|
|
|
file_uuid => $file_uuid,
|
|
|
|
file_name => $file_name,
|
|
|
|
file_directory => $target,
|
|
|
|
file_size => $file_size,
|
|
|
|
file_md5sum => $file_md5sum,
|
|
|
|
file_mtime => $file_mtime,
|
|
|
|
file_type => $say_mimetype,
|
|
|
|
});
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
|
|
|
|
}
|
|
|
|
|
|
|
|
if ($added_to_db)
|
|
|
|
{
|
|
|
|
# Push it to all other hosts now.
|
|
|
|
$anvil->Storage->push_file({
|
|
|
|
debug => 2,
|
|
|
|
file => $full_path,
|
|
|
|
file_uuid => $file_uuid,
|
|
|
|
});
|
|
|
|
|
|
|
|
# Call track_files, it'll make sure the file_locations are setup.
|
|
|
|
$anvil->Database->track_files({debug => 2});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return(0);
|
|
|
|
}
|
|
|
|
|
|
|
|
# This method finds a file elsewhere on the network and pulls it to here.
|
|
|
|
sub pull_file
|
|
|
|
{
|
|
|
|
my ($anvil, $file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum) = @_;
|
|
|
|
$file_uuid = "" if not defined $file_uuid;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
file_uuid => $file_uuid,
|
|
|
|
recorded_size => $recorded_size,
|
|
|
|
recorded_mtime => $recorded_mtime,
|
|
|
|
recorded_md5sum => $recorded_md5sum,
|
|
|
|
}});
|
|
|
|
|
|
|
|
# Find the hosts with this file, then connect to it to see if the size is the same as what we want.
|
|
|
|
# If so, pull it down...
|
|
|
|
### TODO
|
|
|
|
|
|
|
|
return(0);
|
|
|
|
}
|
|
|
|
|
|
|
|
# This gets the file_uuid for a given file name and/or md5sum. If the file isn't found, an empty string is
|
|
|
|
# returned. If it is found, the file size as recorded in the database is returned.
|
|
|
|
sub get_file_db_info
|
|
|
|
{
|
|
|
|
my ($anvil, $file_md5sum, $file_name) = @_;
|
|
|
|
$file_md5sum = "" if not defined $file_md5sum;
|
|
|
|
$file_name = "" if not defined $file_name;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
|
|
|
|
file_md5sum => $file_md5sum,
|
|
|
|
file_name => $file_name,
|
|
|
|
}});
|
|
|
|
|
|
|
|
# Get the file size and file uuid, if possible.
|
|
|
|
|
|
|
|
# If I have the md5sum, search using that. If I have the filename only, then we'll fall back to that.
|
|
|
|
my $query = "
|
|
|
|
SELECT
|
|
|
|
file_uuid,
|
|
|
|
file_size,
|
|
|
|
file_mtime,
|
|
|
|
file_md5sum
|
|
|
|
FROM
|
|
|
|
files
|
|
|
|
WHERE
|
|
|
|
";
|
|
|
|
if ($file_md5sum)
|
|
|
|
{
|
|
|
|
$query .= " file_md5sum = ".$anvil->Database->quote($file_md5sum)."\n";
|
|
|
|
}
|
|
|
|
elsif ($file_name)
|
|
|
|
{
|
|
|
|
$query .= " file_name = ".$anvil->Database->quote($file_name)."\n";
|
|
|
|
}
|
|
|
|
$query .= ";";
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
|
|
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
|
|
|
|
my $count = @{$results};
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
results => $results,
|
|
|
|
count => $count,
|
|
|
|
}});
|
|
|
|
if (not $count)
|
|
|
|
{
|
|
|
|
# File wasn't found in the database
|
|
|
|
return("", 0, 0, "");
|
|
|
|
}
|
|
|
|
my $file_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : "";
|
|
|
|
my $file_size = defined $results->[0]->[1] ? $results->[0]->[1] : 0;
|
|
|
|
my $file_mtime = defined $results->[0]->[2] ? $results->[0]->[2] : 0;
|
|
|
|
$file_md5sum = defined $results->[0]->[3] ? $results->[0]->[3] : "";
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
file_uuid => $file_uuid,
|
|
|
|
file_size => $file_size,
|
|
|
|
file_mtime => $file_mtime,
|
|
|
|
file_md5sum => $file_md5sum,
|
|
|
|
}});
|
|
|
|
|
|
|
|
return($file_uuid, $file_size, $file_mtime, $file_md5sum);
|
|
|
|
}
|
|
|
|
|
|
|
|
# This handles toggling a file to marked or unmarked as a script.
|
|
|
|
sub handle_script
|
|
|
|
{
|
|
|
|
my ($anvil) = @_;
|
|
|
|
|
|
|
|
if (not $anvil->data->{switches}{file})
|
|
|
|
{
|
|
|
|
# Um...
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0055"});
|
|
|
|
$anvil->Job->update_progress({
|
|
|
|
progress => 0,
|
|
|
|
message => "error_0055",
|
|
|
|
job_uuid => $anvil->data->{jobs}{'job-uuid'},
|
|
|
|
});
|
|
|
|
$anvil->nice_exit({exit_code => 3});
|
|
|
|
}
|
|
|
|
|
|
|
|
# Find the file_uuid (we don't actually care about the file size, mtime or md5sum).
|
|
|
|
my ($file_uuid, $file_size, $recorded_mtime, $file_md5sum) = get_file_db_info($anvil, "", $anvil->data->{switches}{file});
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
|
|
|
|
|
|
|
|
# Toggle the executable bits.
|
|
|
|
my $executable = 0;
|
|
|
|
if ($anvil->data->{switches}{'is-script'})
|
|
|
|
{
|
|
|
|
# Is it already executable?
|
|
|
|
if (-x $anvil->data->{switches}{file})
|
|
|
|
{
|
|
|
|
# Switch it on
|
|
|
|
$executable = 1;
|
|
|
|
$anvil->Storage->change_mode({path => $anvil->data->{switches}{file}, mode => "a+x"});
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
# Already a script.
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
# Is it executable?
|
|
|
|
if (-x $anvil->data->{switches}{file})
|
|
|
|
{
|
|
|
|
# Switch it off.
|
|
|
|
$executable = 1;
|
|
|
|
$anvil->Storage->change_mode({path => $anvil->data->{switches}{file}, mode => "a-x"});
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
# Already not a script.
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# If we have a file UUID, update the 'file_type
|
|
|
|
if ($file_uuid)
|
|
|
|
{
|
|
|
|
# Load the details.
|
|
|
|
my $query = "
|
|
|
|
SELECT
|
|
|
|
file_name,
|
|
|
|
file_directory,
|
|
|
|
file_size,
|
|
|
|
file_md5sum,
|
|
|
|
file_type,
|
|
|
|
file_mtime
|
|
|
|
FROM
|
|
|
|
files
|
|
|
|
WHERE
|
|
|
|
file_uuid = ".$anvil->Database->quote($file_uuid)."
|
|
|
|
;";
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
|
|
|
|
|
|
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results }});
|
|
|
|
my $file_name = $results->[0]->[0];
|
|
|
|
my $file_directory = $results->[0]->[1];
|
|
|
|
my $file_size = $results->[0]->[2];
|
|
|
|
my $file_md5sum = $results->[0]->[3];
|
|
|
|
my $file_type = $results->[0]->[4];
|
|
|
|
my $file_mtime = $results->[0]->[5];
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
file_name => $file_name,
|
|
|
|
file_directory => $file_directory,
|
|
|
|
file_size => $file_size,
|
|
|
|
file_md5sum => $file_md5sum,
|
|
|
|
file_type => $file_type,
|
|
|
|
file_mtime => $file_mtime,
|
|
|
|
}});
|
|
|
|
if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'}))
|
|
|
|
{
|
|
|
|
# Figure out what the file type is and update.
|
|
|
|
my $mimetype = mimetype($file_name);
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mimetype => $mimetype }});
|
|
|
|
|
|
|
|
my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, 0);
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }});
|
|
|
|
$anvil->Database->insert_or_update_files({
|
|
|
|
debug => 2,
|
|
|
|
file_uuid => $file_uuid,
|
|
|
|
file_name => $anvil->data->{switches}{file},
|
|
|
|
file_directory => $file_directory,
|
|
|
|
file_size => $file_size,
|
|
|
|
file_md5sum => $file_md5sum,
|
|
|
|
file_mtime => $file_mtime,
|
|
|
|
file_type => $say_mimetype,
|
|
|
|
});
|
|
|
|
}
|
|
|
|
elsif (($file_type ne "script") && ($anvil->data->{switches}{'is-script'}))
|
|
|
|
{
|
|
|
|
# Change the file tpye to "script"
|
|
|
|
$anvil->Database->insert_or_update_files({
|
|
|
|
debug => 2,
|
|
|
|
file_uuid => $file_uuid,
|
|
|
|
file_name => $anvil->data->{switches}{file},
|
|
|
|
file_directory => $file_directory,
|
|
|
|
file_size => $file_size,
|
|
|
|
file_md5sum => $file_md5sum,
|
|
|
|
file_mtime => $file_mtime,
|
|
|
|
file_type => "script",
|
|
|
|
});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return(0);
|
|
|
|
}
|
|
|
|
|
|
|
|
# This takes the mimetype as reported by the 'mimetype' method and returns a file type we care about.
|
|
|
|
sub convert_mimetype
|
|
|
|
{
|
|
|
|
my ($anvil, $mimetype, $file, $executable) = @_;
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
|
|
|
|
mimetype => $mimetype,
|
|
|
|
file => $file,
|
|
|
|
executable => $executable,
|
|
|
|
}});
|
|
|
|
|
|
|
|
my $say_mimetype = "other";
|
|
|
|
if ($mimetype)
|
|
|
|
{
|
|
|
|
if ($mimetype =~ /cd-image/)
|
|
|
|
{
|
|
|
|
$say_mimetype = "iso";
|
|
|
|
}
|
|
|
|
elsif ($mimetype =~ /xml/)
|
|
|
|
{
|
|
|
|
# This might be a definition, but look inside it to be sure.
|
|
|
|
my ($is_domain, $return_code) = $anvil->System->call({debug => 3, shell_call => "if \$(".$anvil->data->{path}{exe}{'grep'}." -q '</domain>' ".$file."); then ".$anvil->data->{path}{exe}{echo}." 1; else ".$anvil->data->{path}{exe}{echo}." 0; fi" });
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { is_domain => $is_domain, return_code => $return_code }});
|
|
|
|
|
|
|
|
if ($is_domain)
|
|
|
|
{
|
|
|
|
$say_mimetype = "definition";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
elsif ($mimetype =~ /rpm$/)
|
|
|
|
{
|
|
|
|
$say_mimetype = "rpm";
|
|
|
|
}
|
|
|
|
elsif ($mimetype =~ /disk-image/)
|
|
|
|
{
|
|
|
|
$say_mimetype = "disk-image";
|
|
|
|
}
|
|
|
|
elsif ($executable)
|
|
|
|
{
|
|
|
|
$say_mimetype = "script";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { say_mimetype => $say_mimetype }});
|
|
|
|
return($say_mimetype);
|
|
|
|
}
|
|
|
|
|
|
|
|
# This looks for any files with the file_type of 'DELETED'. Any that are found will be deleted.
|
|
|
|
sub check_for_deletes
|
|
|
|
{
|
|
|
|
my ($anvil) = @_;
|
|
|
|
|
|
|
|
# Get a list of files.
|
|
|
|
my $query = "SELECT file_uuid, file_directory || '/' || file_name AS full_path FROM files WHERE file_type = 'DELETED';";
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
|
|
|
|
|
|
# This will be used to make sure we only delete from files
|
|
|
|
my $files_directory = $anvil->data->{path}{directories}{shared}{files};
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { files_directory => $files_directory }});
|
|
|
|
|
|
|
|
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
|
|
|
|
my $count = @{$results};
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
results => $results,
|
|
|
|
count => $count,
|
|
|
|
}});
|
|
|
|
foreach my $row (@{$results})
|
|
|
|
{
|
|
|
|
my $file_uuid = $row->[0];
|
|
|
|
my $full_path = $row->[1];
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
's1:file_uuid' => $file_uuid,
|
|
|
|
's2:full_path' => $full_path,
|
|
|
|
}});
|
|
|
|
|
|
|
|
# Only delete if the file is in the shared directory.
|
|
|
|
next if $full_path !~ /^$files_directory/;
|
|
|
|
|
|
|
|
# Get rid of it (if it actually exists).
|
|
|
|
if (-e $full_path)
|
|
|
|
{
|
|
|
|
# Delete it.
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0285", variables => { file => $full_path }});
|
|
|
|
|
|
|
|
unlink $full_path;
|
|
|
|
|
|
|
|
# Sleep and verify
|
|
|
|
sleep 1;
|
|
|
|
if (-e $full_path)
|
|
|
|
{
|
|
|
|
# Failed to delete...
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0284", variables => { file => $full_path }});
|
|
|
|
return("");
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
# Deleted successfully.
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0283"});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# If we're a Striker, check for any file_locations that point to this file and DELETE them.
|
|
|
|
my $host_type = $anvil->Get->host_type();
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
|
|
|
|
if ($host_type eq "striker")
|
|
|
|
{
|
|
|
|
my $query = "SELECT file_location_uuid FROM file_locations WHERE file_location_file_uuid = ".$anvil->Database->quote($file_uuid).";";
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
|
|
|
|
|
|
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
|
|
|
|
my $count = @{$results};
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
results => $results,
|
|
|
|
count => $count,
|
|
|
|
}});
|
|
|
|
foreach my $row (@{$results})
|
|
|
|
{
|
|
|
|
my $file_location_uuid = $row->[0];
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
|
|
|
|
|
|
|
|
# Delete the entry from file_locations, if needed.
|
|
|
|
my $query = "DELETE FROM history.file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";";
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
|
|
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
|
|
|
|
|
|
|
|
$query = "DELETE FROM file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";";
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
|
|
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return(0);
|
|
|
|
}
|
|
|
|
|
|
|
|
# This handles deleting a file. If the requested deletion target doesn't exist, we'll just clear the
|
|
|
|
# database. If that doesn't exist either, we still don't error. This is to handle broad requests to delete a
|
|
|
|
# file everywhere. If we're asked to delete it everywhere, then we'll register a job against all hosts.
|
|
|
|
sub handle_delete
|
|
|
|
{
|
|
|
|
my ($anvil) = @_;
|
|
|
|
|
|
|
|
my $full_path = $anvil->data->{switches}{file};
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { full_path => $full_path }});
|
|
|
|
if (not $full_path)
|
|
|
|
{
|
|
|
|
# Um...
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"});
|
|
|
|
$anvil->Job->update_progress({
|
|
|
|
progress => 100,
|
|
|
|
message => "error_0052",
|
|
|
|
job_uuid => $anvil->data->{jobs}{'job-uuid'},
|
|
|
|
});
|
|
|
|
$anvil->nice_exit({exit_code => 3});
|
|
|
|
}
|
|
|
|
elsif ($full_path !~ /^\/mnt\/shared\//)
|
|
|
|
{
|
|
|
|
# We don't do that here...
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0053", variables => { file => $full_path }});
|
|
|
|
$anvil->Job->update_progress({
|
|
|
|
progress => 100,
|
|
|
|
message => "error_0053,!!file!".$full_path."!!",
|
|
|
|
job_uuid => $anvil->data->{jobs}{'job-uuid'},
|
|
|
|
});
|
|
|
|
$anvil->nice_exit({exit_code => 6});
|
|
|
|
}
|
|
|
|
|
|
|
|
# What's the UUID of this file? I collect all the data for the update, if appropriate.
|
|
|
|
my $file_name = ($full_path =~ /^.*\/(.*)$/)[0];
|
|
|
|
my $file_directory = ($full_path =~ /^(.*?)\/$file_name$/)[0];
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
full_name => $file_name,
|
|
|
|
file_directory => $file_directory,
|
|
|
|
}});
|
|
|
|
my $query = "
|
|
|
|
SELECT
|
|
|
|
file_uuid,
|
|
|
|
file_size,
|
|
|
|
file_md5sum,
|
|
|
|
file_type,
|
|
|
|
file_mtime
|
|
|
|
FROM
|
|
|
|
files
|
|
|
|
WHERE
|
|
|
|
file_name = ".$anvil->Database->quote($file_name)."
|
|
|
|
AND
|
|
|
|
file_directory = ".$anvil->Database->quote($file_directory)."
|
|
|
|
;";
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
|
|
|
|
|
|
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
|
|
|
|
my $count = @{$results};
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
results => $results,
|
|
|
|
count => $count,
|
|
|
|
}});
|
|
|
|
|
|
|
|
my $file_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : "";
|
|
|
|
my $file_size = defined $results->[0]->[1] ? $results->[0]->[1] : "";
|
|
|
|
my $file_md5sum = defined $results->[0]->[2] ? $results->[0]->[2] : "";
|
|
|
|
my $file_type = defined $results->[0]->[3] ? $results->[0]->[3] : "";
|
|
|
|
my $file_mtime = defined $results->[0]->[4] ? $results->[0]->[4] : "";
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
|
|
|
|
file_uuid => $file_uuid,
|
|
|
|
file_size => $file_size,
|
|
|
|
file_md5sum => $file_md5sum,
|
|
|
|
file_type => $file_type,
|
|
|
|
file_mtime => $file_mtime,
|
|
|
|
}});
|
|
|
|
|
|
|
|
# If I have the file_uuid, and we've been asked to delete it everywhere, mark it as DELETEd in the
|
|
|
|
# database. This way, if anything goes wrong below, future runs will try (again) to delete the file
|
|
|
|
# itself. If it's only beind deleted locally, and it fails for some reason, there will be no attempt
|
|
|
|
# to try again.
|
|
|
|
if ($file_uuid)
|
|
|
|
{
|
|
|
|
# Yup.
|
|
|
|
($file_uuid) = $anvil->Database->insert_or_update_files({
|
|
|
|
debug => 2,
|
|
|
|
file_uuid => $file_uuid,
|
|
|
|
file_name => $file_name,
|
|
|
|
file_directory => $file_directory,
|
|
|
|
file_size => $file_size,
|
|
|
|
file_md5sum => $file_md5sum,
|
|
|
|
file_type => "DELETED",
|
|
|
|
file_mtime => $file_mtime,
|
|
|
|
});
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
|
|
|
|
}
|
|
|
|
|
|
|
|
# Does the file exist?
|
|
|
|
if (-e $full_path)
|
|
|
|
{
|
|
|
|
# Delete it.
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0263", variables => { file => $full_path }});
|
|
|
|
|
|
|
|
unlink $full_path;
|
|
|
|
|
|
|
|
# Sleep and verify
|
|
|
|
sleep 1;
|
|
|
|
if (-e $full_path)
|
|
|
|
{
|
|
|
|
# Failed to delete...
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0284"});
|
|
|
|
return("");
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
# Deleted successfully.
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0283"});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# Now, if I have a file_uuid, delete it from file_locations if it exists. It's ok if peers in an
|
|
|
|
# Anvil! haven't deleted yet, as they'll see the file marked as 'DELETED' and purge their local
|
|
|
|
# copies.
|
|
|
|
if ($file_uuid)
|
|
|
|
{
|
|
|
|
# Delete the entry from file_locations, if needed.
|
|
|
|
my $query = "DELETE FROM file_locations WHERE file_location_file_uuid = ".$anvil->Database->quote($file_uuid).";";
|
|
|
|
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
|
|
|
|
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
|
|
|
|
}
|
|
|
|
|
|
|
|
return(0);
|
|
|
|
}
|
|
|
|
|
|
|
|
# This handles renaming files.
|
|
|
|
sub handle_rename
|
|
|
|
{
|
|
|
|
my ($anvil) = @_;
|
|
|
|
|
|
|
|
# Do we have the current file name and the new one?
|
|
|
|
if (not $anvil->data->{switches}{file})
|
|
|
|
{
|
|
|
|
# Um...
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0049"});
|
|
|
|
$anvil->Job->update_progress({
|
|
|
|
progress => 0,
|
|
|
|
message => "error_0049",
|
|
|
|
job_uuid => $anvil->data->{jobs}{'job-uuid'},
|
|
|
|
});
|
|
|
|
$anvil->nice_exit({exit_code => 3});
|
|
|
|
}
|
|
|
|
elsif (not $anvil->data->{switches}{to})
|
|
|
|
{
|
|
|
|
# We need a target
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0050", variables => { file => $anvil->data->{switches}{file} }});
|
|
|
|
$anvil->Job->update_progress({
|
|
|
|
progress => 0,
|
|
|
|
message => "error_0050,!!file!".$anvil->data->{switches}{file}."!!",
|
|
|
|
job_uuid => $anvil->data->{jobs}{'job-uuid'},
|
|
|
|
});
|
|
|
|
$anvil->nice_exit({exit_code => 4});
|
|
|
|
}
|
|
|
|
elsif (not -f $anvil->data->{switches}{file})
|
|
|
|
{
|
|
|
|
# The file to rename doesn't exist.
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0051", variables => { file => $anvil->data->{switches}{file} }});
|
|
|
|
$anvil->Job->update_progress({
|
|
|
|
progress => 0,
|
|
|
|
message => "error_0051,!!file!".$anvil->data->{switches}{file}."!!",
|
|
|
|
job_uuid => $anvil->data->{jobs}{'job-uuid'},
|
|
|
|
});
|
|
|
|
$anvil->nice_exit({exit_code => 5});
|
|
|
|
}
|
|
|
|
elsif (-e $anvil->data->{switches}{to})
|
|
|
|
{
|
|
|
|
# There's already a file (or directory or something) with that name.
|
|
|
|
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0056", variables => { file => $anvil->data->{switches}{file}, to => $anvil->data->{switches}{to} }});
|
|
|
|
$anvil->Job->update_progress({
|
|
|
|
progress => 0,
|
|
|
|
message => "error_0056,!!file!".$anvil->data->{switches}{file}."!!,!!to!".$anvil->data->{switches}{to}."!!",
|
|
|
|
job_uuid => $anvil->data->{jobs}{'job-uuid'},
|
|
|
|
});
|
|
|
|
$anvil->nice_exit({exit_code => 6});
|
|
|
|
}
|
|
|
|
|
|
|
|
### TODO: Left off here
|
|
|
|
|
|
|
|
return(0);
|
|
|
|
}
|