seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Local modifications to ClusterLabs/Anvil by Alteeve
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
594 Commits
2 Branches
0 Tags
21 MiB
 
 
 
 
 
 
Tag: Branch: Tree: 8f9892650b
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '8f9892650b'
${ noResults }
anvil/tools/anvil-manage-files

1242 lines
46 KiB
Raw Blame History

#!/usr/bin/perl
#
# This handles moving around and managing files on Anvil! nodes, DR hosts and Striker dashboards.
#
# When this is called (periodically by the daemon of after an upload / ISO generation);
# - 1. Check 'incoming/' for files. For any found, generate an md5sum and see if the file name and sum match
# anything in the database from any host;
# - If so, update/add an entry in 'file_locations'
# - If not, create a new entry in 'files' and then add the first entry in 'file_locations'
# - 2. Check 'file_locations' for any files on this system, and verify they exist still.
# - If not, check the other files for one with a matching md5sum. If found, handle as a rename.
# - If not found at all, search for the file according to the search rules and copy to here if found.
# - If not found anywhere, remove it from 'file_locations' and send an alert.
# - If found, check the size. If it differs, recalculate the md5sum.
# - 3. If called with '--rename --file <filename> --to <newname>', rename the file and update 'files'.
# - 4. If called with '--delete', remove from 'file_locations' and then remove from the local storage. If
# also used with '--everywhere', then all copies on all systems we know about will be deleted. This is
# done by registering a job against all known hosts. As such, if this is called and the target file
# doesn't exist, it just clears the job and then exits.
# - 5. If called with '--is-script=[0|1]', mark as 'script' in the 'files' table and set/remove the executable bit.
#
# Exit codes;
# 0 = Normal exit or md5sum of this program changed and it exited to reload.
# 1 = No databases available.
# 2 = Another process that is still running has picked up this job.
# 3 = No filename specified when needed by another switch.
# 4 = Request to change the file name but the new name wasn't given.
# 5 = The file specified was not found.
# 6 = The file to delete is not under '/mnt/shared/'.
#
# TODO:
# -
#
# NOTE:
# -
#
use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
# Disable buffering
$| = 1;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
$anvil->data->{switches}{'delete'} = "";
$anvil->data->{switches}{download} = "";
$anvil->data->{switches}{everywhere} = "";
$anvil->data->{switches}{file} = "";
$anvil->data->{switches}{'is-script'} = "";
$anvil->data->{switches}{'job-uuid'} = "";
$anvil->data->{switches}{'rename'} = "";
$anvil->data->{switches}{to} = "";
$anvil->Get->switches;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"switches::delete" => $anvil->data->{switches}{'delete'},
"switches::download" => $anvil->data->{switches}{download},
"switches::everywhere" => $anvil->data->{switches}{everywhere},
"switches::file" => $anvil->data->{switches}{file},
"switches::is-script" => $anvil->data->{switches}{'is-script'},
"switches::job-uuid" => $anvil->data->{switches}{'job-uuid'},
"switches::rename" => $anvil->data->{switches}{'rename'},
"switches::to" => $anvil->data->{switches}{to},
}});
# Connect or die
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0003"});
$anvil->nice_exit({exit_code => 1});
}
# If we have a job_uuid, pick it up.
if ($anvil->data->{switches}{'job-uuid'})
{
my $return = $anvil->Job->get_job_details({check => 1, job_uuid => $anvil->data->{switches}{'job-uuid'}});
if ($return == 1)
{
# It's not a UUID.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
$anvil->nice_exit({exit_code => 2});
}
if ($return == 2)
{
# This job is being handled by another process that is still alive.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
$anvil->nice_exit({exit_code => 3});
}
# If there's a progress, clear it.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'job::job_progress' => $anvil->data->{job}{job_progress} }});
if ($anvil->data->{job}{job_progress})
{
$anvil->Job->update_progress({
progress => 1,
message => "clear",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
}
}
# What are we doing?
if ($anvil->data->{switches}{'rename'})
{
handle_rename($anvil);
}
elsif ($anvil->data->{switches}{'delete'})
{
handle_delete($anvil);
}
elsif ($anvil->data->{switches}{'is-script'})
{
handle_script($anvil);
}
else
{
# Check for files scheduled for deletion.
check_for_deletes($anvil);
# Check for new files
check_incoming($anvil);
# Check for files we should have but don't yet have.
find_missing_files($anvil);
}
# We're done
$anvil->nice_exit({exit_code => 0});
#############################################################################################################
# Private functions. #
#############################################################################################################
# This looks to see if there are any entries in 'file_locations' for us that, the pointed to file, doesn't
# exist on this machine. For those entries, we will search for the file on other machines. The one exception
# is server definition files. For those, we write the file out from the server's 'definition' table entry.
#
# When a missing entry is found, and an entry doesn't exist, the file will be found (if possible) and copied
# to this host. Only machines on the same subnet are searched. The search pattern is;
#
# Nodes; 1. Check for the file on the peer.
# 2. Check for the file on Strikers, in alphabetical order.
# 3. Check for the file on DR host, if available.
# 4. Check other nodes, in alphabetical order.
# 5. Check other DR hosts, in alphabetical order.
# Striker; 1. Check for the file on other Strikers, in alphabetical order.
# 2. Check for the file on DR hosts, if available
# 3. Check for the file on Anvil! nodes.
# DR Host; 1. Check for the file on Strikers, in alphabetical order.
# 2. Check for the file on Anvil! nodes.
# * If a file can't be found, it will try again every so often until it is found.
# * When a file is found, it is copied to '/mnt/shared/incoming'. Only when the file has arrived and the
# md5sum matches. At this point, it is moved into the proper directory.
sub find_missing_files
{
my ($anvil) = @_;
# What am I? This will impact how missing files are found.
my $query = "
SELECT
a.file_uuid,
a.file_directory,
a.file_name,
a.file_size,
a.file_md5sum
FROM
files a,
hosts b,
file_locations c
WHERE
b.host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
AND
a.file_uuid = c.file_location_file_uuid
AND
b.host_uuid = c.file_location_host_uuid
ORDER BY
a.file_directory ASC,
a.file_name ASC,
b.host_name ASC
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $file_uuid = $row->[0];
my $file_directory = $row->[1];
my $file_name = $row->[2];
my $file_size = $row->[3];
my $file_md5sum = $row->[4];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:file_uuid' => $file_uuid,
's2:file_directory' => $file_directory,
's3:file_name' => $file_name,
}});
my $test_file = $file_directory."/".$file_name;
$test_file =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { test_file => $test_file }});
if (not -e $test_file)
{
# Missing file!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0269", variables => { file => $test_file }});
# Find what target, if any, we'll the file from.
my ($found) = find_file($anvil, $file_uuid, $test_file, $file_size, $file_md5sum);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
}
}
### TODO: Left off here.
return(0);
}
### TODO: Add sorting by preferred target
# This looks for a file on another system. The exact order of search depends on what kind of machine we are.
sub find_file
{
my ($anvil, $file_uuid, $full_path, $file_size, $file_md5sum) = @_;
my $found = 0;
# What are my IPs?
my $local_host = $anvil->Get->short_host_name();
$anvil->Network->get_ips();
foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
{
next if not $anvil->data->{network}{$local_host}{interface}{$interface}{ip};
next if not $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask};
my $ip = $anvil->data->{network}{$local_host}{interface}{$interface}{ip};
my $subnet_mask = $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask};
my $network = $anvil->Network->get_network({ip => $ip, subnet_mask => $subnet_mask});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:interface' => $interface,
's2:ip' => $ip,
's3:subnet_mask' => $subnet_mask,
's4:network' => $network,
}});
my $type = "other";
my $sort = $interface;
if ($interface =~ /^((?:bc|s|if)n)(\d+)/)
{
$type = $1;
$sort = $2;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:type' => $type,
's2:sort' => $sort,
}});
}
$anvil->data->{local_ip}{by_network}{$network}{type} = $type;
$anvil->data->{local_ip}{by_network}{$network}{'sort'} = $sort;
$anvil->data->{local_ip}{by_network}{$network}{interface} = $interface;
$anvil->data->{local_ip}{by_network}{$network}{ip} = $ip;
$anvil->data->{local_ip}{by_network}{$network}{subnet_mask} = $subnet_mask;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:local_ip::by_network::${network}::type" => $anvil->data->{local_ip}{by_network}{$network}{type},
"s1:local_ip::by_network::${network}::sort" => $anvil->data->{local_ip}{by_network}{$network}{'sort'},
"s2:local_ip::by_network::${network}::interface" => $anvil->data->{local_ip}{by_network}{$network}{interface},
"s3:local_ip::by_network::${network}::ip" => $anvil->data->{local_ip}{by_network}{$network}{ip},
"s4:local_ip::by_network::${network}::subnet_mask" => $anvil->data->{local_ip}{by_network}{$network}{subnet_mask},
}});
}
# Create a hash we can sort through that are on the same subnet_mask as us.
my $query = "
SELECT
a.host_uuid,
a.host_name,
a.host_type,
b.file_directory,
b.file_name,
b.file_size,
b.file_md5sum
FROM
hosts a,
files b,
file_locations c
WHERE
a.host_uuid = c.file_location_host_uuid
AND
b.file_uuid = c.file_location_file_uuid
AND
b.file_uuid = ".$anvil->Database->quote($file_uuid)."
AND
a.host_uuid != ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
ORDER BY
file_mtime DESC;
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $host_uuid = $row->[0];
my $host_name = $row->[1];
my $host_type = $row->[2];
my $file_directory = $row->[3];
my $file_name = $row->[4];
my $file_size = $row->[5];
my $file_md5sum = $row->[6];
my $test_file = $file_directory."/".$file_name;
$test_file =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
host_uuid => $host_uuid,
host_name => $host_name,
host_type => $host_type,
file_directory => $file_directory,
file_name => $file_name,
file_size => $file_size,
file_md5sum => $file_md5sum,
test_file => $test_file,
}});
# What IP addresses are on this machine?
my $query = "
SELECT
ip_address_address,
ip_address_subnet_mask
FROM
ip_addresses
WHERE
ip_address_note != 'DELETED'
AND
ip_address_host_uuid = ".$anvil->Database->quote($host_uuid)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $ip_address_address = $row->[0];
my $ip_address_subnet_mask = $row->[1];
my $network = $anvil->Network->get_network({ip => $ip_address_address, subnet_mask => $ip_address_subnet_mask});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
ip_address_address => $ip_address_address,
ip_address_subnet_mask => $ip_address_subnet_mask,
network => $network,
}});
# Are we on the same subnet?
if (exists $anvil->data->{local_ip}{by_network}{$network})
{
# We're on the same subnet!
my $type = $anvil->data->{local_ip}{by_network}{$network}{type};
my $sort = $anvil->data->{local_ip}{by_network}{$network}{'sort'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:type" => $type,
"s2:sort" => $sort,
}});
# Record.
$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip} = $ip_address_address;
$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name} = $host_name;
$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid} = $host_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:peer_ip::${host_type}::${type}::${sort}::name" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name},
"s2:peer_ip::${host_type}::${type}::${sort}::ip" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip},
"s3:peer_ip::${host_type}::${type}::${sort}::host_uuid" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid},
}});
}
}
}
### TODO: Do this according to what we are.
# Sort through what we've found.
my $file_found = 0;
my $searched_hosts = {};
foreach my $host_type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}})
{
last if $file_found;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }});
foreach my $type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}})
{
last if $file_found;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { type => $type }});
foreach my $sort (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}{$type}})
{
last if $file_found;
my $ip = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip};
my $name = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name};
my $host_uuid = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid};
my $remote_user = "admin";
my $password = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:host_type' => $host_type,
's2:type' => $type,
's3:sort' => $sort,
's4:name' => $name,
's5:ip' => $ip,
's6:host_uuid' => $host_uuid,
's7:remote_user' => $remote_user,
's8:password' => $password,
}});
if ((exists $searched_hosts->{$name}) && ($searched_hosts->{$name}))
{
# Already searched this host, this is just a different IP. Skip it.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "log_0282", variables => {
host_name => $name,
ip => $ip,
}});
next;
}
$searched_hosts->{$name} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { "searched_hosts->{$name}" => $searched_hosts->{$name} }});
### NOTE: There's a bug in Net::SSH2 on RHEL8 where passwordless SSH doesn't
### work. So for now, we'll manually pull in passwords from the
### anvil.conf using 'hosts::<host_name>::password' or
### 'hosts::<host_uuid>::password'. This will be removed when the bug
### is fixed.
if ((exists $anvil->data->{hosts}{$host_uuid}{password}{$remote_user}) && ($anvil->data->{hosts}{$host_uuid}{password}{$remote_user}))
{
$password = $anvil->data->{hosts}{$host_uuid}{password}{$remote_user};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }});
}
elsif ((exists $anvil->data->{hosts}{$name}{password}{$remote_user}) && ($anvil->data->{hosts}{$name}{password}{$remote_user}))
{
$password = $anvil->data->{hosts}{$host_uuid}{password};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }});
}
### NOTE: This might take a while for the call to return if we're md5sum'ing
### a large file like an ISO.
### TODO: Should we ask for the size, then follow up with the md5sum for
### files that are over a certain size? Or will this get called rarely
### enough in practice that it doesn't really matter?
# If the file is found, we'll parse these out.
my $remote_size = 0;
my $remote_md5sum = "";
my ($output, $error, $return_code) = $anvil->Remote->call({
shell_call => $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path." --with-md5sum",
remote_user => $remote_user,
password => $password,
target => $ip,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
error => $error,
output => $output,
return_code => $return_code,
}});
foreach my $line (split/\n/, $output)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
if ($line =~ /^size: \[(\d+)\]$/)
{
$remote_size = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_size => $remote_size }});
}
if ($line =~ /^md5sum: \[(.*)\]$/)
{
$remote_md5sum = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_md5sum => $remote_md5sum }});
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:remote_size' => $remote_size,
's2:file_size' => $file_size,
's3:remote_md5sum' => $remote_md5sum,
's4:file_md5sum' => $file_md5sum,
}});
### Do I really need to match sizes if the md5sum is the same?
if (($remote_size eq $file_size) && ($remote_md5sum eq $file_md5sum))
{
# Pull it over!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0276", variables => {
file => $full_path,
host_name => $name,
ip => $ip,
}});
my $failed = $anvil->Storage->rsync({
debug => 2,
destination => $full_path,
password => $password,
source => $remote_user."\@".$ip.":".$full_path,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
if (-f $full_path)
{
# Got it!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0277", variables => { file => $full_path }});
# Verify the md5sum.
my $local_md5sum = $anvil->Get->md5sum({file => $full_path});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
local_md5sum => $local_md5sum,
file_md5sum => $file_md5sum,
}});
if ($local_md5sum eq $file_md5sum)
{
# Success!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0278", variables => { file => $full_path }});
$file_found = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_found => $file_found }});
last;
}
else
{
# Failed. :(
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0279", variables => { file => $full_path }});
unlink $full_path;
}
}
else
{
# Failed to rsync.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0280", variables => {
file => $full_path,
host_name => $name,
ip => $ip,
}});
}
}
else
{
# Doesn't match what we're looking for.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0281", variables => {
file => $full_path,
host_name => $name,
ip => $ip,
remote_md5sum => $remote_md5sum,
file_md5sum => $file_md5sum,
file_size => $file_size,
say_file_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
remote_size => $remote_size,
say_remote_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}),
}});
}
}
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
return($found);
}
# This handles deleting a file.
sub check_incoming
{
my ($anvil) = @_;
# This hash shouldn't exist, but lets be safe...
if (exists $anvil->data->{scan}{directories})
{
delete $anvil->data->{scan}{directories};
}
# Read any files in '/mnt/shared/incoming'.
$anvil->Storage->scan_directory({
debug => 3,
directory => $anvil->data->{path}{directories}{shared}{base},
recursive => 1,
});
my $incoming_directory = $anvil->data->{path}{directories}{shared}{incoming};
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0264"});
foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}})
{
# Skip this if it's under '/mnt/shared/temp' (that's used for files being downloaded, etc)
next if $full_path =~ /^\/mnt\/shared\/temp\//;
# Skip if this isn't a file.
my $file_type = $anvil->data->{scan}{directories}{$full_path}{type};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
full_path => $full_path,
file_type => $file_type,
}});
next if $file_type ne "file";
my $file_name = $anvil->data->{scan}{directories}{$full_path}{name};
my $file_directory = $anvil->data->{scan}{directories}{$full_path}{directory};
my $file_size = $anvil->data->{scan}{directories}{$full_path}{size};
my $file_mtime = $anvil->data->{scan}{directories}{$full_path}{mtime};
my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype};
my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0;
my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
file_name => $file_name,
file_size => $file_size,
file_mtime => $file_mtime,
file_mimetype => $file_mimetype,
file_executable => $file_executable,
say_mimetype => $say_mimetype,
}});
# Do I know about this file? If so, is the file the same size? If either is no, calculate the md5sum.
my ($file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum) = get_file_db_info($anvil, "", $file_name);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
file_uuid => $file_uuid,
recorded_size => $recorded_size,
recorded_mtime => $recorded_mtime,
recorded_md5sum => $recorded_md5sum,
}});
# Calculate the md5sum?
my $file_md5sum = $recorded_md5sum;
if ((not $file_uuid) or ($file_size != $recorded_size))
{
# Yes. But first, do we have a size mismatch? If so, see if we need to pull a newer
# version down from elsewhere.
if (($file_uuid) && ($file_mtime <= $recorded_mtime))
{
# We've got an older file, we need to update.
pull_file($anvil, $file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum);
# TODO: Now see if it exists and, if it does, re-stat it. If not, loop to the
# next file and skip this one.
}
# Now generate the md5sum. If this file is over 128 MiB, warn the user that it might
# take a while.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0265", variables => { file => $full_path }});
if ($file_size > (128 * (2 ** 20)))
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0266", variables => {
size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
}});
}
# Update (or get) the md5sum.
$file_md5sum = $anvil->Get->md5sum({debug => 2, file => $full_path});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
say_mimetype => $say_mimetype,
file_md5sum => $file_md5sum,
}});
# Insert or update the files entry.
($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 3,
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
}
# If we still don't have a file UUID for some reason, skip this file.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
next if not $file_uuid;
# Make sure we know about this file on this system
my ($file_locatiom_uuid) = $anvil->Database->insert_or_update_file_locations({
debug => 3,
file_location_file_uuid => $file_uuid,
file_location_host_uuid => $anvil->data->{sys}{host_uuid},
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_locatiom_uuid => $file_locatiom_uuid }});
# Are we in the incoming directory? If so, move the file.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
full_path => $full_path,
incoming_directory => $incoming_directory,
}});
if ($full_path =~ /^$incoming_directory/)
{
# If it's a definition file, we'll move it to
# 'path::directories::shared::definitions', otherwise we'll move it to
# 'path::directories::shared::files'.
my $target = $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files};
$target .= "/";
$target =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { target => $target }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0268", variables => {
file => $full_path,
target => $target,
}});
$anvil->Storage->move_file({
debug => 3,
source_file => $full_path,
target_file => $target,
});
# Update the file_directory.
my $say_directory =~ s/\/$//;
($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 3,
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $say_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
}
}
return(0);
}
# This method finds a file elsewhere on the network and pulls it to here.
sub pull_file
{
my ($anvil, $file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum) = @_;
$file_uuid = "" if not defined $file_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_uuid => $file_uuid,
recorded_size => $recorded_size,
recorded_mtime => $recorded_mtime,
recorded_md5sum => $recorded_md5sum,
}});
# Find the hosts with this file, then connect to it to see if the size is the same as what we want.
# If so, pull it down...
### TODO
return(0);
}
# This gets the file_uuid for a given file name and/or md5sum. If the file isn't found, an empty string is
# returned. If it is found, the file size as recorded in the database is returned.
sub get_file_db_info
{
my ($anvil, $file_md5sum, $file_name) = @_;
$file_md5sum = "" if not defined $file_md5sum;
$file_name = "" if not defined $file_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
file_md5sum => $file_md5sum,
file_name => $file_name,
}});
# Get the file size and file uuid, if possible.
# If I have the md5sum, search using that. If I have the filename only, then we'll fall back to that.
my $query = "
SELECT
file_uuid,
file_size,
file_mtime,
file_md5sum
FROM
files
WHERE
";
if ($file_md5sum)
{
$query .= " file_md5sum = ".$anvil->Database->quote($file_md5sum)."\n";
}
elsif ($file_name)
{
$query .= " file_name = ".$anvil->Database->quote($file_name)."\n";
}
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
if (not $count)
{
# File wasn't found in the database
return("", 0, 0, "");
}
my $file_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : "";
my $file_size = defined $results->[0]->[1] ? $results->[0]->[1] : 0;
my $file_mtime = defined $results->[0]->[2] ? $results->[0]->[2] : 0;
$file_md5sum = defined $results->[0]->[3] ? $results->[0]->[3] : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
file_uuid => $file_uuid,
file_size => $file_size,
file_mtime => $file_mtime,
file_md5sum => $file_md5sum,
}});
return($file_uuid, $file_size, $file_mtime, $file_md5sum);
}
# This handles toggling a file to marked or unmarked as a script.
sub handle_script
{
my ($anvil) = @_;
if (not $anvil->data->{switches}{file})
{
# Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0055"});
$anvil->Job->update_progress({
progress => 0,
message => "error_0055",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 3});
}
# Find the file_uuid (we don't actually care about the file size, mtime or md5sum).
my ($file_uuid, $file_size, $recorded_mtime, $file_md5sum) = get_file_db_info($anvil, "", $anvil->data->{switches}{file});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
# Toggle the executable bits.
my $executable = 0;
if ($anvil->data->{switches}{'is-script'})
{
# Is it already executable?
if (-x $anvil->data->{switches}{file})
{
# Switch it on
$executable = 1;
$anvil->Storage->change_mode({path => $anvil->data->{switches}{file}, mode => "a+x"});
}
else
{
# Already a script.
}
}
else
{
# Is it executable?
if (-x $anvil->data->{switches}{file})
{
# Switch it off.
$executable = 1;
$anvil->Storage->change_mode({path => $anvil->data->{switches}{file}, mode => "a-x"});
}
else
{
# Already not a script.
}
}
# If we have a file UUID, update the 'file_type
if ($file_uuid)
{
# Load the details.
my $query = "
SELECT
file_name,
file_directory,
file_size,
file_md5sum,
file_type,
file_mtime
FROM
files
WHERE
file_uuid = ".$anvil->Database->quote($file_uuid)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results }});
my $file_name = $results->[0]->[0];
my $file_directory = $results->[0]->[1];
my $file_size = $results->[0]->[2];
my $file_md5sum = $results->[0]->[3];
my $file_type = $results->[0]->[4];
my $file_mtime = $results->[0]->[5];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_name => $file_name,
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => $file_type,
file_mtime => $file_mtime,
}});
if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'}))
{
# Figure out what the file type is and update.
my $mimetype = mimetype($file_name);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mimetype => $mimetype }});
my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, 0);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }});
$anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file},
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype,
});
}
elsif (($file_type ne "script") && ($anvil->data->{switches}{'is-script'}))
{
# Change the file tpye to "script"
$anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file},
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => "script",
});
}
}
return(0);
}
# This takes the mimetype as reported by the 'mimetype' method and returns a file type we care about.
sub convert_mimetype
{
my ($anvil, $mimetype, $file, $executable) = @_;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
mimetype => $mimetype,
file => $file,
executable => $executable,
}});
my $say_mimetype = "other";
if ($mimetype)
{
if ($mimetype =~ /cd-image/)
{
$say_mimetype = "iso";
}
elsif ($mimetype =~ /xml/)
{
# This might be a definition, but look inside it to be sure.
my ($is_domain, $return_code) = $anvil->System->call({debug => 3, shell_call => "if \$(".$anvil->data->{path}{exe}{'grep'}." -q '</domain>' ".$file."); then ".$anvil->data->{path}{exe}{echo}." 1; else ".$anvil->data->{path}{exe}{echo}." 0; fi" });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { is_domain => $is_domain, return_code => $return_code }});
if ($is_domain)
{
$say_mimetype = "definition";
}
}
elsif ($mimetype =~ /rpm$/)
{
$say_mimetype = "rpm";
}
elsif ($mimetype =~ /disk-image/)
{
$say_mimetype = "disk-image";
}
elsif ($executable)
{
$say_mimetype = "script";
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { say_mimetype => $say_mimetype }});
return($say_mimetype);
}
# This looks for any files with the file_type of 'DELETED' and that has an entry in file_locations for this
# machine. Any that are found will have their file deleted and the file_locations entry deleted.
sub check_for_deletes
{
my ($anvil) = @_;
my $query = "
SELECT
a.file_uuid,
b.file_location_uuid,
a.file_directory || '/' || a.file_name AS full_path
FROM
files a,
file_locations b
WHERE
a.file_uuid = b.file_location_file_uuid
AND
a.file_type = 'DELETED'
AND
b.file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $file_uuid = $row->[0];
my $file_location_uuid = $row->[1];
my $full_path = $row->[2];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:file_uuid' => $file_uuid,
's2:file_location_uuid' => $file_location_uuid,
's3:full_path' => $full_path,
}});
# Get rid of it (if it actually exists) and then remove it from file_locations.
if (-e $full_path)
{
# Delete it.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0285", variables => { file => $full_path }});
unlink $full_path;
# Sleep and verify
sleep 1;
if (-e $full_path)
{
# Failed to delete...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0284", variables => { file => $full_path }});
return("");
}
else
{
# Deleted successfully.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0283"});
}
}
# Delete the entry from file_locations, if needed.
$query = "DELETE FROM file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
}
return(0);
}
# This handles deleting a file. If the requested deletion target doesn't exist, we'll just clear the
# database. If that doesn't exist either, we still don't error. This is to handle broad requests to delete a
# file everywhere. If we're asked to delete it everywhere, then we'll register a job against all hosts.
sub handle_delete
{
my ($anvil) = @_;
my $full_path = $anvil->data->{switches}{file};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { full_path => $full_path }});
if (not $full_path)
{
# Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"});
$anvil->Job->update_progress({
progress => 0,
message => "error_0052",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 3});
}
elsif ($full_path !~ /^\/mnt\/shared\//)
{
# We don't do that here...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0053", variables => { file => $full_path }});
$anvil->Job->update_progress({
progress => 0,
message => "error_0053,!!file!".$full_path."!!",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 6});
}
# What's the UUID of this file? I collect all the data for the update, if appropriate.
my $file_name = ($full_path =~ /^.*\/(.*)$/)[0];
my $file_directory = ($full_path =~ /^(.*?)\/$file_name$/)[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
full_name => $file_name,
file_directory => $file_directory,
}});
my $query = "
SELECT
file_uuid,
file_size,
file_md5sum,
file_type,
file_mtime
FROM
files
WHERE
file_name = ".$anvil->Database->quote($file_name)."
AND
file_directory = ".$anvil->Database->quote($file_directory)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
my $file_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : "";
my $file_size = defined $results->[0]->[1] ? $results->[0]->[1] : "";
my $file_md5sum = defined $results->[0]->[2] ? $results->[0]->[2] : "";
my $file_type = defined $results->[0]->[3] ? $results->[0]->[3] : "";
my $file_mtime = defined $results->[0]->[4] ? $results->[0]->[4] : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_uuid => $file_uuid,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => $file_type,
file_mtime => $file_mtime,
}});
# If I have the file_uuid, and we've been asked to delete it everywhere, mark it as DELETEd in the
# database. This way, if anything goes wrong below, future runs will try (again) to delete the file
# itself. If it's only beind deleted locally, and it fails for some reason, there will be no attempt
# to try again.
if (($file_uuid) && ($anvil->data->{switches}{everywhere}))
{
# Yup.
($file_uuid) = $anvil->Database->insert_or_update_files({
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => "DELETED",
file_mtime => $file_mtime,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
}
# Does the file exist?
if (-e $full_path)
{
# Delete it.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0263", variables => { file => $full_path }});
unlink $full_path;
# Sleep and verify
sleep 1;
if (-e $full_path)
{
# Failed to delete...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0284"});
return("");
}
else
{
# Deleted successfully.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0283"});
}
}
# Now, if I have a file_uuid, delete it from file_locations if it exists.
if ($file_uuid)
{
# Delete the entry from file_locations, if needed.
my $query = "
DELETE FROM
file_locations
WHERE
file_location_file_uuid = ".$anvil->Database->quote($file_uuid)."
AND
file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
}
return(0);
}
# This handles renaming files.
sub handle_rename
{
my ($anvil) = @_;
# Do we have the current file name and the new one?
if (not $anvil->data->{switches}{file})
{
# Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0049"});
$anvil->Job->update_progress({
progress => 0,
message => "error_0049",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 3});
}
elsif (not $anvil->data->{switches}{to})
{
# We need a target
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0050", variables => { file => $anvil->data->{switches}{file} }});
$anvil->Job->update_progress({
progress => 0,
message => "error_0050,!!file!".$anvil->data->{switches}{file}."!!",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 4});
}
elsif (not -f $anvil->data->{switches}{file})
{
# The file to rename doesn't exist.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0051", variables => { file => $anvil->data->{switches}{file} }});
$anvil->Job->update_progress({
progress => 0,
message => "error_0051,!!file!".$anvil->data->{switches}{file}."!!",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 5});
}
elsif (-e $anvil->data->{switches}{to})
{
# There's already a file (or directory or something) with that name.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0056", variables => { file => $anvil->data->{switches}{file}, to => $anvil->data->{switches}{to} }});
$anvil->Job->update_progress({
progress => 0,
message => "error_0056,!!file!".$anvil->data->{switches}{file}."!!,!!to!".$anvil->data->{switches}{to}."!!",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 6});
}
### TODO: Left off here
return(0);
}